{
  "metadata": {
    "name": "ScBench-Long",
    "evalCount": 21,
    "exampleCount": 4,
    "source": "scbench-long-internal",
    "examplesSource": "examples",
    "artifactRunCount": 209,
    "artifactTrajectoryCount": 209,
    "generatedAt": "2026-07-14T21:37:42.111Z",
    "scbenchLongV1Merged": [
      "sclong-sonnet-5-claude-code",
      "sclong-gpt-5-6-luna-codex",
      "sclong-gpt-5-6-sol-codex",
      "sclong-gpt-5-6-terra-codex",
      "sclong-sonnet-5-pi",
      "sclong-gpt-5-6-luna-pi",
      "sclong-gpt-5-6-sol-pi",
      "sclong-gpt-5-6-terra-pi",
      "sclong-grok-4-5-pi",
      "sclong-opus-5-claude-code",
      "sclong-opus-5",
      "sclong-kimi-k3-pi"
    ],
    "verifiedEvalCount": 22
  },
  "modelResults": [
    {
      "id": "sclong-opus-5",
      "modelName": "claude-opus-5",
      "harness": "pi",
      "passRate": 41.27,
      "ci": [
        29.96,
        53.58
      ],
      "passed": 9,
      "runs": 63,
      "avgCost": 3.6899,
      "avgDuration": 2758.55,
      "nEvals": 21,
      "subset": "full"
    },
    {
      "id": "sclong-opus-5-claude-code",
      "modelName": "claude-opus-5",
      "harness": "claude-code",
      "passRate": 39.68,
      "ci": [
        28.53,
        52.02
      ],
      "passed": 8,
      "runs": 63,
      "avgCost": 10.4015,
      "avgDuration": 3907.65,
      "nEvals": 21,
      "subset": "full"
    },
    {
      "id": "sclong-gpt-5-6-sol-pi",
      "modelName": "gpt-5.6-sol",
      "harness": "pi",
      "passRate": 38.1,
      "ci": [
        27.12,
        50.44
      ],
      "passed": 8,
      "runs": 63,
      "avgCost": 2.9788,
      "avgDuration": 1729.21,
      "nEvals": 21,
      "subset": "full"
    },
    {
      "id": "sclong-opus-4-8-claude-code",
      "modelName": "claude-opus-4-8",
      "harness": "claude-code",
      "passRate": 25.4,
      "ci": [
        16.28,
        37.34
      ],
      "passed": 5,
      "runs": 63,
      "avgCost": 6.973,
      "avgDuration": 2231,
      "nEvals": 21,
      "subset": "full"
    },
    {
      "id": "sclong-gpt-5-6-sol-codex",
      "modelName": "gpt-5.6-sol",
      "harness": "openai-codex",
      "passRate": 25.4,
      "ci": [
        16.28,
        37.34
      ],
      "passed": 5,
      "runs": 63,
      "avgCost": 3.4904,
      "avgDuration": 849.23,
      "nEvals": 21,
      "subset": "full"
    },
    {
      "id": "sclong-gpt-5-6-terra-codex",
      "modelName": "gpt-5.6-terra",
      "harness": "openai-codex",
      "passRate": 23.81,
      "ci": [
        14.99,
        35.64
      ],
      "passed": 5,
      "runs": 63,
      "avgCost": 1.4616,
      "avgDuration": 574.08,
      "nEvals": 21,
      "subset": "full"
    },
    {
      "id": "sclong-gemini-3-5-flash-pi",
      "modelName": "gemini-3.5-flash",
      "harness": "pi",
      "passRate": 22.22,
      "ci": [
        13.72,
        33.91
      ],
      "passed": 5,
      "runs": 63,
      "avgCost": 1.449,
      "avgDuration": 1042.6,
      "nEvals": 21,
      "subset": "full"
    },
    {
      "id": "sclong-opus-4-8-pi",
      "modelName": "claude-opus-4-8",
      "harness": "pi",
      "passRate": 20.63,
      "ci": [
        12.48,
        32.17
      ],
      "passed": 4,
      "runs": 63,
      "avgCost": 3.4334,
      "avgDuration": 2069.16,
      "nEvals": 21,
      "subset": "full"
    },
    {
      "id": "sclong-gpt-5-6-terra-pi",
      "modelName": "gpt-5.6-terra",
      "harness": "pi",
      "passRate": 20.63,
      "ci": [
        12.48,
        32.17
      ],
      "passed": 4,
      "runs": 63,
      "avgCost": 1.3807,
      "avgDuration": 1374.82,
      "nEvals": 21,
      "subset": "full"
    },
    {
      "id": "sclong-gpt-5-6-luna-pi",
      "modelName": "gpt-5.6-luna",
      "harness": "pi",
      "passRate": 19.05,
      "ci": [
        11.25,
        30.41
      ],
      "passed": 4,
      "runs": 63,
      "avgCost": 0.8527,
      "avgDuration": 1876.93,
      "nEvals": 21,
      "subset": "full"
    },
    {
      "id": "sclong-gemini-3-1-pro-preview-pi",
      "modelName": "gemini-3.1-pro-preview",
      "harness": "pi",
      "passRate": 17.46,
      "ci": [
        10.04,
        28.62
      ],
      "passed": 4,
      "runs": 63,
      "avgCost": 0.7942,
      "avgDuration": 1128.38,
      "nEvals": 21,
      "subset": "full"
    },
    {
      "id": "sclong-gpt-5-5-codex",
      "modelName": "gpt-5.5",
      "harness": "openai-codex",
      "passRate": 15.87,
      "ci": [
        8.86,
        26.81
      ],
      "passed": 3,
      "runs": 63,
      "avgCost": 2.2401,
      "avgDuration": 934.52,
      "nEvals": 21,
      "subset": "full"
    },
    {
      "id": "sclong-gpt-5-5-pi",
      "modelName": "gpt-5.5",
      "harness": "pi",
      "passRate": 15.87,
      "ci": [
        8.86,
        26.81
      ],
      "passed": 3,
      "runs": 63,
      "avgCost": 2.3331,
      "avgDuration": 1576.54,
      "nEvals": 21,
      "subset": "full"
    },
    {
      "id": "sclong-gpt-5-6-luna-codex",
      "modelName": "gpt-5.6-luna",
      "harness": "openai-codex",
      "passRate": 14.29,
      "ci": [
        7.7,
        24.97
      ],
      "passed": 3,
      "runs": 63,
      "avgCost": 0.9944,
      "avgDuration": 782.48,
      "nEvals": 21,
      "subset": "full"
    },
    {
      "id": "sclong-sonnet-5-pi",
      "modelName": "claude-sonnet-5",
      "harness": "pi",
      "passRate": 14.29,
      "ci": [
        7.7,
        24.97
      ],
      "passed": 3,
      "runs": 63,
      "avgCost": 1.519,
      "avgDuration": 2418.62,
      "nEvals": 21,
      "subset": "full"
    },
    {
      "id": "sclong-sonnet-5-claude-code",
      "modelName": "claude-sonnet-5",
      "harness": "claude-code",
      "passRate": 12.7,
      "ci": [
        6.58,
        23.11
      ],
      "passed": 3,
      "runs": 63,
      "avgCost": 6.6667,
      "avgDuration": 3512.74,
      "nEvals": 21,
      "subset": "full"
    },
    {
      "id": "sclong-grok-4-5-pi",
      "modelName": "grok-4.5",
      "harness": "pi",
      "passRate": 12.7,
      "ci": [
        6.58,
        23.11
      ],
      "passed": 3,
      "runs": 63,
      "avgCost": 0.6266,
      "avgDuration": 1961.25,
      "nEvals": 21,
      "subset": "full"
    },
    {
      "id": "sclong-kimi-k3-pi",
      "modelName": "kimi-k3",
      "harness": "pi",
      "passRate": 12.7,
      "ci": [
        6.58,
        23.11
      ],
      "passed": 3,
      "runs": 63,
      "avgCost": 1.8131,
      "avgDuration": 3087.81,
      "nEvals": 21,
      "subset": "full"
    },
    {
      "id": "sclong-opus-4-7-claude-code",
      "modelName": "claude-opus-4-7",
      "harness": "claude-code",
      "passRate": 11.11,
      "ci": [
        5.49,
        21.2
      ],
      "passed": 2,
      "runs": 63,
      "avgCost": 4.285,
      "avgDuration": 1352.44,
      "nEvals": 21,
      "subset": "full"
    },
    {
      "id": "sclong-glm-5p2-pi",
      "modelName": "glm-5p2",
      "harness": "pi",
      "passRate": 9.52,
      "ci": [
        4.44,
        19.26
      ],
      "passed": 2,
      "runs": 63,
      "avgCost": 3.3857,
      "avgDuration": 3894.83,
      "nEvals": 21,
      "subset": "full"
    },
    {
      "id": "sclong-opus-4-7-pi",
      "modelName": "claude-opus-4-7",
      "harness": "pi",
      "passRate": 9.52,
      "ci": [
        4.44,
        19.26
      ],
      "passed": 2,
      "runs": 63,
      "avgCost": 2.157,
      "avgDuration": 1569.17,
      "nEvals": 21,
      "subset": "full"
    },
    {
      "id": "sclong-opus-4-6-pi",
      "modelName": "claude-opus-4-6",
      "harness": "pi",
      "passRate": 9.52,
      "ci": [
        4.44,
        19.26
      ],
      "passed": 2,
      "runs": 63,
      "avgCost": 2.1951,
      "avgDuration": 2124.76,
      "nEvals": 21,
      "subset": "full"
    },
    {
      "id": "sclong-gpt-5-4-pi",
      "modelName": "gpt-5.4",
      "harness": "pi",
      "passRate": 9.52,
      "ci": [
        4.44,
        19.26
      ],
      "passed": 2,
      "runs": 63,
      "avgCost": 1.5224,
      "avgDuration": 2565.92,
      "nEvals": 21,
      "subset": "full"
    },
    {
      "id": "sclong-opus-4-6-claude-code",
      "modelName": "claude-opus-4-6",
      "harness": "claude-code",
      "passRate": 7.94,
      "ci": [
        3.44,
        17.27
      ],
      "passed": 2,
      "runs": 63,
      "avgCost": 2.3671,
      "avgDuration": 2013.79,
      "nEvals": 21,
      "subset": "full"
    },
    {
      "id": "sclong-sonnet-4-6-pi",
      "modelName": "claude-sonnet-4-6",
      "harness": "pi",
      "passRate": 7.94,
      "ci": [
        3.44,
        17.27
      ],
      "passed": 2,
      "runs": 63,
      "avgCost": 1.9379,
      "avgDuration": 2374.27,
      "nEvals": 21,
      "subset": "full"
    },
    {
      "id": "sclong-grok-4-20-beta-reasoning-pi",
      "modelName": "grok-4.20-beta-0309-reasoning",
      "harness": "pi",
      "passRate": 6.35,
      "ci": [
        2.5,
        15.22
      ],
      "passed": 1,
      "runs": 63,
      "avgCost": 0.0663,
      "avgDuration": 536.11,
      "nEvals": 21,
      "subset": "full"
    },
    {
      "id": "sclong-kimi-k2p6-pi",
      "modelName": "kimi-k2p6",
      "harness": "pi",
      "passRate": 4.76,
      "ci": [
        1.63,
        13.09
      ],
      "passed": 1,
      "runs": 63,
      "avgCost": 0.7916,
      "avgDuration": 2472.18,
      "nEvals": 21,
      "subset": "full"
    },
    {
      "id": "sclong-gpt-5-4-codex",
      "modelName": "gpt-5.4",
      "harness": "openai-codex",
      "passRate": 3.17,
      "ci": [
        0.87,
        10.86
      ],
      "passed": 1,
      "runs": 63,
      "avgCost": 2.1831,
      "avgDuration": 1039.97,
      "nEvals": 21,
      "subset": "full"
    },
    {
      "id": "sclong-grok-4-3-pi",
      "modelName": "grok-4.3",
      "harness": "pi",
      "passRate": 1.59,
      "ci": [
        0.28,
        8.46
      ],
      "passed": 0,
      "runs": 63,
      "avgCost": 0.0205,
      "avgDuration": 215.44,
      "nEvals": 21,
      "subset": "full"
    }
  ],
  "runResults": [
    {
      "modelId": "sclong-opus-4-6-claude-code",
      "modelName": "claude-opus-4-6",
      "harness": "claude-code",
      "task": "10x_5p_melanoma_cluster_enriched_apc_states_v2",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 1411,
      "cost": 2.1661869999999994,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "top_3_cluster_enriched_apc_states",
            "kind": "list",
            "matchKey": "apc_state",
            "k": 3,
            "expected": [
              {
                "key": "plasmacytoid_DC",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "T_cell_stimulatory_role_supported",
                      "role_direction_unresolved"
                    ],
                    "canonical": "T_cell_stimulatory_role_supported"
                  }
                }
              },
              {
                "key": "C1qhigh_lipid_associated_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "C1qhigh_inflammatory_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "plasma_cell",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "cohort_significant_per_patient_inconsistent"
                    ],
                    "canonical": "cohort_significant_per_patient_inconsistent"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CD80_CD86": {
                "op": "in",
                "value": [
                  "mreg_DC",
                  "shared_across_APC_states"
                ]
              },
              "PD_L1_in_APC_clusters": {
                "op": "in",
                "value": [
                  "C1qhigh_macrophage_either_subtype",
                  "shared_across_APC_states",
                  "mreg_DC"
                ]
              },
              "type_I_IFN": {
                "op": "in",
                "value": [
                  "plasmacytoid_DC",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "cf_mreg_dc_claimed_enriched",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "mreg_DC",
              "mature_DC_enriched_in_immunoregulatory_molecules"
            ]
          },
          {
            "name": "cf_framework_offcompartment_or_generic_substitution",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "cDC1",
              "Maier_mregDC",
              "Cheng_C1QC_TAM",
              "Cheng_SPP1_TAM",
              "Cheng_FOLR2_TAM",
              "Villani_pDC",
              "Zhang_inflammatory_monocyte",
              "LAMP3_pos_DC",
              "myeloid_derived_suppressor_cell",
              "MDSC",
              "tumor_associated_neutrophil",
              "mast_cell",
              "APC",
              "antigen_presenting_cell",
              "macrophage",
              "dendritic_cell",
              "tumor_associated_macrophage",
              "TAM"
            ]
          },
          {
            "name": "cf_direction_commit_on_unresolvable_slot",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_not_resolvable_hedge",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].functional_attribution_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_type_I_IFN_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.type_I_IFN",
            "kind": "value",
            "op": "in",
            "expected": [
              "mreg_DC",
              "C1qhigh_macrophage_either_subtype"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-6-claude-code",
      "modelName": "claude-opus-4-6",
      "harness": "claude-code",
      "task": "10x_5p_melanoma_cluster_enriched_apc_states_v2",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 1668,
      "cost": 2.6551997499999995,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "top_3_cluster_enriched_apc_states",
            "kind": "list",
            "matchKey": "apc_state",
            "k": 3,
            "expected": [
              {
                "key": "plasmacytoid_DC",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "T_cell_stimulatory_role_supported",
                      "role_direction_unresolved"
                    ],
                    "canonical": "T_cell_stimulatory_role_supported"
                  }
                }
              },
              {
                "key": "C1qhigh_lipid_associated_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "C1qhigh_inflammatory_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "plasma_cell",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "cohort_significant_per_patient_inconsistent"
                    ],
                    "canonical": "cohort_significant_per_patient_inconsistent"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CD80_CD86": {
                "op": "in",
                "value": [
                  "mreg_DC",
                  "shared_across_APC_states"
                ]
              },
              "PD_L1_in_APC_clusters": {
                "op": "in",
                "value": [
                  "C1qhigh_macrophage_either_subtype",
                  "shared_across_APC_states",
                  "mreg_DC"
                ]
              },
              "type_I_IFN": {
                "op": "in",
                "value": [
                  "plasmacytoid_DC",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "cf_mreg_dc_claimed_enriched",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "mreg_DC",
              "mature_DC_enriched_in_immunoregulatory_molecules"
            ]
          },
          {
            "name": "cf_framework_offcompartment_or_generic_substitution",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "cDC1",
              "Maier_mregDC",
              "Cheng_C1QC_TAM",
              "Cheng_SPP1_TAM",
              "Cheng_FOLR2_TAM",
              "Villani_pDC",
              "Zhang_inflammatory_monocyte",
              "LAMP3_pos_DC",
              "myeloid_derived_suppressor_cell",
              "MDSC",
              "tumor_associated_neutrophil",
              "mast_cell",
              "APC",
              "antigen_presenting_cell",
              "macrophage",
              "dendritic_cell",
              "tumor_associated_macrophage",
              "TAM"
            ]
          },
          {
            "name": "cf_direction_commit_on_unresolvable_slot",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_not_resolvable_hedge",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].functional_attribution_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_type_I_IFN_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.type_I_IFN",
            "kind": "value",
            "op": "in",
            "expected": [
              "mreg_DC",
              "C1qhigh_macrophage_either_subtype"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-6-claude-code",
      "modelName": "claude-opus-4-6",
      "harness": "claude-code",
      "task": "10x_5p_melanoma_cluster_enriched_apc_states_v2",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": true,
      "score": null,
      "durationS": 1686,
      "cost": 1.4965882499999998,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "top_3_cluster_enriched_apc_states",
            "kind": "list",
            "matchKey": "apc_state",
            "k": 3,
            "expected": [
              {
                "key": "plasmacytoid_DC",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "T_cell_stimulatory_role_supported",
                      "role_direction_unresolved"
                    ],
                    "canonical": "T_cell_stimulatory_role_supported"
                  }
                }
              },
              {
                "key": "C1qhigh_lipid_associated_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "C1qhigh_inflammatory_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "plasma_cell",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "cohort_significant_per_patient_inconsistent"
                    ],
                    "canonical": "cohort_significant_per_patient_inconsistent"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CD80_CD86": {
                "op": "in",
                "value": [
                  "mreg_DC",
                  "shared_across_APC_states"
                ]
              },
              "PD_L1_in_APC_clusters": {
                "op": "in",
                "value": [
                  "C1qhigh_macrophage_either_subtype",
                  "shared_across_APC_states",
                  "mreg_DC"
                ]
              },
              "type_I_IFN": {
                "op": "in",
                "value": [
                  "plasmacytoid_DC",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "cf_mreg_dc_claimed_enriched",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "mreg_DC",
              "mature_DC_enriched_in_immunoregulatory_molecules"
            ]
          },
          {
            "name": "cf_framework_offcompartment_or_generic_substitution",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "cDC1",
              "Maier_mregDC",
              "Cheng_C1QC_TAM",
              "Cheng_SPP1_TAM",
              "Cheng_FOLR2_TAM",
              "Villani_pDC",
              "Zhang_inflammatory_monocyte",
              "LAMP3_pos_DC",
              "myeloid_derived_suppressor_cell",
              "MDSC",
              "tumor_associated_neutrophil",
              "mast_cell",
              "APC",
              "antigen_presenting_cell",
              "macrophage",
              "dendritic_cell",
              "tumor_associated_macrophage",
              "TAM"
            ]
          },
          {
            "name": "cf_direction_commit_on_unresolvable_slot",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_not_resolvable_hedge",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].functional_attribution_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_type_I_IFN_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.type_I_IFN",
            "kind": "value",
            "op": "in",
            "expected": [
              "mreg_DC",
              "C1qhigh_macrophage_either_subtype"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-6-pi",
      "modelName": "claude-opus-4-6",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_enriched_apc_states_v2",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 2477,
      "cost": 3.2328762500000003,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "top_3_cluster_enriched_apc_states",
            "kind": "list",
            "matchKey": "apc_state",
            "k": 3,
            "expected": [
              {
                "key": "plasmacytoid_DC",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "T_cell_stimulatory_role_supported",
                      "role_direction_unresolved"
                    ],
                    "canonical": "T_cell_stimulatory_role_supported"
                  }
                }
              },
              {
                "key": "C1qhigh_lipid_associated_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "C1qhigh_inflammatory_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "plasma_cell",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "cohort_significant_per_patient_inconsistent"
                    ],
                    "canonical": "cohort_significant_per_patient_inconsistent"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CD80_CD86": {
                "op": "in",
                "value": [
                  "mreg_DC",
                  "shared_across_APC_states"
                ]
              },
              "PD_L1_in_APC_clusters": {
                "op": "in",
                "value": [
                  "C1qhigh_macrophage_either_subtype",
                  "shared_across_APC_states",
                  "mreg_DC"
                ]
              },
              "type_I_IFN": {
                "op": "in",
                "value": [
                  "plasmacytoid_DC",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "cf_mreg_dc_claimed_enriched",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "mreg_DC",
              "mature_DC_enriched_in_immunoregulatory_molecules"
            ]
          },
          {
            "name": "cf_framework_offcompartment_or_generic_substitution",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "cDC1",
              "Maier_mregDC",
              "Cheng_C1QC_TAM",
              "Cheng_SPP1_TAM",
              "Cheng_FOLR2_TAM",
              "Villani_pDC",
              "Zhang_inflammatory_monocyte",
              "LAMP3_pos_DC",
              "myeloid_derived_suppressor_cell",
              "MDSC",
              "tumor_associated_neutrophil",
              "mast_cell",
              "APC",
              "antigen_presenting_cell",
              "macrophage",
              "dendritic_cell",
              "tumor_associated_macrophage",
              "TAM"
            ]
          },
          {
            "name": "cf_direction_commit_on_unresolvable_slot",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_not_resolvable_hedge",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].functional_attribution_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_type_I_IFN_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.type_I_IFN",
            "kind": "value",
            "op": "in",
            "expected": [
              "mreg_DC",
              "C1qhigh_macrophage_either_subtype"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-6-pi",
      "modelName": "claude-opus-4-6",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_enriched_apc_states_v2",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 1560,
      "cost": 2.1110325000000003,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "top_3_cluster_enriched_apc_states",
            "kind": "list",
            "matchKey": "apc_state",
            "k": 3,
            "expected": [
              {
                "key": "plasmacytoid_DC",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "T_cell_stimulatory_role_supported",
                      "role_direction_unresolved"
                    ],
                    "canonical": "T_cell_stimulatory_role_supported"
                  }
                }
              },
              {
                "key": "C1qhigh_lipid_associated_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "C1qhigh_inflammatory_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "plasma_cell",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "cohort_significant_per_patient_inconsistent"
                    ],
                    "canonical": "cohort_significant_per_patient_inconsistent"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CD80_CD86": {
                "op": "in",
                "value": [
                  "mreg_DC",
                  "shared_across_APC_states"
                ]
              },
              "PD_L1_in_APC_clusters": {
                "op": "in",
                "value": [
                  "C1qhigh_macrophage_either_subtype",
                  "shared_across_APC_states",
                  "mreg_DC"
                ]
              },
              "type_I_IFN": {
                "op": "in",
                "value": [
                  "plasmacytoid_DC",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "cf_mreg_dc_claimed_enriched",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "mreg_DC",
              "mature_DC_enriched_in_immunoregulatory_molecules"
            ]
          },
          {
            "name": "cf_framework_offcompartment_or_generic_substitution",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "cDC1",
              "Maier_mregDC",
              "Cheng_C1QC_TAM",
              "Cheng_SPP1_TAM",
              "Cheng_FOLR2_TAM",
              "Villani_pDC",
              "Zhang_inflammatory_monocyte",
              "LAMP3_pos_DC",
              "myeloid_derived_suppressor_cell",
              "MDSC",
              "tumor_associated_neutrophil",
              "mast_cell",
              "APC",
              "antigen_presenting_cell",
              "macrophage",
              "dendritic_cell",
              "tumor_associated_macrophage",
              "TAM"
            ]
          },
          {
            "name": "cf_direction_commit_on_unresolvable_slot",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_not_resolvable_hedge",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].functional_attribution_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_type_I_IFN_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.type_I_IFN",
            "kind": "value",
            "op": "in",
            "expected": [
              "mreg_DC",
              "C1qhigh_macrophage_either_subtype"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-6-pi",
      "modelName": "claude-opus-4-6",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_enriched_apc_states_v2",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1136,
      "cost": 1.8104624999999999,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "top_3_cluster_enriched_apc_states",
            "kind": "list",
            "matchKey": "apc_state",
            "k": 3,
            "expected": [
              {
                "key": "plasmacytoid_DC",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "T_cell_stimulatory_role_supported",
                      "role_direction_unresolved"
                    ],
                    "canonical": "T_cell_stimulatory_role_supported"
                  }
                }
              },
              {
                "key": "C1qhigh_lipid_associated_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "C1qhigh_inflammatory_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "plasma_cell",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "cohort_significant_per_patient_inconsistent"
                    ],
                    "canonical": "cohort_significant_per_patient_inconsistent"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CD80_CD86": {
                "op": "in",
                "value": [
                  "mreg_DC",
                  "shared_across_APC_states"
                ]
              },
              "PD_L1_in_APC_clusters": {
                "op": "in",
                "value": [
                  "C1qhigh_macrophage_either_subtype",
                  "shared_across_APC_states",
                  "mreg_DC"
                ]
              },
              "type_I_IFN": {
                "op": "in",
                "value": [
                  "plasmacytoid_DC",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "cf_mreg_dc_claimed_enriched",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "mreg_DC",
              "mature_DC_enriched_in_immunoregulatory_molecules"
            ]
          },
          {
            "name": "cf_framework_offcompartment_or_generic_substitution",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "cDC1",
              "Maier_mregDC",
              "Cheng_C1QC_TAM",
              "Cheng_SPP1_TAM",
              "Cheng_FOLR2_TAM",
              "Villani_pDC",
              "Zhang_inflammatory_monocyte",
              "LAMP3_pos_DC",
              "myeloid_derived_suppressor_cell",
              "MDSC",
              "tumor_associated_neutrophil",
              "mast_cell",
              "APC",
              "antigen_presenting_cell",
              "macrophage",
              "dendritic_cell",
              "tumor_associated_macrophage",
              "TAM"
            ]
          },
          {
            "name": "cf_direction_commit_on_unresolvable_slot",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_not_resolvable_hedge",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].functional_attribution_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_type_I_IFN_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.type_I_IFN",
            "kind": "value",
            "op": "in",
            "expected": [
              "mreg_DC",
              "C1qhigh_macrophage_either_subtype"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-7-claude-code",
      "modelName": "claude-opus-4-7",
      "harness": "claude-code",
      "task": "10x_5p_melanoma_cluster_enriched_apc_states_v2",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 2012,
      "cost": 5.78447875,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "top_3_cluster_enriched_apc_states",
            "kind": "list",
            "matchKey": "apc_state",
            "k": 3,
            "expected": [
              {
                "key": "plasmacytoid_DC",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "T_cell_stimulatory_role_supported",
                      "role_direction_unresolved"
                    ],
                    "canonical": "T_cell_stimulatory_role_supported"
                  }
                }
              },
              {
                "key": "C1qhigh_lipid_associated_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "C1qhigh_inflammatory_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "plasma_cell",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "cohort_significant_per_patient_inconsistent"
                    ],
                    "canonical": "cohort_significant_per_patient_inconsistent"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CD80_CD86": {
                "op": "in",
                "value": [
                  "mreg_DC",
                  "shared_across_APC_states"
                ]
              },
              "PD_L1_in_APC_clusters": {
                "op": "in",
                "value": [
                  "C1qhigh_macrophage_either_subtype",
                  "shared_across_APC_states",
                  "mreg_DC"
                ]
              },
              "type_I_IFN": {
                "op": "in",
                "value": [
                  "plasmacytoid_DC",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "cf_mreg_dc_claimed_enriched",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "mreg_DC",
              "mature_DC_enriched_in_immunoregulatory_molecules"
            ]
          },
          {
            "name": "cf_framework_offcompartment_or_generic_substitution",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "cDC1",
              "Maier_mregDC",
              "Cheng_C1QC_TAM",
              "Cheng_SPP1_TAM",
              "Cheng_FOLR2_TAM",
              "Villani_pDC",
              "Zhang_inflammatory_monocyte",
              "LAMP3_pos_DC",
              "myeloid_derived_suppressor_cell",
              "MDSC",
              "tumor_associated_neutrophil",
              "mast_cell",
              "APC",
              "antigen_presenting_cell",
              "macrophage",
              "dendritic_cell",
              "tumor_associated_macrophage",
              "TAM"
            ]
          },
          {
            "name": "cf_direction_commit_on_unresolvable_slot",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_not_resolvable_hedge",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].functional_attribution_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_type_I_IFN_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.type_I_IFN",
            "kind": "value",
            "op": "in",
            "expected": [
              "mreg_DC",
              "C1qhigh_macrophage_either_subtype"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-7-claude-code",
      "modelName": "claude-opus-4-7",
      "harness": "claude-code",
      "task": "10x_5p_melanoma_cluster_enriched_apc_states_v2",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 1369,
      "cost": 4.268600500000001,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "top_3_cluster_enriched_apc_states",
            "kind": "list",
            "matchKey": "apc_state",
            "k": 3,
            "expected": [
              {
                "key": "plasmacytoid_DC",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "T_cell_stimulatory_role_supported",
                      "role_direction_unresolved"
                    ],
                    "canonical": "T_cell_stimulatory_role_supported"
                  }
                }
              },
              {
                "key": "C1qhigh_lipid_associated_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "C1qhigh_inflammatory_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "plasma_cell",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "cohort_significant_per_patient_inconsistent"
                    ],
                    "canonical": "cohort_significant_per_patient_inconsistent"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CD80_CD86": {
                "op": "in",
                "value": [
                  "mreg_DC",
                  "shared_across_APC_states"
                ]
              },
              "PD_L1_in_APC_clusters": {
                "op": "in",
                "value": [
                  "C1qhigh_macrophage_either_subtype",
                  "shared_across_APC_states",
                  "mreg_DC"
                ]
              },
              "type_I_IFN": {
                "op": "in",
                "value": [
                  "plasmacytoid_DC",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "cf_mreg_dc_claimed_enriched",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "mreg_DC",
              "mature_DC_enriched_in_immunoregulatory_molecules"
            ]
          },
          {
            "name": "cf_framework_offcompartment_or_generic_substitution",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "cDC1",
              "Maier_mregDC",
              "Cheng_C1QC_TAM",
              "Cheng_SPP1_TAM",
              "Cheng_FOLR2_TAM",
              "Villani_pDC",
              "Zhang_inflammatory_monocyte",
              "LAMP3_pos_DC",
              "myeloid_derived_suppressor_cell",
              "MDSC",
              "tumor_associated_neutrophil",
              "mast_cell",
              "APC",
              "antigen_presenting_cell",
              "macrophage",
              "dendritic_cell",
              "tumor_associated_macrophage",
              "TAM"
            ]
          },
          {
            "name": "cf_direction_commit_on_unresolvable_slot",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_not_resolvable_hedge",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].functional_attribution_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_type_I_IFN_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.type_I_IFN",
            "kind": "value",
            "op": "in",
            "expected": [
              "mreg_DC",
              "C1qhigh_macrophage_either_subtype"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-7-claude-code",
      "modelName": "claude-opus-4-7",
      "harness": "claude-code",
      "task": "10x_5p_melanoma_cluster_enriched_apc_states_v2",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1674,
      "cost": 6.42840675,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "top_3_cluster_enriched_apc_states",
            "kind": "list",
            "matchKey": "apc_state",
            "k": 3,
            "expected": [
              {
                "key": "plasmacytoid_DC",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "T_cell_stimulatory_role_supported",
                      "role_direction_unresolved"
                    ],
                    "canonical": "T_cell_stimulatory_role_supported"
                  }
                }
              },
              {
                "key": "C1qhigh_lipid_associated_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "C1qhigh_inflammatory_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "plasma_cell",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "cohort_significant_per_patient_inconsistent"
                    ],
                    "canonical": "cohort_significant_per_patient_inconsistent"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CD80_CD86": {
                "op": "in",
                "value": [
                  "mreg_DC",
                  "shared_across_APC_states"
                ]
              },
              "PD_L1_in_APC_clusters": {
                "op": "in",
                "value": [
                  "C1qhigh_macrophage_either_subtype",
                  "shared_across_APC_states",
                  "mreg_DC"
                ]
              },
              "type_I_IFN": {
                "op": "in",
                "value": [
                  "plasmacytoid_DC",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "cf_mreg_dc_claimed_enriched",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "mreg_DC",
              "mature_DC_enriched_in_immunoregulatory_molecules"
            ]
          },
          {
            "name": "cf_framework_offcompartment_or_generic_substitution",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "cDC1",
              "Maier_mregDC",
              "Cheng_C1QC_TAM",
              "Cheng_SPP1_TAM",
              "Cheng_FOLR2_TAM",
              "Villani_pDC",
              "Zhang_inflammatory_monocyte",
              "LAMP3_pos_DC",
              "myeloid_derived_suppressor_cell",
              "MDSC",
              "tumor_associated_neutrophil",
              "mast_cell",
              "APC",
              "antigen_presenting_cell",
              "macrophage",
              "dendritic_cell",
              "tumor_associated_macrophage",
              "TAM"
            ]
          },
          {
            "name": "cf_direction_commit_on_unresolvable_slot",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_not_resolvable_hedge",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].functional_attribution_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_type_I_IFN_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.type_I_IFN",
            "kind": "value",
            "op": "in",
            "expected": [
              "mreg_DC",
              "C1qhigh_macrophage_either_subtype"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-7-pi",
      "modelName": "claude-opus-4-7",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_enriched_apc_states_v2",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 2362,
      "cost": 4.296908000000001,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "top_3_cluster_enriched_apc_states",
            "kind": "list",
            "matchKey": "apc_state",
            "k": 3,
            "expected": [
              {
                "key": "plasmacytoid_DC",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "T_cell_stimulatory_role_supported",
                      "role_direction_unresolved"
                    ],
                    "canonical": "T_cell_stimulatory_role_supported"
                  }
                }
              },
              {
                "key": "C1qhigh_lipid_associated_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "C1qhigh_inflammatory_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "plasma_cell",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "cohort_significant_per_patient_inconsistent"
                    ],
                    "canonical": "cohort_significant_per_patient_inconsistent"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CD80_CD86": {
                "op": "in",
                "value": [
                  "mreg_DC",
                  "shared_across_APC_states"
                ]
              },
              "PD_L1_in_APC_clusters": {
                "op": "in",
                "value": [
                  "C1qhigh_macrophage_either_subtype",
                  "shared_across_APC_states",
                  "mreg_DC"
                ]
              },
              "type_I_IFN": {
                "op": "in",
                "value": [
                  "plasmacytoid_DC",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "cf_mreg_dc_claimed_enriched",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "mreg_DC",
              "mature_DC_enriched_in_immunoregulatory_molecules"
            ]
          },
          {
            "name": "cf_framework_offcompartment_or_generic_substitution",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "cDC1",
              "Maier_mregDC",
              "Cheng_C1QC_TAM",
              "Cheng_SPP1_TAM",
              "Cheng_FOLR2_TAM",
              "Villani_pDC",
              "Zhang_inflammatory_monocyte",
              "LAMP3_pos_DC",
              "myeloid_derived_suppressor_cell",
              "MDSC",
              "tumor_associated_neutrophil",
              "mast_cell",
              "APC",
              "antigen_presenting_cell",
              "macrophage",
              "dendritic_cell",
              "tumor_associated_macrophage",
              "TAM"
            ]
          },
          {
            "name": "cf_direction_commit_on_unresolvable_slot",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_not_resolvable_hedge",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].functional_attribution_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_type_I_IFN_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.type_I_IFN",
            "kind": "value",
            "op": "in",
            "expected": [
              "mreg_DC",
              "C1qhigh_macrophage_either_subtype"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-7-pi",
      "modelName": "claude-opus-4-7",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_enriched_apc_states_v2",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 1368,
      "cost": 1.9590477499999999,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "top_3_cluster_enriched_apc_states",
            "kind": "list",
            "matchKey": "apc_state",
            "k": 3,
            "expected": [
              {
                "key": "plasmacytoid_DC",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "T_cell_stimulatory_role_supported",
                      "role_direction_unresolved"
                    ],
                    "canonical": "T_cell_stimulatory_role_supported"
                  }
                }
              },
              {
                "key": "C1qhigh_lipid_associated_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "C1qhigh_inflammatory_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "plasma_cell",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "cohort_significant_per_patient_inconsistent"
                    ],
                    "canonical": "cohort_significant_per_patient_inconsistent"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CD80_CD86": {
                "op": "in",
                "value": [
                  "mreg_DC",
                  "shared_across_APC_states"
                ]
              },
              "PD_L1_in_APC_clusters": {
                "op": "in",
                "value": [
                  "C1qhigh_macrophage_either_subtype",
                  "shared_across_APC_states",
                  "mreg_DC"
                ]
              },
              "type_I_IFN": {
                "op": "in",
                "value": [
                  "plasmacytoid_DC",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "cf_mreg_dc_claimed_enriched",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "mreg_DC",
              "mature_DC_enriched_in_immunoregulatory_molecules"
            ]
          },
          {
            "name": "cf_framework_offcompartment_or_generic_substitution",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "cDC1",
              "Maier_mregDC",
              "Cheng_C1QC_TAM",
              "Cheng_SPP1_TAM",
              "Cheng_FOLR2_TAM",
              "Villani_pDC",
              "Zhang_inflammatory_monocyte",
              "LAMP3_pos_DC",
              "myeloid_derived_suppressor_cell",
              "MDSC",
              "tumor_associated_neutrophil",
              "mast_cell",
              "APC",
              "antigen_presenting_cell",
              "macrophage",
              "dendritic_cell",
              "tumor_associated_macrophage",
              "TAM"
            ]
          },
          {
            "name": "cf_direction_commit_on_unresolvable_slot",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_not_resolvable_hedge",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].functional_attribution_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_type_I_IFN_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.type_I_IFN",
            "kind": "value",
            "op": "in",
            "expected": [
              "mreg_DC",
              "C1qhigh_macrophage_either_subtype"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-7-pi",
      "modelName": "claude-opus-4-7",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_enriched_apc_states_v2",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 872,
      "cost": 1.1000805,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "top_3_cluster_enriched_apc_states",
            "kind": "list",
            "matchKey": "apc_state",
            "k": 3,
            "expected": [
              {
                "key": "plasmacytoid_DC",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "T_cell_stimulatory_role_supported",
                      "role_direction_unresolved"
                    ],
                    "canonical": "T_cell_stimulatory_role_supported"
                  }
                }
              },
              {
                "key": "C1qhigh_lipid_associated_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "C1qhigh_inflammatory_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "plasma_cell",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "cohort_significant_per_patient_inconsistent"
                    ],
                    "canonical": "cohort_significant_per_patient_inconsistent"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CD80_CD86": {
                "op": "in",
                "value": [
                  "mreg_DC",
                  "shared_across_APC_states"
                ]
              },
              "PD_L1_in_APC_clusters": {
                "op": "in",
                "value": [
                  "C1qhigh_macrophage_either_subtype",
                  "shared_across_APC_states",
                  "mreg_DC"
                ]
              },
              "type_I_IFN": {
                "op": "in",
                "value": [
                  "plasmacytoid_DC",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "cf_mreg_dc_claimed_enriched",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "mreg_DC",
              "mature_DC_enriched_in_immunoregulatory_molecules"
            ]
          },
          {
            "name": "cf_framework_offcompartment_or_generic_substitution",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "cDC1",
              "Maier_mregDC",
              "Cheng_C1QC_TAM",
              "Cheng_SPP1_TAM",
              "Cheng_FOLR2_TAM",
              "Villani_pDC",
              "Zhang_inflammatory_monocyte",
              "LAMP3_pos_DC",
              "myeloid_derived_suppressor_cell",
              "MDSC",
              "tumor_associated_neutrophil",
              "mast_cell",
              "APC",
              "antigen_presenting_cell",
              "macrophage",
              "dendritic_cell",
              "tumor_associated_macrophage",
              "TAM"
            ]
          },
          {
            "name": "cf_direction_commit_on_unresolvable_slot",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_not_resolvable_hedge",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].functional_attribution_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_type_I_IFN_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.type_I_IFN",
            "kind": "value",
            "op": "in",
            "expected": [
              "mreg_DC",
              "C1qhigh_macrophage_either_subtype"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-8-claude-code",
      "modelName": "claude-opus-4-8",
      "harness": "claude-code",
      "task": "10x_5p_melanoma_cluster_enriched_apc_states_v2",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 3319,
      "cost": 8.798775749999997,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "top_3_cluster_enriched_apc_states",
            "kind": "list",
            "matchKey": "apc_state",
            "k": 3,
            "expected": [
              {
                "key": "plasmacytoid_DC",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "T_cell_stimulatory_role_supported",
                      "role_direction_unresolved"
                    ],
                    "canonical": "T_cell_stimulatory_role_supported"
                  }
                }
              },
              {
                "key": "C1qhigh_lipid_associated_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "C1qhigh_inflammatory_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "plasma_cell",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "cohort_significant_per_patient_inconsistent"
                    ],
                    "canonical": "cohort_significant_per_patient_inconsistent"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CD80_CD86": {
                "op": "in",
                "value": [
                  "mreg_DC",
                  "shared_across_APC_states"
                ]
              },
              "PD_L1_in_APC_clusters": {
                "op": "in",
                "value": [
                  "C1qhigh_macrophage_either_subtype",
                  "shared_across_APC_states",
                  "mreg_DC"
                ]
              },
              "type_I_IFN": {
                "op": "in",
                "value": [
                  "plasmacytoid_DC",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "cf_mreg_dc_claimed_enriched",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "mreg_DC",
              "mature_DC_enriched_in_immunoregulatory_molecules"
            ]
          },
          {
            "name": "cf_framework_offcompartment_or_generic_substitution",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "cDC1",
              "Maier_mregDC",
              "Cheng_C1QC_TAM",
              "Cheng_SPP1_TAM",
              "Cheng_FOLR2_TAM",
              "Villani_pDC",
              "Zhang_inflammatory_monocyte",
              "LAMP3_pos_DC",
              "myeloid_derived_suppressor_cell",
              "MDSC",
              "tumor_associated_neutrophil",
              "mast_cell",
              "APC",
              "antigen_presenting_cell",
              "macrophage",
              "dendritic_cell",
              "tumor_associated_macrophage",
              "TAM"
            ]
          },
          {
            "name": "cf_direction_commit_on_unresolvable_slot",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_not_resolvable_hedge",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].functional_attribution_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_type_I_IFN_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.type_I_IFN",
            "kind": "value",
            "op": "in",
            "expected": [
              "mreg_DC",
              "C1qhigh_macrophage_either_subtype"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-8-claude-code",
      "modelName": "claude-opus-4-8",
      "harness": "claude-code",
      "task": "10x_5p_melanoma_cluster_enriched_apc_states_v2",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 3125,
      "cost": 10.381759250000004,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "top_3_cluster_enriched_apc_states",
            "kind": "list",
            "matchKey": "apc_state",
            "k": 3,
            "expected": [
              {
                "key": "plasmacytoid_DC",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "T_cell_stimulatory_role_supported",
                      "role_direction_unresolved"
                    ],
                    "canonical": "T_cell_stimulatory_role_supported"
                  }
                }
              },
              {
                "key": "C1qhigh_lipid_associated_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "C1qhigh_inflammatory_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "plasma_cell",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "cohort_significant_per_patient_inconsistent"
                    ],
                    "canonical": "cohort_significant_per_patient_inconsistent"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CD80_CD86": {
                "op": "in",
                "value": [
                  "mreg_DC",
                  "shared_across_APC_states"
                ]
              },
              "PD_L1_in_APC_clusters": {
                "op": "in",
                "value": [
                  "C1qhigh_macrophage_either_subtype",
                  "shared_across_APC_states",
                  "mreg_DC"
                ]
              },
              "type_I_IFN": {
                "op": "in",
                "value": [
                  "plasmacytoid_DC",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "cf_mreg_dc_claimed_enriched",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "mreg_DC",
              "mature_DC_enriched_in_immunoregulatory_molecules"
            ]
          },
          {
            "name": "cf_framework_offcompartment_or_generic_substitution",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "cDC1",
              "Maier_mregDC",
              "Cheng_C1QC_TAM",
              "Cheng_SPP1_TAM",
              "Cheng_FOLR2_TAM",
              "Villani_pDC",
              "Zhang_inflammatory_monocyte",
              "LAMP3_pos_DC",
              "myeloid_derived_suppressor_cell",
              "MDSC",
              "tumor_associated_neutrophil",
              "mast_cell",
              "APC",
              "antigen_presenting_cell",
              "macrophage",
              "dendritic_cell",
              "tumor_associated_macrophage",
              "TAM"
            ]
          },
          {
            "name": "cf_direction_commit_on_unresolvable_slot",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_not_resolvable_hedge",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].functional_attribution_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_type_I_IFN_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.type_I_IFN",
            "kind": "value",
            "op": "in",
            "expected": [
              "mreg_DC",
              "C1qhigh_macrophage_either_subtype"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-8-claude-code",
      "modelName": "claude-opus-4-8",
      "harness": "claude-code",
      "task": "10x_5p_melanoma_cluster_enriched_apc_states_v2",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 3280,
      "cost": 9.696351250000001,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "top_3_cluster_enriched_apc_states",
            "kind": "list",
            "matchKey": "apc_state",
            "k": 3,
            "expected": [
              {
                "key": "plasmacytoid_DC",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "T_cell_stimulatory_role_supported",
                      "role_direction_unresolved"
                    ],
                    "canonical": "T_cell_stimulatory_role_supported"
                  }
                }
              },
              {
                "key": "C1qhigh_lipid_associated_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "C1qhigh_inflammatory_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "plasma_cell",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "cohort_significant_per_patient_inconsistent"
                    ],
                    "canonical": "cohort_significant_per_patient_inconsistent"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CD80_CD86": {
                "op": "in",
                "value": [
                  "mreg_DC",
                  "shared_across_APC_states"
                ]
              },
              "PD_L1_in_APC_clusters": {
                "op": "in",
                "value": [
                  "C1qhigh_macrophage_either_subtype",
                  "shared_across_APC_states",
                  "mreg_DC"
                ]
              },
              "type_I_IFN": {
                "op": "in",
                "value": [
                  "plasmacytoid_DC",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "cf_mreg_dc_claimed_enriched",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "mreg_DC",
              "mature_DC_enriched_in_immunoregulatory_molecules"
            ]
          },
          {
            "name": "cf_framework_offcompartment_or_generic_substitution",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "cDC1",
              "Maier_mregDC",
              "Cheng_C1QC_TAM",
              "Cheng_SPP1_TAM",
              "Cheng_FOLR2_TAM",
              "Villani_pDC",
              "Zhang_inflammatory_monocyte",
              "LAMP3_pos_DC",
              "myeloid_derived_suppressor_cell",
              "MDSC",
              "tumor_associated_neutrophil",
              "mast_cell",
              "APC",
              "antigen_presenting_cell",
              "macrophage",
              "dendritic_cell",
              "tumor_associated_macrophage",
              "TAM"
            ]
          },
          {
            "name": "cf_direction_commit_on_unresolvable_slot",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_not_resolvable_hedge",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].functional_attribution_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_type_I_IFN_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.type_I_IFN",
            "kind": "value",
            "op": "in",
            "expected": [
              "mreg_DC",
              "C1qhigh_macrophage_either_subtype"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-8-pi",
      "modelName": "claude-opus-4-8",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_enriched_apc_states_v2",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 1914,
      "cost": 3.6116970000000004,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "top_3_cluster_enriched_apc_states",
            "kind": "list",
            "matchKey": "apc_state",
            "k": 3,
            "expected": [
              {
                "key": "plasmacytoid_DC",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "T_cell_stimulatory_role_supported",
                      "role_direction_unresolved"
                    ],
                    "canonical": "T_cell_stimulatory_role_supported"
                  }
                }
              },
              {
                "key": "C1qhigh_lipid_associated_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "C1qhigh_inflammatory_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "plasma_cell",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "cohort_significant_per_patient_inconsistent"
                    ],
                    "canonical": "cohort_significant_per_patient_inconsistent"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CD80_CD86": {
                "op": "in",
                "value": [
                  "mreg_DC",
                  "shared_across_APC_states"
                ]
              },
              "PD_L1_in_APC_clusters": {
                "op": "in",
                "value": [
                  "C1qhigh_macrophage_either_subtype",
                  "shared_across_APC_states",
                  "mreg_DC"
                ]
              },
              "type_I_IFN": {
                "op": "in",
                "value": [
                  "plasmacytoid_DC",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "cf_mreg_dc_claimed_enriched",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "mreg_DC",
              "mature_DC_enriched_in_immunoregulatory_molecules"
            ]
          },
          {
            "name": "cf_framework_offcompartment_or_generic_substitution",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "cDC1",
              "Maier_mregDC",
              "Cheng_C1QC_TAM",
              "Cheng_SPP1_TAM",
              "Cheng_FOLR2_TAM",
              "Villani_pDC",
              "Zhang_inflammatory_monocyte",
              "LAMP3_pos_DC",
              "myeloid_derived_suppressor_cell",
              "MDSC",
              "tumor_associated_neutrophil",
              "mast_cell",
              "APC",
              "antigen_presenting_cell",
              "macrophage",
              "dendritic_cell",
              "tumor_associated_macrophage",
              "TAM"
            ]
          },
          {
            "name": "cf_direction_commit_on_unresolvable_slot",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_not_resolvable_hedge",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].functional_attribution_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_type_I_IFN_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.type_I_IFN",
            "kind": "value",
            "op": "in",
            "expected": [
              "mreg_DC",
              "C1qhigh_macrophage_either_subtype"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-8-pi",
      "modelName": "claude-opus-4-8",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_enriched_apc_states_v2",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 1537,
      "cost": 3.7651707500000007,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "top_3_cluster_enriched_apc_states",
            "kind": "list",
            "matchKey": "apc_state",
            "k": 3,
            "expected": [
              {
                "key": "plasmacytoid_DC",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "T_cell_stimulatory_role_supported",
                      "role_direction_unresolved"
                    ],
                    "canonical": "T_cell_stimulatory_role_supported"
                  }
                }
              },
              {
                "key": "C1qhigh_lipid_associated_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "C1qhigh_inflammatory_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "plasma_cell",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "cohort_significant_per_patient_inconsistent"
                    ],
                    "canonical": "cohort_significant_per_patient_inconsistent"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CD80_CD86": {
                "op": "in",
                "value": [
                  "mreg_DC",
                  "shared_across_APC_states"
                ]
              },
              "PD_L1_in_APC_clusters": {
                "op": "in",
                "value": [
                  "C1qhigh_macrophage_either_subtype",
                  "shared_across_APC_states",
                  "mreg_DC"
                ]
              },
              "type_I_IFN": {
                "op": "in",
                "value": [
                  "plasmacytoid_DC",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "cf_mreg_dc_claimed_enriched",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "mreg_DC",
              "mature_DC_enriched_in_immunoregulatory_molecules"
            ]
          },
          {
            "name": "cf_framework_offcompartment_or_generic_substitution",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "cDC1",
              "Maier_mregDC",
              "Cheng_C1QC_TAM",
              "Cheng_SPP1_TAM",
              "Cheng_FOLR2_TAM",
              "Villani_pDC",
              "Zhang_inflammatory_monocyte",
              "LAMP3_pos_DC",
              "myeloid_derived_suppressor_cell",
              "MDSC",
              "tumor_associated_neutrophil",
              "mast_cell",
              "APC",
              "antigen_presenting_cell",
              "macrophage",
              "dendritic_cell",
              "tumor_associated_macrophage",
              "TAM"
            ]
          },
          {
            "name": "cf_direction_commit_on_unresolvable_slot",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_not_resolvable_hedge",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].functional_attribution_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_type_I_IFN_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.type_I_IFN",
            "kind": "value",
            "op": "in",
            "expected": [
              "mreg_DC",
              "C1qhigh_macrophage_either_subtype"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-8-pi",
      "modelName": "claude-opus-4-8",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_enriched_apc_states_v2",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1556,
      "cost": 3.645470499999999,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "top_3_cluster_enriched_apc_states",
            "kind": "list",
            "matchKey": "apc_state",
            "k": 3,
            "expected": [
              {
                "key": "plasmacytoid_DC",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "T_cell_stimulatory_role_supported",
                      "role_direction_unresolved"
                    ],
                    "canonical": "T_cell_stimulatory_role_supported"
                  }
                }
              },
              {
                "key": "C1qhigh_lipid_associated_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "C1qhigh_inflammatory_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "plasma_cell",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "cohort_significant_per_patient_inconsistent"
                    ],
                    "canonical": "cohort_significant_per_patient_inconsistent"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CD80_CD86": {
                "op": "in",
                "value": [
                  "mreg_DC",
                  "shared_across_APC_states"
                ]
              },
              "PD_L1_in_APC_clusters": {
                "op": "in",
                "value": [
                  "C1qhigh_macrophage_either_subtype",
                  "shared_across_APC_states",
                  "mreg_DC"
                ]
              },
              "type_I_IFN": {
                "op": "in",
                "value": [
                  "plasmacytoid_DC",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "cf_mreg_dc_claimed_enriched",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "mreg_DC",
              "mature_DC_enriched_in_immunoregulatory_molecules"
            ]
          },
          {
            "name": "cf_framework_offcompartment_or_generic_substitution",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "cDC1",
              "Maier_mregDC",
              "Cheng_C1QC_TAM",
              "Cheng_SPP1_TAM",
              "Cheng_FOLR2_TAM",
              "Villani_pDC",
              "Zhang_inflammatory_monocyte",
              "LAMP3_pos_DC",
              "myeloid_derived_suppressor_cell",
              "MDSC",
              "tumor_associated_neutrophil",
              "mast_cell",
              "APC",
              "antigen_presenting_cell",
              "macrophage",
              "dendritic_cell",
              "tumor_associated_macrophage",
              "TAM"
            ]
          },
          {
            "name": "cf_direction_commit_on_unresolvable_slot",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_not_resolvable_hedge",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].functional_attribution_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_type_I_IFN_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.type_I_IFN",
            "kind": "value",
            "op": "in",
            "expected": [
              "mreg_DC",
              "C1qhigh_macrophage_either_subtype"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-4-6-pi",
      "modelName": "claude-sonnet-4-6",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_enriched_apc_states_v2",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 1,
      "passed": true,
      "score": null,
      "durationS": 2372,
      "cost": 1.5739431,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "top_3_cluster_enriched_apc_states",
            "kind": "list",
            "matchKey": "apc_state",
            "k": 3,
            "expected": [
              {
                "key": "plasmacytoid_DC",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "T_cell_stimulatory_role_supported",
                      "role_direction_unresolved"
                    ],
                    "canonical": "T_cell_stimulatory_role_supported"
                  }
                }
              },
              {
                "key": "C1qhigh_lipid_associated_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "C1qhigh_inflammatory_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "plasma_cell",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "cohort_significant_per_patient_inconsistent"
                    ],
                    "canonical": "cohort_significant_per_patient_inconsistent"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CD80_CD86": {
                "op": "in",
                "value": [
                  "mreg_DC",
                  "shared_across_APC_states"
                ]
              },
              "PD_L1_in_APC_clusters": {
                "op": "in",
                "value": [
                  "C1qhigh_macrophage_either_subtype",
                  "shared_across_APC_states",
                  "mreg_DC"
                ]
              },
              "type_I_IFN": {
                "op": "in",
                "value": [
                  "plasmacytoid_DC",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "cf_mreg_dc_claimed_enriched",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "mreg_DC",
              "mature_DC_enriched_in_immunoregulatory_molecules"
            ]
          },
          {
            "name": "cf_framework_offcompartment_or_generic_substitution",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "cDC1",
              "Maier_mregDC",
              "Cheng_C1QC_TAM",
              "Cheng_SPP1_TAM",
              "Cheng_FOLR2_TAM",
              "Villani_pDC",
              "Zhang_inflammatory_monocyte",
              "LAMP3_pos_DC",
              "myeloid_derived_suppressor_cell",
              "MDSC",
              "tumor_associated_neutrophil",
              "mast_cell",
              "APC",
              "antigen_presenting_cell",
              "macrophage",
              "dendritic_cell",
              "tumor_associated_macrophage",
              "TAM"
            ]
          },
          {
            "name": "cf_direction_commit_on_unresolvable_slot",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_not_resolvable_hedge",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].functional_attribution_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_type_I_IFN_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.type_I_IFN",
            "kind": "value",
            "op": "in",
            "expected": [
              "mreg_DC",
              "C1qhigh_macrophage_either_subtype"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-4-6-pi",
      "modelName": "claude-sonnet-4-6",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_enriched_apc_states_v2",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 2042,
      "cost": 1.65890115,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "top_3_cluster_enriched_apc_states",
            "kind": "list",
            "matchKey": "apc_state",
            "k": 3,
            "expected": [
              {
                "key": "plasmacytoid_DC",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "T_cell_stimulatory_role_supported",
                      "role_direction_unresolved"
                    ],
                    "canonical": "T_cell_stimulatory_role_supported"
                  }
                }
              },
              {
                "key": "C1qhigh_lipid_associated_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "C1qhigh_inflammatory_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "plasma_cell",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "cohort_significant_per_patient_inconsistent"
                    ],
                    "canonical": "cohort_significant_per_patient_inconsistent"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CD80_CD86": {
                "op": "in",
                "value": [
                  "mreg_DC",
                  "shared_across_APC_states"
                ]
              },
              "PD_L1_in_APC_clusters": {
                "op": "in",
                "value": [
                  "C1qhigh_macrophage_either_subtype",
                  "shared_across_APC_states",
                  "mreg_DC"
                ]
              },
              "type_I_IFN": {
                "op": "in",
                "value": [
                  "plasmacytoid_DC",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "cf_mreg_dc_claimed_enriched",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "mreg_DC",
              "mature_DC_enriched_in_immunoregulatory_molecules"
            ]
          },
          {
            "name": "cf_framework_offcompartment_or_generic_substitution",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "cDC1",
              "Maier_mregDC",
              "Cheng_C1QC_TAM",
              "Cheng_SPP1_TAM",
              "Cheng_FOLR2_TAM",
              "Villani_pDC",
              "Zhang_inflammatory_monocyte",
              "LAMP3_pos_DC",
              "myeloid_derived_suppressor_cell",
              "MDSC",
              "tumor_associated_neutrophil",
              "mast_cell",
              "APC",
              "antigen_presenting_cell",
              "macrophage",
              "dendritic_cell",
              "tumor_associated_macrophage",
              "TAM"
            ]
          },
          {
            "name": "cf_direction_commit_on_unresolvable_slot",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_not_resolvable_hedge",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].functional_attribution_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_type_I_IFN_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.type_I_IFN",
            "kind": "value",
            "op": "in",
            "expected": [
              "mreg_DC",
              "C1qhigh_macrophage_either_subtype"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-4-6-pi",
      "modelName": "claude-sonnet-4-6",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_enriched_apc_states_v2",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 2540,
      "cost": 1.6311063000000001,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "top_3_cluster_enriched_apc_states",
            "kind": "list",
            "matchKey": "apc_state",
            "k": 3,
            "expected": [
              {
                "key": "plasmacytoid_DC",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "T_cell_stimulatory_role_supported",
                      "role_direction_unresolved"
                    ],
                    "canonical": "T_cell_stimulatory_role_supported"
                  }
                }
              },
              {
                "key": "C1qhigh_lipid_associated_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "C1qhigh_inflammatory_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "plasma_cell",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "cohort_significant_per_patient_inconsistent"
                    ],
                    "canonical": "cohort_significant_per_patient_inconsistent"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CD80_CD86": {
                "op": "in",
                "value": [
                  "mreg_DC",
                  "shared_across_APC_states"
                ]
              },
              "PD_L1_in_APC_clusters": {
                "op": "in",
                "value": [
                  "C1qhigh_macrophage_either_subtype",
                  "shared_across_APC_states",
                  "mreg_DC"
                ]
              },
              "type_I_IFN": {
                "op": "in",
                "value": [
                  "plasmacytoid_DC",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "cf_mreg_dc_claimed_enriched",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "mreg_DC",
              "mature_DC_enriched_in_immunoregulatory_molecules"
            ]
          },
          {
            "name": "cf_framework_offcompartment_or_generic_substitution",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "cDC1",
              "Maier_mregDC",
              "Cheng_C1QC_TAM",
              "Cheng_SPP1_TAM",
              "Cheng_FOLR2_TAM",
              "Villani_pDC",
              "Zhang_inflammatory_monocyte",
              "LAMP3_pos_DC",
              "myeloid_derived_suppressor_cell",
              "MDSC",
              "tumor_associated_neutrophil",
              "mast_cell",
              "APC",
              "antigen_presenting_cell",
              "macrophage",
              "dendritic_cell",
              "tumor_associated_macrophage",
              "TAM"
            ]
          },
          {
            "name": "cf_direction_commit_on_unresolvable_slot",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_not_resolvable_hedge",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].functional_attribution_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_type_I_IFN_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.type_I_IFN",
            "kind": "value",
            "op": "in",
            "expected": [
              "mreg_DC",
              "C1qhigh_macrophage_either_subtype"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gemini-3-1-pro-preview-pi",
      "modelName": "gemini-3.1-pro-preview",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_enriched_apc_states_v2",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 975,
      "cost": 0.9992188000000002,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "top_3_cluster_enriched_apc_states",
            "kind": "list",
            "matchKey": "apc_state",
            "k": 3,
            "expected": [
              {
                "key": "plasmacytoid_DC",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "T_cell_stimulatory_role_supported",
                      "role_direction_unresolved"
                    ],
                    "canonical": "T_cell_stimulatory_role_supported"
                  }
                }
              },
              {
                "key": "C1qhigh_lipid_associated_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "C1qhigh_inflammatory_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "plasma_cell",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "cohort_significant_per_patient_inconsistent"
                    ],
                    "canonical": "cohort_significant_per_patient_inconsistent"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CD80_CD86": {
                "op": "in",
                "value": [
                  "mreg_DC",
                  "shared_across_APC_states"
                ]
              },
              "PD_L1_in_APC_clusters": {
                "op": "in",
                "value": [
                  "C1qhigh_macrophage_either_subtype",
                  "shared_across_APC_states",
                  "mreg_DC"
                ]
              },
              "type_I_IFN": {
                "op": "in",
                "value": [
                  "plasmacytoid_DC",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "cf_mreg_dc_claimed_enriched",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "mreg_DC",
              "mature_DC_enriched_in_immunoregulatory_molecules"
            ]
          },
          {
            "name": "cf_framework_offcompartment_or_generic_substitution",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "cDC1",
              "Maier_mregDC",
              "Cheng_C1QC_TAM",
              "Cheng_SPP1_TAM",
              "Cheng_FOLR2_TAM",
              "Villani_pDC",
              "Zhang_inflammatory_monocyte",
              "LAMP3_pos_DC",
              "myeloid_derived_suppressor_cell",
              "MDSC",
              "tumor_associated_neutrophil",
              "mast_cell",
              "APC",
              "antigen_presenting_cell",
              "macrophage",
              "dendritic_cell",
              "tumor_associated_macrophage",
              "TAM"
            ]
          },
          {
            "name": "cf_direction_commit_on_unresolvable_slot",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_not_resolvable_hedge",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].functional_attribution_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_type_I_IFN_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.type_I_IFN",
            "kind": "value",
            "op": "in",
            "expected": [
              "mreg_DC",
              "C1qhigh_macrophage_either_subtype"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gemini-3-1-pro-preview-pi",
      "modelName": "gemini-3.1-pro-preview",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_enriched_apc_states_v2",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 1296,
      "cost": 1.2638916000000007,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "top_3_cluster_enriched_apc_states",
            "kind": "list",
            "matchKey": "apc_state",
            "k": 3,
            "expected": [
              {
                "key": "plasmacytoid_DC",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "T_cell_stimulatory_role_supported",
                      "role_direction_unresolved"
                    ],
                    "canonical": "T_cell_stimulatory_role_supported"
                  }
                }
              },
              {
                "key": "C1qhigh_lipid_associated_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "C1qhigh_inflammatory_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "plasma_cell",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "cohort_significant_per_patient_inconsistent"
                    ],
                    "canonical": "cohort_significant_per_patient_inconsistent"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CD80_CD86": {
                "op": "in",
                "value": [
                  "mreg_DC",
                  "shared_across_APC_states"
                ]
              },
              "PD_L1_in_APC_clusters": {
                "op": "in",
                "value": [
                  "C1qhigh_macrophage_either_subtype",
                  "shared_across_APC_states",
                  "mreg_DC"
                ]
              },
              "type_I_IFN": {
                "op": "in",
                "value": [
                  "plasmacytoid_DC",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "cf_mreg_dc_claimed_enriched",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "mreg_DC",
              "mature_DC_enriched_in_immunoregulatory_molecules"
            ]
          },
          {
            "name": "cf_framework_offcompartment_or_generic_substitution",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "cDC1",
              "Maier_mregDC",
              "Cheng_C1QC_TAM",
              "Cheng_SPP1_TAM",
              "Cheng_FOLR2_TAM",
              "Villani_pDC",
              "Zhang_inflammatory_monocyte",
              "LAMP3_pos_DC",
              "myeloid_derived_suppressor_cell",
              "MDSC",
              "tumor_associated_neutrophil",
              "mast_cell",
              "APC",
              "antigen_presenting_cell",
              "macrophage",
              "dendritic_cell",
              "tumor_associated_macrophage",
              "TAM"
            ]
          },
          {
            "name": "cf_direction_commit_on_unresolvable_slot",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_not_resolvable_hedge",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].functional_attribution_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_type_I_IFN_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.type_I_IFN",
            "kind": "value",
            "op": "in",
            "expected": [
              "mreg_DC",
              "C1qhigh_macrophage_either_subtype"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gemini-3-1-pro-preview-pi",
      "modelName": "gemini-3.1-pro-preview",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_enriched_apc_states_v2",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1058,
      "cost": 0.8711088,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "top_3_cluster_enriched_apc_states",
            "kind": "list",
            "matchKey": "apc_state",
            "k": 3,
            "expected": [
              {
                "key": "plasmacytoid_DC",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "T_cell_stimulatory_role_supported",
                      "role_direction_unresolved"
                    ],
                    "canonical": "T_cell_stimulatory_role_supported"
                  }
                }
              },
              {
                "key": "C1qhigh_lipid_associated_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "C1qhigh_inflammatory_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "plasma_cell",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "cohort_significant_per_patient_inconsistent"
                    ],
                    "canonical": "cohort_significant_per_patient_inconsistent"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CD80_CD86": {
                "op": "in",
                "value": [
                  "mreg_DC",
                  "shared_across_APC_states"
                ]
              },
              "PD_L1_in_APC_clusters": {
                "op": "in",
                "value": [
                  "C1qhigh_macrophage_either_subtype",
                  "shared_across_APC_states",
                  "mreg_DC"
                ]
              },
              "type_I_IFN": {
                "op": "in",
                "value": [
                  "plasmacytoid_DC",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "cf_mreg_dc_claimed_enriched",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "mreg_DC",
              "mature_DC_enriched_in_immunoregulatory_molecules"
            ]
          },
          {
            "name": "cf_framework_offcompartment_or_generic_substitution",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "cDC1",
              "Maier_mregDC",
              "Cheng_C1QC_TAM",
              "Cheng_SPP1_TAM",
              "Cheng_FOLR2_TAM",
              "Villani_pDC",
              "Zhang_inflammatory_monocyte",
              "LAMP3_pos_DC",
              "myeloid_derived_suppressor_cell",
              "MDSC",
              "tumor_associated_neutrophil",
              "mast_cell",
              "APC",
              "antigen_presenting_cell",
              "macrophage",
              "dendritic_cell",
              "tumor_associated_macrophage",
              "TAM"
            ]
          },
          {
            "name": "cf_direction_commit_on_unresolvable_slot",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_not_resolvable_hedge",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].functional_attribution_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_type_I_IFN_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.type_I_IFN",
            "kind": "value",
            "op": "in",
            "expected": [
              "mreg_DC",
              "C1qhigh_macrophage_either_subtype"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gemini-3-5-flash-pi",
      "modelName": "gemini-3.5-flash",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_enriched_apc_states_v2",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 1504,
      "cost": 2.3971517999999996,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "top_3_cluster_enriched_apc_states",
            "kind": "list",
            "matchKey": "apc_state",
            "k": 3,
            "expected": [
              {
                "key": "plasmacytoid_DC",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "T_cell_stimulatory_role_supported",
                      "role_direction_unresolved"
                    ],
                    "canonical": "T_cell_stimulatory_role_supported"
                  }
                }
              },
              {
                "key": "C1qhigh_lipid_associated_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "C1qhigh_inflammatory_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "plasma_cell",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "cohort_significant_per_patient_inconsistent"
                    ],
                    "canonical": "cohort_significant_per_patient_inconsistent"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CD80_CD86": {
                "op": "in",
                "value": [
                  "mreg_DC",
                  "shared_across_APC_states"
                ]
              },
              "PD_L1_in_APC_clusters": {
                "op": "in",
                "value": [
                  "C1qhigh_macrophage_either_subtype",
                  "shared_across_APC_states",
                  "mreg_DC"
                ]
              },
              "type_I_IFN": {
                "op": "in",
                "value": [
                  "plasmacytoid_DC",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "cf_mreg_dc_claimed_enriched",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "mreg_DC",
              "mature_DC_enriched_in_immunoregulatory_molecules"
            ]
          },
          {
            "name": "cf_framework_offcompartment_or_generic_substitution",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "cDC1",
              "Maier_mregDC",
              "Cheng_C1QC_TAM",
              "Cheng_SPP1_TAM",
              "Cheng_FOLR2_TAM",
              "Villani_pDC",
              "Zhang_inflammatory_monocyte",
              "LAMP3_pos_DC",
              "myeloid_derived_suppressor_cell",
              "MDSC",
              "tumor_associated_neutrophil",
              "mast_cell",
              "APC",
              "antigen_presenting_cell",
              "macrophage",
              "dendritic_cell",
              "tumor_associated_macrophage",
              "TAM"
            ]
          },
          {
            "name": "cf_direction_commit_on_unresolvable_slot",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_not_resolvable_hedge",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].functional_attribution_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_type_I_IFN_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.type_I_IFN",
            "kind": "value",
            "op": "in",
            "expected": [
              "mreg_DC",
              "C1qhigh_macrophage_either_subtype"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gemini-3-5-flash-pi",
      "modelName": "gemini-3.5-flash",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_enriched_apc_states_v2",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 2081,
      "cost": 2.5613244,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "top_3_cluster_enriched_apc_states",
            "kind": "list",
            "matchKey": "apc_state",
            "k": 3,
            "expected": [
              {
                "key": "plasmacytoid_DC",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "T_cell_stimulatory_role_supported",
                      "role_direction_unresolved"
                    ],
                    "canonical": "T_cell_stimulatory_role_supported"
                  }
                }
              },
              {
                "key": "C1qhigh_lipid_associated_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "C1qhigh_inflammatory_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "plasma_cell",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "cohort_significant_per_patient_inconsistent"
                    ],
                    "canonical": "cohort_significant_per_patient_inconsistent"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CD80_CD86": {
                "op": "in",
                "value": [
                  "mreg_DC",
                  "shared_across_APC_states"
                ]
              },
              "PD_L1_in_APC_clusters": {
                "op": "in",
                "value": [
                  "C1qhigh_macrophage_either_subtype",
                  "shared_across_APC_states",
                  "mreg_DC"
                ]
              },
              "type_I_IFN": {
                "op": "in",
                "value": [
                  "plasmacytoid_DC",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "cf_mreg_dc_claimed_enriched",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "mreg_DC",
              "mature_DC_enriched_in_immunoregulatory_molecules"
            ]
          },
          {
            "name": "cf_framework_offcompartment_or_generic_substitution",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "cDC1",
              "Maier_mregDC",
              "Cheng_C1QC_TAM",
              "Cheng_SPP1_TAM",
              "Cheng_FOLR2_TAM",
              "Villani_pDC",
              "Zhang_inflammatory_monocyte",
              "LAMP3_pos_DC",
              "myeloid_derived_suppressor_cell",
              "MDSC",
              "tumor_associated_neutrophil",
              "mast_cell",
              "APC",
              "antigen_presenting_cell",
              "macrophage",
              "dendritic_cell",
              "tumor_associated_macrophage",
              "TAM"
            ]
          },
          {
            "name": "cf_direction_commit_on_unresolvable_slot",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_not_resolvable_hedge",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].functional_attribution_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_type_I_IFN_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.type_I_IFN",
            "kind": "value",
            "op": "in",
            "expected": [
              "mreg_DC",
              "C1qhigh_macrophage_either_subtype"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gemini-3-5-flash-pi",
      "modelName": "gemini-3.5-flash",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_enriched_apc_states_v2",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 773,
      "cost": 2.77327515,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "top_3_cluster_enriched_apc_states",
            "kind": "list",
            "matchKey": "apc_state",
            "k": 3,
            "expected": [
              {
                "key": "plasmacytoid_DC",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "T_cell_stimulatory_role_supported",
                      "role_direction_unresolved"
                    ],
                    "canonical": "T_cell_stimulatory_role_supported"
                  }
                }
              },
              {
                "key": "C1qhigh_lipid_associated_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "C1qhigh_inflammatory_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "plasma_cell",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "cohort_significant_per_patient_inconsistent"
                    ],
                    "canonical": "cohort_significant_per_patient_inconsistent"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CD80_CD86": {
                "op": "in",
                "value": [
                  "mreg_DC",
                  "shared_across_APC_states"
                ]
              },
              "PD_L1_in_APC_clusters": {
                "op": "in",
                "value": [
                  "C1qhigh_macrophage_either_subtype",
                  "shared_across_APC_states",
                  "mreg_DC"
                ]
              },
              "type_I_IFN": {
                "op": "in",
                "value": [
                  "plasmacytoid_DC",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "cf_mreg_dc_claimed_enriched",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "mreg_DC",
              "mature_DC_enriched_in_immunoregulatory_molecules"
            ]
          },
          {
            "name": "cf_framework_offcompartment_or_generic_substitution",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "cDC1",
              "Maier_mregDC",
              "Cheng_C1QC_TAM",
              "Cheng_SPP1_TAM",
              "Cheng_FOLR2_TAM",
              "Villani_pDC",
              "Zhang_inflammatory_monocyte",
              "LAMP3_pos_DC",
              "myeloid_derived_suppressor_cell",
              "MDSC",
              "tumor_associated_neutrophil",
              "mast_cell",
              "APC",
              "antigen_presenting_cell",
              "macrophage",
              "dendritic_cell",
              "tumor_associated_macrophage",
              "TAM"
            ]
          },
          {
            "name": "cf_direction_commit_on_unresolvable_slot",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_not_resolvable_hedge",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].functional_attribution_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_type_I_IFN_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.type_I_IFN",
            "kind": "value",
            "op": "in",
            "expected": [
              "mreg_DC",
              "C1qhigh_macrophage_either_subtype"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-glm-5p2-pi",
      "modelName": "glm-5p2",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_enriched_apc_states_v2",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 3138,
      "cost": 2.7317736,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "top_3_cluster_enriched_apc_states",
            "kind": "list",
            "matchKey": "apc_state",
            "k": 3,
            "expected": [
              {
                "key": "plasmacytoid_DC",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "T_cell_stimulatory_role_supported",
                      "role_direction_unresolved"
                    ],
                    "canonical": "T_cell_stimulatory_role_supported"
                  }
                }
              },
              {
                "key": "C1qhigh_lipid_associated_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "C1qhigh_inflammatory_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "plasma_cell",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "cohort_significant_per_patient_inconsistent"
                    ],
                    "canonical": "cohort_significant_per_patient_inconsistent"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CD80_CD86": {
                "op": "in",
                "value": [
                  "mreg_DC",
                  "shared_across_APC_states"
                ]
              },
              "PD_L1_in_APC_clusters": {
                "op": "in",
                "value": [
                  "C1qhigh_macrophage_either_subtype",
                  "shared_across_APC_states",
                  "mreg_DC"
                ]
              },
              "type_I_IFN": {
                "op": "in",
                "value": [
                  "plasmacytoid_DC",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "cf_mreg_dc_claimed_enriched",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "mreg_DC",
              "mature_DC_enriched_in_immunoregulatory_molecules"
            ]
          },
          {
            "name": "cf_framework_offcompartment_or_generic_substitution",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "cDC1",
              "Maier_mregDC",
              "Cheng_C1QC_TAM",
              "Cheng_SPP1_TAM",
              "Cheng_FOLR2_TAM",
              "Villani_pDC",
              "Zhang_inflammatory_monocyte",
              "LAMP3_pos_DC",
              "myeloid_derived_suppressor_cell",
              "MDSC",
              "tumor_associated_neutrophil",
              "mast_cell",
              "APC",
              "antigen_presenting_cell",
              "macrophage",
              "dendritic_cell",
              "tumor_associated_macrophage",
              "TAM"
            ]
          },
          {
            "name": "cf_direction_commit_on_unresolvable_slot",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_not_resolvable_hedge",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].functional_attribution_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_type_I_IFN_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.type_I_IFN",
            "kind": "value",
            "op": "in",
            "expected": [
              "mreg_DC",
              "C1qhigh_macrophage_either_subtype"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-glm-5p2-pi",
      "modelName": "glm-5p2",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_enriched_apc_states_v2",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 4592,
      "cost": 5.991437379999998,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "top_3_cluster_enriched_apc_states",
            "kind": "list",
            "matchKey": "apc_state",
            "k": 3,
            "expected": [
              {
                "key": "plasmacytoid_DC",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "T_cell_stimulatory_role_supported",
                      "role_direction_unresolved"
                    ],
                    "canonical": "T_cell_stimulatory_role_supported"
                  }
                }
              },
              {
                "key": "C1qhigh_lipid_associated_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "C1qhigh_inflammatory_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "plasma_cell",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "cohort_significant_per_patient_inconsistent"
                    ],
                    "canonical": "cohort_significant_per_patient_inconsistent"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CD80_CD86": {
                "op": "in",
                "value": [
                  "mreg_DC",
                  "shared_across_APC_states"
                ]
              },
              "PD_L1_in_APC_clusters": {
                "op": "in",
                "value": [
                  "C1qhigh_macrophage_either_subtype",
                  "shared_across_APC_states",
                  "mreg_DC"
                ]
              },
              "type_I_IFN": {
                "op": "in",
                "value": [
                  "plasmacytoid_DC",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "cf_mreg_dc_claimed_enriched",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "mreg_DC",
              "mature_DC_enriched_in_immunoregulatory_molecules"
            ]
          },
          {
            "name": "cf_framework_offcompartment_or_generic_substitution",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "cDC1",
              "Maier_mregDC",
              "Cheng_C1QC_TAM",
              "Cheng_SPP1_TAM",
              "Cheng_FOLR2_TAM",
              "Villani_pDC",
              "Zhang_inflammatory_monocyte",
              "LAMP3_pos_DC",
              "myeloid_derived_suppressor_cell",
              "MDSC",
              "tumor_associated_neutrophil",
              "mast_cell",
              "APC",
              "antigen_presenting_cell",
              "macrophage",
              "dendritic_cell",
              "tumor_associated_macrophage",
              "TAM"
            ]
          },
          {
            "name": "cf_direction_commit_on_unresolvable_slot",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_not_resolvable_hedge",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].functional_attribution_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_type_I_IFN_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.type_I_IFN",
            "kind": "value",
            "op": "in",
            "expected": [
              "mreg_DC",
              "C1qhigh_macrophage_either_subtype"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-glm-5p2-pi",
      "modelName": "glm-5p2",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_enriched_apc_states_v2",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 3735,
      "cost": 4.3609127999999995,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "top_3_cluster_enriched_apc_states",
            "kind": "list",
            "matchKey": "apc_state",
            "k": 3,
            "expected": [
              {
                "key": "plasmacytoid_DC",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "T_cell_stimulatory_role_supported",
                      "role_direction_unresolved"
                    ],
                    "canonical": "T_cell_stimulatory_role_supported"
                  }
                }
              },
              {
                "key": "C1qhigh_lipid_associated_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "C1qhigh_inflammatory_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "plasma_cell",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "cohort_significant_per_patient_inconsistent"
                    ],
                    "canonical": "cohort_significant_per_patient_inconsistent"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CD80_CD86": {
                "op": "in",
                "value": [
                  "mreg_DC",
                  "shared_across_APC_states"
                ]
              },
              "PD_L1_in_APC_clusters": {
                "op": "in",
                "value": [
                  "C1qhigh_macrophage_either_subtype",
                  "shared_across_APC_states",
                  "mreg_DC"
                ]
              },
              "type_I_IFN": {
                "op": "in",
                "value": [
                  "plasmacytoid_DC",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "cf_mreg_dc_claimed_enriched",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "mreg_DC",
              "mature_DC_enriched_in_immunoregulatory_molecules"
            ]
          },
          {
            "name": "cf_framework_offcompartment_or_generic_substitution",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "cDC1",
              "Maier_mregDC",
              "Cheng_C1QC_TAM",
              "Cheng_SPP1_TAM",
              "Cheng_FOLR2_TAM",
              "Villani_pDC",
              "Zhang_inflammatory_monocyte",
              "LAMP3_pos_DC",
              "myeloid_derived_suppressor_cell",
              "MDSC",
              "tumor_associated_neutrophil",
              "mast_cell",
              "APC",
              "antigen_presenting_cell",
              "macrophage",
              "dendritic_cell",
              "tumor_associated_macrophage",
              "TAM"
            ]
          },
          {
            "name": "cf_direction_commit_on_unresolvable_slot",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_not_resolvable_hedge",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].functional_attribution_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_type_I_IFN_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.type_I_IFN",
            "kind": "value",
            "op": "in",
            "expected": [
              "mreg_DC",
              "C1qhigh_macrophage_either_subtype"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-4-codex",
      "modelName": "gpt-5.4",
      "harness": "openai-codex",
      "task": "10x_5p_melanoma_cluster_enriched_apc_states_v2",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 1081,
      "cost": 2.785203,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "top_3_cluster_enriched_apc_states",
            "kind": "list",
            "matchKey": "apc_state",
            "k": 3,
            "expected": [
              {
                "key": "plasmacytoid_DC",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "T_cell_stimulatory_role_supported",
                      "role_direction_unresolved"
                    ],
                    "canonical": "T_cell_stimulatory_role_supported"
                  }
                }
              },
              {
                "key": "C1qhigh_lipid_associated_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "C1qhigh_inflammatory_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "plasma_cell",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "cohort_significant_per_patient_inconsistent"
                    ],
                    "canonical": "cohort_significant_per_patient_inconsistent"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CD80_CD86": {
                "op": "in",
                "value": [
                  "mreg_DC",
                  "shared_across_APC_states"
                ]
              },
              "PD_L1_in_APC_clusters": {
                "op": "in",
                "value": [
                  "C1qhigh_macrophage_either_subtype",
                  "shared_across_APC_states",
                  "mreg_DC"
                ]
              },
              "type_I_IFN": {
                "op": "in",
                "value": [
                  "plasmacytoid_DC",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "cf_mreg_dc_claimed_enriched",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "mreg_DC",
              "mature_DC_enriched_in_immunoregulatory_molecules"
            ]
          },
          {
            "name": "cf_framework_offcompartment_or_generic_substitution",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "cDC1",
              "Maier_mregDC",
              "Cheng_C1QC_TAM",
              "Cheng_SPP1_TAM",
              "Cheng_FOLR2_TAM",
              "Villani_pDC",
              "Zhang_inflammatory_monocyte",
              "LAMP3_pos_DC",
              "myeloid_derived_suppressor_cell",
              "MDSC",
              "tumor_associated_neutrophil",
              "mast_cell",
              "APC",
              "antigen_presenting_cell",
              "macrophage",
              "dendritic_cell",
              "tumor_associated_macrophage",
              "TAM"
            ]
          },
          {
            "name": "cf_direction_commit_on_unresolvable_slot",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_not_resolvable_hedge",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].functional_attribution_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_type_I_IFN_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.type_I_IFN",
            "kind": "value",
            "op": "in",
            "expected": [
              "mreg_DC",
              "C1qhigh_macrophage_either_subtype"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-4-codex",
      "modelName": "gpt-5.4",
      "harness": "openai-codex",
      "task": "10x_5p_melanoma_cluster_enriched_apc_states_v2",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 1469,
      "cost": 2.4771555000000003,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "top_3_cluster_enriched_apc_states",
            "kind": "list",
            "matchKey": "apc_state",
            "k": 3,
            "expected": [
              {
                "key": "plasmacytoid_DC",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "T_cell_stimulatory_role_supported",
                      "role_direction_unresolved"
                    ],
                    "canonical": "T_cell_stimulatory_role_supported"
                  }
                }
              },
              {
                "key": "C1qhigh_lipid_associated_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "C1qhigh_inflammatory_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "plasma_cell",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "cohort_significant_per_patient_inconsistent"
                    ],
                    "canonical": "cohort_significant_per_patient_inconsistent"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CD80_CD86": {
                "op": "in",
                "value": [
                  "mreg_DC",
                  "shared_across_APC_states"
                ]
              },
              "PD_L1_in_APC_clusters": {
                "op": "in",
                "value": [
                  "C1qhigh_macrophage_either_subtype",
                  "shared_across_APC_states",
                  "mreg_DC"
                ]
              },
              "type_I_IFN": {
                "op": "in",
                "value": [
                  "plasmacytoid_DC",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "cf_mreg_dc_claimed_enriched",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "mreg_DC",
              "mature_DC_enriched_in_immunoregulatory_molecules"
            ]
          },
          {
            "name": "cf_framework_offcompartment_or_generic_substitution",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "cDC1",
              "Maier_mregDC",
              "Cheng_C1QC_TAM",
              "Cheng_SPP1_TAM",
              "Cheng_FOLR2_TAM",
              "Villani_pDC",
              "Zhang_inflammatory_monocyte",
              "LAMP3_pos_DC",
              "myeloid_derived_suppressor_cell",
              "MDSC",
              "tumor_associated_neutrophil",
              "mast_cell",
              "APC",
              "antigen_presenting_cell",
              "macrophage",
              "dendritic_cell",
              "tumor_associated_macrophage",
              "TAM"
            ]
          },
          {
            "name": "cf_direction_commit_on_unresolvable_slot",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_not_resolvable_hedge",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].functional_attribution_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_type_I_IFN_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.type_I_IFN",
            "kind": "value",
            "op": "in",
            "expected": [
              "mreg_DC",
              "C1qhigh_macrophage_either_subtype"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-4-codex",
      "modelName": "gpt-5.4",
      "harness": "openai-codex",
      "task": "10x_5p_melanoma_cluster_enriched_apc_states_v2",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 905,
      "cost": 2.030033,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "top_3_cluster_enriched_apc_states",
            "kind": "list",
            "matchKey": "apc_state",
            "k": 3,
            "expected": [
              {
                "key": "plasmacytoid_DC",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "T_cell_stimulatory_role_supported",
                      "role_direction_unresolved"
                    ],
                    "canonical": "T_cell_stimulatory_role_supported"
                  }
                }
              },
              {
                "key": "C1qhigh_lipid_associated_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "C1qhigh_inflammatory_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "plasma_cell",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "cohort_significant_per_patient_inconsistent"
                    ],
                    "canonical": "cohort_significant_per_patient_inconsistent"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CD80_CD86": {
                "op": "in",
                "value": [
                  "mreg_DC",
                  "shared_across_APC_states"
                ]
              },
              "PD_L1_in_APC_clusters": {
                "op": "in",
                "value": [
                  "C1qhigh_macrophage_either_subtype",
                  "shared_across_APC_states",
                  "mreg_DC"
                ]
              },
              "type_I_IFN": {
                "op": "in",
                "value": [
                  "plasmacytoid_DC",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "cf_mreg_dc_claimed_enriched",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "mreg_DC",
              "mature_DC_enriched_in_immunoregulatory_molecules"
            ]
          },
          {
            "name": "cf_framework_offcompartment_or_generic_substitution",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "cDC1",
              "Maier_mregDC",
              "Cheng_C1QC_TAM",
              "Cheng_SPP1_TAM",
              "Cheng_FOLR2_TAM",
              "Villani_pDC",
              "Zhang_inflammatory_monocyte",
              "LAMP3_pos_DC",
              "myeloid_derived_suppressor_cell",
              "MDSC",
              "tumor_associated_neutrophil",
              "mast_cell",
              "APC",
              "antigen_presenting_cell",
              "macrophage",
              "dendritic_cell",
              "tumor_associated_macrophage",
              "TAM"
            ]
          },
          {
            "name": "cf_direction_commit_on_unresolvable_slot",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_not_resolvable_hedge",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].functional_attribution_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_type_I_IFN_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.type_I_IFN",
            "kind": "value",
            "op": "in",
            "expected": [
              "mreg_DC",
              "C1qhigh_macrophage_either_subtype"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-4-pi",
      "modelName": "gpt-5.4",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_enriched_apc_states_v2",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 2313,
      "cost": 1.5989315,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "top_3_cluster_enriched_apc_states",
            "kind": "list",
            "matchKey": "apc_state",
            "k": 3,
            "expected": [
              {
                "key": "plasmacytoid_DC",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "T_cell_stimulatory_role_supported",
                      "role_direction_unresolved"
                    ],
                    "canonical": "T_cell_stimulatory_role_supported"
                  }
                }
              },
              {
                "key": "C1qhigh_lipid_associated_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "C1qhigh_inflammatory_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "plasma_cell",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "cohort_significant_per_patient_inconsistent"
                    ],
                    "canonical": "cohort_significant_per_patient_inconsistent"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CD80_CD86": {
                "op": "in",
                "value": [
                  "mreg_DC",
                  "shared_across_APC_states"
                ]
              },
              "PD_L1_in_APC_clusters": {
                "op": "in",
                "value": [
                  "C1qhigh_macrophage_either_subtype",
                  "shared_across_APC_states",
                  "mreg_DC"
                ]
              },
              "type_I_IFN": {
                "op": "in",
                "value": [
                  "plasmacytoid_DC",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "cf_mreg_dc_claimed_enriched",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "mreg_DC",
              "mature_DC_enriched_in_immunoregulatory_molecules"
            ]
          },
          {
            "name": "cf_framework_offcompartment_or_generic_substitution",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "cDC1",
              "Maier_mregDC",
              "Cheng_C1QC_TAM",
              "Cheng_SPP1_TAM",
              "Cheng_FOLR2_TAM",
              "Villani_pDC",
              "Zhang_inflammatory_monocyte",
              "LAMP3_pos_DC",
              "myeloid_derived_suppressor_cell",
              "MDSC",
              "tumor_associated_neutrophil",
              "mast_cell",
              "APC",
              "antigen_presenting_cell",
              "macrophage",
              "dendritic_cell",
              "tumor_associated_macrophage",
              "TAM"
            ]
          },
          {
            "name": "cf_direction_commit_on_unresolvable_slot",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_not_resolvable_hedge",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].functional_attribution_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_type_I_IFN_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.type_I_IFN",
            "kind": "value",
            "op": "in",
            "expected": [
              "mreg_DC",
              "C1qhigh_macrophage_either_subtype"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-4-pi",
      "modelName": "gpt-5.4",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_enriched_apc_states_v2",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 3117,
      "cost": 1.4047915000000002,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "top_3_cluster_enriched_apc_states",
            "kind": "list",
            "matchKey": "apc_state",
            "k": 3,
            "expected": [
              {
                "key": "plasmacytoid_DC",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "T_cell_stimulatory_role_supported",
                      "role_direction_unresolved"
                    ],
                    "canonical": "T_cell_stimulatory_role_supported"
                  }
                }
              },
              {
                "key": "C1qhigh_lipid_associated_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "C1qhigh_inflammatory_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "plasma_cell",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "cohort_significant_per_patient_inconsistent"
                    ],
                    "canonical": "cohort_significant_per_patient_inconsistent"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CD80_CD86": {
                "op": "in",
                "value": [
                  "mreg_DC",
                  "shared_across_APC_states"
                ]
              },
              "PD_L1_in_APC_clusters": {
                "op": "in",
                "value": [
                  "C1qhigh_macrophage_either_subtype",
                  "shared_across_APC_states",
                  "mreg_DC"
                ]
              },
              "type_I_IFN": {
                "op": "in",
                "value": [
                  "plasmacytoid_DC",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "cf_mreg_dc_claimed_enriched",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "mreg_DC",
              "mature_DC_enriched_in_immunoregulatory_molecules"
            ]
          },
          {
            "name": "cf_framework_offcompartment_or_generic_substitution",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "cDC1",
              "Maier_mregDC",
              "Cheng_C1QC_TAM",
              "Cheng_SPP1_TAM",
              "Cheng_FOLR2_TAM",
              "Villani_pDC",
              "Zhang_inflammatory_monocyte",
              "LAMP3_pos_DC",
              "myeloid_derived_suppressor_cell",
              "MDSC",
              "tumor_associated_neutrophil",
              "mast_cell",
              "APC",
              "antigen_presenting_cell",
              "macrophage",
              "dendritic_cell",
              "tumor_associated_macrophage",
              "TAM"
            ]
          },
          {
            "name": "cf_direction_commit_on_unresolvable_slot",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_not_resolvable_hedge",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].functional_attribution_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_type_I_IFN_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.type_I_IFN",
            "kind": "value",
            "op": "in",
            "expected": [
              "mreg_DC",
              "C1qhigh_macrophage_either_subtype"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-4-pi",
      "modelName": "gpt-5.4",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_enriched_apc_states_v2",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 2415,
      "cost": 1.9225155000000003,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "top_3_cluster_enriched_apc_states",
            "kind": "list",
            "matchKey": "apc_state",
            "k": 3,
            "expected": [
              {
                "key": "plasmacytoid_DC",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "T_cell_stimulatory_role_supported",
                      "role_direction_unresolved"
                    ],
                    "canonical": "T_cell_stimulatory_role_supported"
                  }
                }
              },
              {
                "key": "C1qhigh_lipid_associated_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "C1qhigh_inflammatory_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "plasma_cell",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "cohort_significant_per_patient_inconsistent"
                    ],
                    "canonical": "cohort_significant_per_patient_inconsistent"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CD80_CD86": {
                "op": "in",
                "value": [
                  "mreg_DC",
                  "shared_across_APC_states"
                ]
              },
              "PD_L1_in_APC_clusters": {
                "op": "in",
                "value": [
                  "C1qhigh_macrophage_either_subtype",
                  "shared_across_APC_states",
                  "mreg_DC"
                ]
              },
              "type_I_IFN": {
                "op": "in",
                "value": [
                  "plasmacytoid_DC",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "cf_mreg_dc_claimed_enriched",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "mreg_DC",
              "mature_DC_enriched_in_immunoregulatory_molecules"
            ]
          },
          {
            "name": "cf_framework_offcompartment_or_generic_substitution",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "cDC1",
              "Maier_mregDC",
              "Cheng_C1QC_TAM",
              "Cheng_SPP1_TAM",
              "Cheng_FOLR2_TAM",
              "Villani_pDC",
              "Zhang_inflammatory_monocyte",
              "LAMP3_pos_DC",
              "myeloid_derived_suppressor_cell",
              "MDSC",
              "tumor_associated_neutrophil",
              "mast_cell",
              "APC",
              "antigen_presenting_cell",
              "macrophage",
              "dendritic_cell",
              "tumor_associated_macrophage",
              "TAM"
            ]
          },
          {
            "name": "cf_direction_commit_on_unresolvable_slot",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_not_resolvable_hedge",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].functional_attribution_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_type_I_IFN_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.type_I_IFN",
            "kind": "value",
            "op": "in",
            "expected": [
              "mreg_DC",
              "C1qhigh_macrophage_either_subtype"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-5-codex",
      "modelName": "gpt-5.5",
      "harness": "openai-codex",
      "task": "10x_5p_melanoma_cluster_enriched_apc_states_v2",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 1064,
      "cost": 3.231802,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "top_3_cluster_enriched_apc_states",
            "kind": "list",
            "matchKey": "apc_state",
            "k": 3,
            "expected": [
              {
                "key": "plasmacytoid_DC",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "T_cell_stimulatory_role_supported",
                      "role_direction_unresolved"
                    ],
                    "canonical": "T_cell_stimulatory_role_supported"
                  }
                }
              },
              {
                "key": "C1qhigh_lipid_associated_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "C1qhigh_inflammatory_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "plasma_cell",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "cohort_significant_per_patient_inconsistent"
                    ],
                    "canonical": "cohort_significant_per_patient_inconsistent"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CD80_CD86": {
                "op": "in",
                "value": [
                  "mreg_DC",
                  "shared_across_APC_states"
                ]
              },
              "PD_L1_in_APC_clusters": {
                "op": "in",
                "value": [
                  "C1qhigh_macrophage_either_subtype",
                  "shared_across_APC_states",
                  "mreg_DC"
                ]
              },
              "type_I_IFN": {
                "op": "in",
                "value": [
                  "plasmacytoid_DC",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "cf_mreg_dc_claimed_enriched",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "mreg_DC",
              "mature_DC_enriched_in_immunoregulatory_molecules"
            ]
          },
          {
            "name": "cf_framework_offcompartment_or_generic_substitution",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "cDC1",
              "Maier_mregDC",
              "Cheng_C1QC_TAM",
              "Cheng_SPP1_TAM",
              "Cheng_FOLR2_TAM",
              "Villani_pDC",
              "Zhang_inflammatory_monocyte",
              "LAMP3_pos_DC",
              "myeloid_derived_suppressor_cell",
              "MDSC",
              "tumor_associated_neutrophil",
              "mast_cell",
              "APC",
              "antigen_presenting_cell",
              "macrophage",
              "dendritic_cell",
              "tumor_associated_macrophage",
              "TAM"
            ]
          },
          {
            "name": "cf_direction_commit_on_unresolvable_slot",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_not_resolvable_hedge",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].functional_attribution_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_type_I_IFN_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.type_I_IFN",
            "kind": "value",
            "op": "in",
            "expected": [
              "mreg_DC",
              "C1qhigh_macrophage_either_subtype"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-5-codex",
      "modelName": "gpt-5.5",
      "harness": "openai-codex",
      "task": "10x_5p_melanoma_cluster_enriched_apc_states_v2",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 767,
      "cost": 2.6874130000000003,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "top_3_cluster_enriched_apc_states",
            "kind": "list",
            "matchKey": "apc_state",
            "k": 3,
            "expected": [
              {
                "key": "plasmacytoid_DC",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "T_cell_stimulatory_role_supported",
                      "role_direction_unresolved"
                    ],
                    "canonical": "T_cell_stimulatory_role_supported"
                  }
                }
              },
              {
                "key": "C1qhigh_lipid_associated_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "C1qhigh_inflammatory_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "plasma_cell",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "cohort_significant_per_patient_inconsistent"
                    ],
                    "canonical": "cohort_significant_per_patient_inconsistent"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CD80_CD86": {
                "op": "in",
                "value": [
                  "mreg_DC",
                  "shared_across_APC_states"
                ]
              },
              "PD_L1_in_APC_clusters": {
                "op": "in",
                "value": [
                  "C1qhigh_macrophage_either_subtype",
                  "shared_across_APC_states",
                  "mreg_DC"
                ]
              },
              "type_I_IFN": {
                "op": "in",
                "value": [
                  "plasmacytoid_DC",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "cf_mreg_dc_claimed_enriched",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "mreg_DC",
              "mature_DC_enriched_in_immunoregulatory_molecules"
            ]
          },
          {
            "name": "cf_framework_offcompartment_or_generic_substitution",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "cDC1",
              "Maier_mregDC",
              "Cheng_C1QC_TAM",
              "Cheng_SPP1_TAM",
              "Cheng_FOLR2_TAM",
              "Villani_pDC",
              "Zhang_inflammatory_monocyte",
              "LAMP3_pos_DC",
              "myeloid_derived_suppressor_cell",
              "MDSC",
              "tumor_associated_neutrophil",
              "mast_cell",
              "APC",
              "antigen_presenting_cell",
              "macrophage",
              "dendritic_cell",
              "tumor_associated_macrophage",
              "TAM"
            ]
          },
          {
            "name": "cf_direction_commit_on_unresolvable_slot",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_not_resolvable_hedge",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].functional_attribution_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_type_I_IFN_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.type_I_IFN",
            "kind": "value",
            "op": "in",
            "expected": [
              "mreg_DC",
              "C1qhigh_macrophage_either_subtype"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-5-codex",
      "modelName": "gpt-5.5",
      "harness": "openai-codex",
      "task": "10x_5p_melanoma_cluster_enriched_apc_states_v2",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1035,
      "cost": 2.603696,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "top_3_cluster_enriched_apc_states",
            "kind": "list",
            "matchKey": "apc_state",
            "k": 3,
            "expected": [
              {
                "key": "plasmacytoid_DC",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "T_cell_stimulatory_role_supported",
                      "role_direction_unresolved"
                    ],
                    "canonical": "T_cell_stimulatory_role_supported"
                  }
                }
              },
              {
                "key": "C1qhigh_lipid_associated_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "C1qhigh_inflammatory_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "plasma_cell",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "cohort_significant_per_patient_inconsistent"
                    ],
                    "canonical": "cohort_significant_per_patient_inconsistent"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CD80_CD86": {
                "op": "in",
                "value": [
                  "mreg_DC",
                  "shared_across_APC_states"
                ]
              },
              "PD_L1_in_APC_clusters": {
                "op": "in",
                "value": [
                  "C1qhigh_macrophage_either_subtype",
                  "shared_across_APC_states",
                  "mreg_DC"
                ]
              },
              "type_I_IFN": {
                "op": "in",
                "value": [
                  "plasmacytoid_DC",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "cf_mreg_dc_claimed_enriched",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "mreg_DC",
              "mature_DC_enriched_in_immunoregulatory_molecules"
            ]
          },
          {
            "name": "cf_framework_offcompartment_or_generic_substitution",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "cDC1",
              "Maier_mregDC",
              "Cheng_C1QC_TAM",
              "Cheng_SPP1_TAM",
              "Cheng_FOLR2_TAM",
              "Villani_pDC",
              "Zhang_inflammatory_monocyte",
              "LAMP3_pos_DC",
              "myeloid_derived_suppressor_cell",
              "MDSC",
              "tumor_associated_neutrophil",
              "mast_cell",
              "APC",
              "antigen_presenting_cell",
              "macrophage",
              "dendritic_cell",
              "tumor_associated_macrophage",
              "TAM"
            ]
          },
          {
            "name": "cf_direction_commit_on_unresolvable_slot",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_not_resolvable_hedge",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].functional_attribution_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_type_I_IFN_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.type_I_IFN",
            "kind": "value",
            "op": "in",
            "expected": [
              "mreg_DC",
              "C1qhigh_macrophage_either_subtype"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-5-pi",
      "modelName": "gpt-5.5",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_enriched_apc_states_v2",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 1661,
      "cost": 4.2407829999999995,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "top_3_cluster_enriched_apc_states",
            "kind": "list",
            "matchKey": "apc_state",
            "k": 3,
            "expected": [
              {
                "key": "plasmacytoid_DC",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "T_cell_stimulatory_role_supported",
                      "role_direction_unresolved"
                    ],
                    "canonical": "T_cell_stimulatory_role_supported"
                  }
                }
              },
              {
                "key": "C1qhigh_lipid_associated_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "C1qhigh_inflammatory_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "plasma_cell",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "cohort_significant_per_patient_inconsistent"
                    ],
                    "canonical": "cohort_significant_per_patient_inconsistent"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CD80_CD86": {
                "op": "in",
                "value": [
                  "mreg_DC",
                  "shared_across_APC_states"
                ]
              },
              "PD_L1_in_APC_clusters": {
                "op": "in",
                "value": [
                  "C1qhigh_macrophage_either_subtype",
                  "shared_across_APC_states",
                  "mreg_DC"
                ]
              },
              "type_I_IFN": {
                "op": "in",
                "value": [
                  "plasmacytoid_DC",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "cf_mreg_dc_claimed_enriched",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "mreg_DC",
              "mature_DC_enriched_in_immunoregulatory_molecules"
            ]
          },
          {
            "name": "cf_framework_offcompartment_or_generic_substitution",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "cDC1",
              "Maier_mregDC",
              "Cheng_C1QC_TAM",
              "Cheng_SPP1_TAM",
              "Cheng_FOLR2_TAM",
              "Villani_pDC",
              "Zhang_inflammatory_monocyte",
              "LAMP3_pos_DC",
              "myeloid_derived_suppressor_cell",
              "MDSC",
              "tumor_associated_neutrophil",
              "mast_cell",
              "APC",
              "antigen_presenting_cell",
              "macrophage",
              "dendritic_cell",
              "tumor_associated_macrophage",
              "TAM"
            ]
          },
          {
            "name": "cf_direction_commit_on_unresolvable_slot",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_not_resolvable_hedge",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].functional_attribution_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_type_I_IFN_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.type_I_IFN",
            "kind": "value",
            "op": "in",
            "expected": [
              "mreg_DC",
              "C1qhigh_macrophage_either_subtype"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-5-pi",
      "modelName": "gpt-5.5",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_enriched_apc_states_v2",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 1353,
      "cost": 2.6342229999999995,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "top_3_cluster_enriched_apc_states",
            "kind": "list",
            "matchKey": "apc_state",
            "k": 3,
            "expected": [
              {
                "key": "plasmacytoid_DC",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "T_cell_stimulatory_role_supported",
                      "role_direction_unresolved"
                    ],
                    "canonical": "T_cell_stimulatory_role_supported"
                  }
                }
              },
              {
                "key": "C1qhigh_lipid_associated_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "C1qhigh_inflammatory_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "plasma_cell",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "cohort_significant_per_patient_inconsistent"
                    ],
                    "canonical": "cohort_significant_per_patient_inconsistent"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CD80_CD86": {
                "op": "in",
                "value": [
                  "mreg_DC",
                  "shared_across_APC_states"
                ]
              },
              "PD_L1_in_APC_clusters": {
                "op": "in",
                "value": [
                  "C1qhigh_macrophage_either_subtype",
                  "shared_across_APC_states",
                  "mreg_DC"
                ]
              },
              "type_I_IFN": {
                "op": "in",
                "value": [
                  "plasmacytoid_DC",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "cf_mreg_dc_claimed_enriched",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "mreg_DC",
              "mature_DC_enriched_in_immunoregulatory_molecules"
            ]
          },
          {
            "name": "cf_framework_offcompartment_or_generic_substitution",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "cDC1",
              "Maier_mregDC",
              "Cheng_C1QC_TAM",
              "Cheng_SPP1_TAM",
              "Cheng_FOLR2_TAM",
              "Villani_pDC",
              "Zhang_inflammatory_monocyte",
              "LAMP3_pos_DC",
              "myeloid_derived_suppressor_cell",
              "MDSC",
              "tumor_associated_neutrophil",
              "mast_cell",
              "APC",
              "antigen_presenting_cell",
              "macrophage",
              "dendritic_cell",
              "tumor_associated_macrophage",
              "TAM"
            ]
          },
          {
            "name": "cf_direction_commit_on_unresolvable_slot",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_not_resolvable_hedge",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].functional_attribution_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_type_I_IFN_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.type_I_IFN",
            "kind": "value",
            "op": "in",
            "expected": [
              "mreg_DC",
              "C1qhigh_macrophage_either_subtype"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-5-pi",
      "modelName": "gpt-5.5",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_enriched_apc_states_v2",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1297,
      "cost": 3.5808219999999995,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "top_3_cluster_enriched_apc_states",
            "kind": "list",
            "matchKey": "apc_state",
            "k": 3,
            "expected": [
              {
                "key": "plasmacytoid_DC",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "T_cell_stimulatory_role_supported",
                      "role_direction_unresolved"
                    ],
                    "canonical": "T_cell_stimulatory_role_supported"
                  }
                }
              },
              {
                "key": "C1qhigh_lipid_associated_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "C1qhigh_inflammatory_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "plasma_cell",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "cohort_significant_per_patient_inconsistent"
                    ],
                    "canonical": "cohort_significant_per_patient_inconsistent"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CD80_CD86": {
                "op": "in",
                "value": [
                  "mreg_DC",
                  "shared_across_APC_states"
                ]
              },
              "PD_L1_in_APC_clusters": {
                "op": "in",
                "value": [
                  "C1qhigh_macrophage_either_subtype",
                  "shared_across_APC_states",
                  "mreg_DC"
                ]
              },
              "type_I_IFN": {
                "op": "in",
                "value": [
                  "plasmacytoid_DC",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "cf_mreg_dc_claimed_enriched",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "mreg_DC",
              "mature_DC_enriched_in_immunoregulatory_molecules"
            ]
          },
          {
            "name": "cf_framework_offcompartment_or_generic_substitution",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "cDC1",
              "Maier_mregDC",
              "Cheng_C1QC_TAM",
              "Cheng_SPP1_TAM",
              "Cheng_FOLR2_TAM",
              "Villani_pDC",
              "Zhang_inflammatory_monocyte",
              "LAMP3_pos_DC",
              "myeloid_derived_suppressor_cell",
              "MDSC",
              "tumor_associated_neutrophil",
              "mast_cell",
              "APC",
              "antigen_presenting_cell",
              "macrophage",
              "dendritic_cell",
              "tumor_associated_macrophage",
              "TAM"
            ]
          },
          {
            "name": "cf_direction_commit_on_unresolvable_slot",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_not_resolvable_hedge",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].functional_attribution_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_type_I_IFN_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.type_I_IFN",
            "kind": "value",
            "op": "in",
            "expected": [
              "mreg_DC",
              "C1qhigh_macrophage_either_subtype"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-20-beta-reasoning-pi",
      "modelName": "grok-4.20-beta-0309-reasoning",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_enriched_apc_states_v2",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 551,
      "cost": 0.04135625000000001,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "top_3_cluster_enriched_apc_states",
            "kind": "list",
            "matchKey": "apc_state",
            "k": 3,
            "expected": [
              {
                "key": "plasmacytoid_DC",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "T_cell_stimulatory_role_supported",
                      "role_direction_unresolved"
                    ],
                    "canonical": "T_cell_stimulatory_role_supported"
                  }
                }
              },
              {
                "key": "C1qhigh_lipid_associated_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "C1qhigh_inflammatory_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "plasma_cell",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "cohort_significant_per_patient_inconsistent"
                    ],
                    "canonical": "cohort_significant_per_patient_inconsistent"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CD80_CD86": {
                "op": "in",
                "value": [
                  "mreg_DC",
                  "shared_across_APC_states"
                ]
              },
              "PD_L1_in_APC_clusters": {
                "op": "in",
                "value": [
                  "C1qhigh_macrophage_either_subtype",
                  "shared_across_APC_states",
                  "mreg_DC"
                ]
              },
              "type_I_IFN": {
                "op": "in",
                "value": [
                  "plasmacytoid_DC",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "cf_mreg_dc_claimed_enriched",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "mreg_DC",
              "mature_DC_enriched_in_immunoregulatory_molecules"
            ]
          },
          {
            "name": "cf_framework_offcompartment_or_generic_substitution",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "cDC1",
              "Maier_mregDC",
              "Cheng_C1QC_TAM",
              "Cheng_SPP1_TAM",
              "Cheng_FOLR2_TAM",
              "Villani_pDC",
              "Zhang_inflammatory_monocyte",
              "LAMP3_pos_DC",
              "myeloid_derived_suppressor_cell",
              "MDSC",
              "tumor_associated_neutrophil",
              "mast_cell",
              "APC",
              "antigen_presenting_cell",
              "macrophage",
              "dendritic_cell",
              "tumor_associated_macrophage",
              "TAM"
            ]
          },
          {
            "name": "cf_direction_commit_on_unresolvable_slot",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_not_resolvable_hedge",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].functional_attribution_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_type_I_IFN_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.type_I_IFN",
            "kind": "value",
            "op": "in",
            "expected": [
              "mreg_DC",
              "C1qhigh_macrophage_either_subtype"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-20-beta-reasoning-pi",
      "modelName": "grok-4.20-beta-0309-reasoning",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_enriched_apc_states_v2",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 654,
      "cost": 0.0686977,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "top_3_cluster_enriched_apc_states",
            "kind": "list",
            "matchKey": "apc_state",
            "k": 3,
            "expected": [
              {
                "key": "plasmacytoid_DC",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "T_cell_stimulatory_role_supported",
                      "role_direction_unresolved"
                    ],
                    "canonical": "T_cell_stimulatory_role_supported"
                  }
                }
              },
              {
                "key": "C1qhigh_lipid_associated_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "C1qhigh_inflammatory_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "plasma_cell",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "cohort_significant_per_patient_inconsistent"
                    ],
                    "canonical": "cohort_significant_per_patient_inconsistent"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CD80_CD86": {
                "op": "in",
                "value": [
                  "mreg_DC",
                  "shared_across_APC_states"
                ]
              },
              "PD_L1_in_APC_clusters": {
                "op": "in",
                "value": [
                  "C1qhigh_macrophage_either_subtype",
                  "shared_across_APC_states",
                  "mreg_DC"
                ]
              },
              "type_I_IFN": {
                "op": "in",
                "value": [
                  "plasmacytoid_DC",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "cf_mreg_dc_claimed_enriched",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "mreg_DC",
              "mature_DC_enriched_in_immunoregulatory_molecules"
            ]
          },
          {
            "name": "cf_framework_offcompartment_or_generic_substitution",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "cDC1",
              "Maier_mregDC",
              "Cheng_C1QC_TAM",
              "Cheng_SPP1_TAM",
              "Cheng_FOLR2_TAM",
              "Villani_pDC",
              "Zhang_inflammatory_monocyte",
              "LAMP3_pos_DC",
              "myeloid_derived_suppressor_cell",
              "MDSC",
              "tumor_associated_neutrophil",
              "mast_cell",
              "APC",
              "antigen_presenting_cell",
              "macrophage",
              "dendritic_cell",
              "tumor_associated_macrophage",
              "TAM"
            ]
          },
          {
            "name": "cf_direction_commit_on_unresolvable_slot",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_not_resolvable_hedge",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].functional_attribution_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_type_I_IFN_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.type_I_IFN",
            "kind": "value",
            "op": "in",
            "expected": [
              "mreg_DC",
              "C1qhigh_macrophage_either_subtype"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-20-beta-reasoning-pi",
      "modelName": "grok-4.20-beta-0309-reasoning",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_enriched_apc_states_v2",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 54,
      "cost": 0.035325,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "top_3_cluster_enriched_apc_states",
            "kind": "list",
            "matchKey": "apc_state",
            "k": 3,
            "expected": [
              {
                "key": "plasmacytoid_DC",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "T_cell_stimulatory_role_supported",
                      "role_direction_unresolved"
                    ],
                    "canonical": "T_cell_stimulatory_role_supported"
                  }
                }
              },
              {
                "key": "C1qhigh_lipid_associated_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "C1qhigh_inflammatory_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "plasma_cell",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "cohort_significant_per_patient_inconsistent"
                    ],
                    "canonical": "cohort_significant_per_patient_inconsistent"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CD80_CD86": {
                "op": "in",
                "value": [
                  "mreg_DC",
                  "shared_across_APC_states"
                ]
              },
              "PD_L1_in_APC_clusters": {
                "op": "in",
                "value": [
                  "C1qhigh_macrophage_either_subtype",
                  "shared_across_APC_states",
                  "mreg_DC"
                ]
              },
              "type_I_IFN": {
                "op": "in",
                "value": [
                  "plasmacytoid_DC",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "cf_mreg_dc_claimed_enriched",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "mreg_DC",
              "mature_DC_enriched_in_immunoregulatory_molecules"
            ]
          },
          {
            "name": "cf_framework_offcompartment_or_generic_substitution",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "cDC1",
              "Maier_mregDC",
              "Cheng_C1QC_TAM",
              "Cheng_SPP1_TAM",
              "Cheng_FOLR2_TAM",
              "Villani_pDC",
              "Zhang_inflammatory_monocyte",
              "LAMP3_pos_DC",
              "myeloid_derived_suppressor_cell",
              "MDSC",
              "tumor_associated_neutrophil",
              "mast_cell",
              "APC",
              "antigen_presenting_cell",
              "macrophage",
              "dendritic_cell",
              "tumor_associated_macrophage",
              "TAM"
            ]
          },
          {
            "name": "cf_direction_commit_on_unresolvable_slot",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_not_resolvable_hedge",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].functional_attribution_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_type_I_IFN_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.type_I_IFN",
            "kind": "value",
            "op": "in",
            "expected": [
              "mreg_DC",
              "C1qhigh_macrophage_either_subtype"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-3-pi",
      "modelName": "grok-4.3",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_enriched_apc_states_v2",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 81,
      "cost": 0.01236925,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "top_3_cluster_enriched_apc_states",
            "kind": "list",
            "matchKey": "apc_state",
            "k": 3,
            "expected": [
              {
                "key": "plasmacytoid_DC",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "T_cell_stimulatory_role_supported",
                      "role_direction_unresolved"
                    ],
                    "canonical": "T_cell_stimulatory_role_supported"
                  }
                }
              },
              {
                "key": "C1qhigh_lipid_associated_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "C1qhigh_inflammatory_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "plasma_cell",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "cohort_significant_per_patient_inconsistent"
                    ],
                    "canonical": "cohort_significant_per_patient_inconsistent"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CD80_CD86": {
                "op": "in",
                "value": [
                  "mreg_DC",
                  "shared_across_APC_states"
                ]
              },
              "PD_L1_in_APC_clusters": {
                "op": "in",
                "value": [
                  "C1qhigh_macrophage_either_subtype",
                  "shared_across_APC_states",
                  "mreg_DC"
                ]
              },
              "type_I_IFN": {
                "op": "in",
                "value": [
                  "plasmacytoid_DC",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "cf_mreg_dc_claimed_enriched",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "mreg_DC",
              "mature_DC_enriched_in_immunoregulatory_molecules"
            ]
          },
          {
            "name": "cf_framework_offcompartment_or_generic_substitution",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "cDC1",
              "Maier_mregDC",
              "Cheng_C1QC_TAM",
              "Cheng_SPP1_TAM",
              "Cheng_FOLR2_TAM",
              "Villani_pDC",
              "Zhang_inflammatory_monocyte",
              "LAMP3_pos_DC",
              "myeloid_derived_suppressor_cell",
              "MDSC",
              "tumor_associated_neutrophil",
              "mast_cell",
              "APC",
              "antigen_presenting_cell",
              "macrophage",
              "dendritic_cell",
              "tumor_associated_macrophage",
              "TAM"
            ]
          },
          {
            "name": "cf_direction_commit_on_unresolvable_slot",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_not_resolvable_hedge",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].functional_attribution_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_type_I_IFN_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.type_I_IFN",
            "kind": "value",
            "op": "in",
            "expected": [
              "mreg_DC",
              "C1qhigh_macrophage_either_subtype"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-3-pi",
      "modelName": "grok-4.3",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_enriched_apc_states_v2",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 68,
      "cost": 0.010244250000000002,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "top_3_cluster_enriched_apc_states",
            "kind": "list",
            "matchKey": "apc_state",
            "k": 3,
            "expected": [
              {
                "key": "plasmacytoid_DC",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "T_cell_stimulatory_role_supported",
                      "role_direction_unresolved"
                    ],
                    "canonical": "T_cell_stimulatory_role_supported"
                  }
                }
              },
              {
                "key": "C1qhigh_lipid_associated_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "C1qhigh_inflammatory_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "plasma_cell",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "cohort_significant_per_patient_inconsistent"
                    ],
                    "canonical": "cohort_significant_per_patient_inconsistent"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CD80_CD86": {
                "op": "in",
                "value": [
                  "mreg_DC",
                  "shared_across_APC_states"
                ]
              },
              "PD_L1_in_APC_clusters": {
                "op": "in",
                "value": [
                  "C1qhigh_macrophage_either_subtype",
                  "shared_across_APC_states",
                  "mreg_DC"
                ]
              },
              "type_I_IFN": {
                "op": "in",
                "value": [
                  "plasmacytoid_DC",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "cf_mreg_dc_claimed_enriched",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "mreg_DC",
              "mature_DC_enriched_in_immunoregulatory_molecules"
            ]
          },
          {
            "name": "cf_framework_offcompartment_or_generic_substitution",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "cDC1",
              "Maier_mregDC",
              "Cheng_C1QC_TAM",
              "Cheng_SPP1_TAM",
              "Cheng_FOLR2_TAM",
              "Villani_pDC",
              "Zhang_inflammatory_monocyte",
              "LAMP3_pos_DC",
              "myeloid_derived_suppressor_cell",
              "MDSC",
              "tumor_associated_neutrophil",
              "mast_cell",
              "APC",
              "antigen_presenting_cell",
              "macrophage",
              "dendritic_cell",
              "tumor_associated_macrophage",
              "TAM"
            ]
          },
          {
            "name": "cf_direction_commit_on_unresolvable_slot",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_not_resolvable_hedge",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].functional_attribution_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_type_I_IFN_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.type_I_IFN",
            "kind": "value",
            "op": "in",
            "expected": [
              "mreg_DC",
              "C1qhigh_macrophage_either_subtype"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-3-pi",
      "modelName": "grok-4.3",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_enriched_apc_states_v2",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 81,
      "cost": 0.011299900000000002,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "top_3_cluster_enriched_apc_states",
            "kind": "list",
            "matchKey": "apc_state",
            "k": 3,
            "expected": [
              {
                "key": "plasmacytoid_DC",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "T_cell_stimulatory_role_supported",
                      "role_direction_unresolved"
                    ],
                    "canonical": "T_cell_stimulatory_role_supported"
                  }
                }
              },
              {
                "key": "C1qhigh_lipid_associated_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "C1qhigh_inflammatory_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "plasma_cell",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "cohort_significant_per_patient_inconsistent"
                    ],
                    "canonical": "cohort_significant_per_patient_inconsistent"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CD80_CD86": {
                "op": "in",
                "value": [
                  "mreg_DC",
                  "shared_across_APC_states"
                ]
              },
              "PD_L1_in_APC_clusters": {
                "op": "in",
                "value": [
                  "C1qhigh_macrophage_either_subtype",
                  "shared_across_APC_states",
                  "mreg_DC"
                ]
              },
              "type_I_IFN": {
                "op": "in",
                "value": [
                  "plasmacytoid_DC",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "cf_mreg_dc_claimed_enriched",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "mreg_DC",
              "mature_DC_enriched_in_immunoregulatory_molecules"
            ]
          },
          {
            "name": "cf_framework_offcompartment_or_generic_substitution",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "cDC1",
              "Maier_mregDC",
              "Cheng_C1QC_TAM",
              "Cheng_SPP1_TAM",
              "Cheng_FOLR2_TAM",
              "Villani_pDC",
              "Zhang_inflammatory_monocyte",
              "LAMP3_pos_DC",
              "myeloid_derived_suppressor_cell",
              "MDSC",
              "tumor_associated_neutrophil",
              "mast_cell",
              "APC",
              "antigen_presenting_cell",
              "macrophage",
              "dendritic_cell",
              "tumor_associated_macrophage",
              "TAM"
            ]
          },
          {
            "name": "cf_direction_commit_on_unresolvable_slot",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_not_resolvable_hedge",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].functional_attribution_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_type_I_IFN_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.type_I_IFN",
            "kind": "value",
            "op": "in",
            "expected": [
              "mreg_DC",
              "C1qhigh_macrophage_either_subtype"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-kimi-k2p6-pi",
      "modelName": "kimi-k2p6",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_enriched_apc_states_v2",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 2046,
      "cost": 0.16325689000000002,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "top_3_cluster_enriched_apc_states",
            "kind": "list",
            "matchKey": "apc_state",
            "k": 3,
            "expected": [
              {
                "key": "plasmacytoid_DC",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "T_cell_stimulatory_role_supported",
                      "role_direction_unresolved"
                    ],
                    "canonical": "T_cell_stimulatory_role_supported"
                  }
                }
              },
              {
                "key": "C1qhigh_lipid_associated_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "C1qhigh_inflammatory_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "plasma_cell",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "cohort_significant_per_patient_inconsistent"
                    ],
                    "canonical": "cohort_significant_per_patient_inconsistent"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CD80_CD86": {
                "op": "in",
                "value": [
                  "mreg_DC",
                  "shared_across_APC_states"
                ]
              },
              "PD_L1_in_APC_clusters": {
                "op": "in",
                "value": [
                  "C1qhigh_macrophage_either_subtype",
                  "shared_across_APC_states",
                  "mreg_DC"
                ]
              },
              "type_I_IFN": {
                "op": "in",
                "value": [
                  "plasmacytoid_DC",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "cf_mreg_dc_claimed_enriched",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "mreg_DC",
              "mature_DC_enriched_in_immunoregulatory_molecules"
            ]
          },
          {
            "name": "cf_framework_offcompartment_or_generic_substitution",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "cDC1",
              "Maier_mregDC",
              "Cheng_C1QC_TAM",
              "Cheng_SPP1_TAM",
              "Cheng_FOLR2_TAM",
              "Villani_pDC",
              "Zhang_inflammatory_monocyte",
              "LAMP3_pos_DC",
              "myeloid_derived_suppressor_cell",
              "MDSC",
              "tumor_associated_neutrophil",
              "mast_cell",
              "APC",
              "antigen_presenting_cell",
              "macrophage",
              "dendritic_cell",
              "tumor_associated_macrophage",
              "TAM"
            ]
          },
          {
            "name": "cf_direction_commit_on_unresolvable_slot",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_not_resolvable_hedge",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].functional_attribution_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_type_I_IFN_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.type_I_IFN",
            "kind": "value",
            "op": "in",
            "expected": [
              "mreg_DC",
              "C1qhigh_macrophage_either_subtype"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-kimi-k2p6-pi",
      "modelName": "kimi-k2p6",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_enriched_apc_states_v2",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 2386,
      "cost": 1.44108949,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "top_3_cluster_enriched_apc_states",
            "kind": "list",
            "matchKey": "apc_state",
            "k": 3,
            "expected": [
              {
                "key": "plasmacytoid_DC",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "T_cell_stimulatory_role_supported",
                      "role_direction_unresolved"
                    ],
                    "canonical": "T_cell_stimulatory_role_supported"
                  }
                }
              },
              {
                "key": "C1qhigh_lipid_associated_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "C1qhigh_inflammatory_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "plasma_cell",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "cohort_significant_per_patient_inconsistent"
                    ],
                    "canonical": "cohort_significant_per_patient_inconsistent"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CD80_CD86": {
                "op": "in",
                "value": [
                  "mreg_DC",
                  "shared_across_APC_states"
                ]
              },
              "PD_L1_in_APC_clusters": {
                "op": "in",
                "value": [
                  "C1qhigh_macrophage_either_subtype",
                  "shared_across_APC_states",
                  "mreg_DC"
                ]
              },
              "type_I_IFN": {
                "op": "in",
                "value": [
                  "plasmacytoid_DC",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "cf_mreg_dc_claimed_enriched",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "mreg_DC",
              "mature_DC_enriched_in_immunoregulatory_molecules"
            ]
          },
          {
            "name": "cf_framework_offcompartment_or_generic_substitution",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "cDC1",
              "Maier_mregDC",
              "Cheng_C1QC_TAM",
              "Cheng_SPP1_TAM",
              "Cheng_FOLR2_TAM",
              "Villani_pDC",
              "Zhang_inflammatory_monocyte",
              "LAMP3_pos_DC",
              "myeloid_derived_suppressor_cell",
              "MDSC",
              "tumor_associated_neutrophil",
              "mast_cell",
              "APC",
              "antigen_presenting_cell",
              "macrophage",
              "dendritic_cell",
              "tumor_associated_macrophage",
              "TAM"
            ]
          },
          {
            "name": "cf_direction_commit_on_unresolvable_slot",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_not_resolvable_hedge",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].functional_attribution_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_type_I_IFN_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.type_I_IFN",
            "kind": "value",
            "op": "in",
            "expected": [
              "mreg_DC",
              "C1qhigh_macrophage_either_subtype"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-kimi-k2p6-pi",
      "modelName": "kimi-k2p6",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_enriched_apc_states_v2",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1918,
      "cost": 0.7198824799999999,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "top_3_cluster_enriched_apc_states",
            "kind": "list",
            "matchKey": "apc_state",
            "k": 3,
            "expected": [
              {
                "key": "plasmacytoid_DC",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "T_cell_stimulatory_role_supported",
                      "role_direction_unresolved"
                    ],
                    "canonical": "T_cell_stimulatory_role_supported"
                  }
                }
              },
              {
                "key": "C1qhigh_lipid_associated_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "C1qhigh_inflammatory_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "plasma_cell",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "cohort_significant_per_patient_inconsistent"
                    ],
                    "canonical": "cohort_significant_per_patient_inconsistent"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CD80_CD86": {
                "op": "in",
                "value": [
                  "mreg_DC",
                  "shared_across_APC_states"
                ]
              },
              "PD_L1_in_APC_clusters": {
                "op": "in",
                "value": [
                  "C1qhigh_macrophage_either_subtype",
                  "shared_across_APC_states",
                  "mreg_DC"
                ]
              },
              "type_I_IFN": {
                "op": "in",
                "value": [
                  "plasmacytoid_DC",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "cf_mreg_dc_claimed_enriched",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "mreg_DC",
              "mature_DC_enriched_in_immunoregulatory_molecules"
            ]
          },
          {
            "name": "cf_framework_offcompartment_or_generic_substitution",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "cDC1",
              "Maier_mregDC",
              "Cheng_C1QC_TAM",
              "Cheng_SPP1_TAM",
              "Cheng_FOLR2_TAM",
              "Villani_pDC",
              "Zhang_inflammatory_monocyte",
              "LAMP3_pos_DC",
              "myeloid_derived_suppressor_cell",
              "MDSC",
              "tumor_associated_neutrophil",
              "mast_cell",
              "APC",
              "antigen_presenting_cell",
              "macrophage",
              "dendritic_cell",
              "tumor_associated_macrophage",
              "TAM"
            ]
          },
          {
            "name": "cf_direction_commit_on_unresolvable_slot",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_not_resolvable_hedge",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].functional_attribution_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_type_I_IFN_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.type_I_IFN",
            "kind": "value",
            "op": "in",
            "expected": [
              "mreg_DC",
              "C1qhigh_macrophage_either_subtype"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-6-claude-code",
      "modelName": "claude-opus-4-6",
      "harness": "claude-code",
      "task": "10x_5p_melanoma_cluster_tumor_state_causation_v5",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 2529,
      "cost": 2.709991,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "enriched_tumor_states",
            "kind": "list",
            "matchKey": "tumor_state",
            "k": 3,
            "expected": [
              {
                "key": "immune_response",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "direction_unresolved_cross_sectional"
                    ],
                    "canonical": "direction_unresolved_cross_sectional"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "IFN_response_axis",
                      "broad_antigen_presentation_MHC"
                    ],
                    "canonical": "IFN_response_axis"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "enrichment_diminished_by_decontamination",
                      "threshold_sensitive"
                    ],
                    "canonical": "enrichment_diminished_by_decontamination"
                  }
                }
              },
              {
                "key": "stress_hypoxia",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "data_supports_cell_intrinsic_cause"
                    ],
                    "canonical": "data_supports_cell_intrinsic_cause"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "HIF_hypoxia_program_specific"
                    ],
                    "canonical": "HIF_hypoxia_program_specific"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "robust_across_methods",
                      "robust_after_decontamination"
                    ],
                    "canonical": "robust_across_methods"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CXCL9_CXCL10": {
                "op": "in",
                "value": [
                  "immune_response_tumor_state"
                ]
              },
              "CCL5": {
                "op": "in",
                "value": [
                  "shared_across_tumor_states",
                  "partner_lymphocyte_derived",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "gate_conjugate_gate_confounder",
            "role": "gate",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "partner_lymphocyte_capture"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "antigen_presentation_decomposition",
            "kind": "dict",
            "expected": {
              "MHC_class_I": {
                "op": "in",
                "value": [
                  "co_regulated_with_dominant_program",
                  "tumor_intrinsic_enriched_independent"
                ]
              },
              "MHC_class_II": {
                "op": "in",
                "value": [
                  "partner_cell_attributable",
                  "flat_not_enriched",
                  "method_sensitive_no_robust_call",
                  "co_regulated_with_dominant_program"
                ]
              }
            }
          },
          {
            "name": "cf_non_robust_state_claimed_enriched",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "transitory_melanocytic",
              "intermediate_melanocytic",
              "melanocytic",
              "MITF_high",
              "neural_crest_like",
              "mitotic",
              "stress_p53",
              "undifferentiated",
              "dedifferentiated",
              "mesenchymal_tumor",
              "AXL_high_tumor",
              "invasive_tumor",
              "senescence_like",
              "EMT_transition",
              "cancer_stem_cell_like",
              "oxidative_stress_tumor"
            ]
          },
          {
            "name": "cf_framework_generic_or_offtissue_substitution",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "Tsoi_undifferentiated",
              "Tsoi_neural_crest_like",
              "Tsoi_transitory",
              "Tsoi_melanocytic",
              "Rambow_NCSC",
              "Wouters_mesenchymal",
              "Pozniak_TCF4_dependent",
              "tumor_cell",
              "melanoma_cell",
              "malignant_cell",
              "tumor_subpopulation",
              "keratinocyte",
              "hepatocyte",
              "lung_adenocarcinoma_cell",
              "colon_adenocarcinoma_cell",
              "Tirosh_cell_cycle_program",
              "Jerby_Arnon_program",
              "glioblastoma_cell",
              "breast_carcinoma_cell"
            ]
          },
          {
            "name": "cf_immune_response_direction_committed",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_immune_robustness_recall",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_ambiguous_hedge",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].causation_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_stress_generic_stress_response_evidence",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_conjugate_confounder_contamination_denied",
            "role": "hard_fail",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected"
            ]
          },
          {
            "name": "cf_ccl5_recall_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CCL5",
            "kind": "avoid",
            "expected": [
              "immune_response_tumor_state"
            ]
          },
          {
            "name": "cf_cxcl9_cxcl10_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CXCL9_CXCL10",
            "kind": "avoid",
            "expected": [
              "stress_hypoxia_tumor_state"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-6-claude-code",
      "modelName": "claude-opus-4-6",
      "harness": "claude-code",
      "task": "10x_5p_melanoma_cluster_tumor_state_causation_v5",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 3279,
      "cost": 4.8706457499999996,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "enriched_tumor_states",
            "kind": "list",
            "matchKey": "tumor_state",
            "k": 3,
            "expected": [
              {
                "key": "immune_response",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "direction_unresolved_cross_sectional"
                    ],
                    "canonical": "direction_unresolved_cross_sectional"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "IFN_response_axis",
                      "broad_antigen_presentation_MHC"
                    ],
                    "canonical": "IFN_response_axis"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "enrichment_diminished_by_decontamination",
                      "threshold_sensitive"
                    ],
                    "canonical": "enrichment_diminished_by_decontamination"
                  }
                }
              },
              {
                "key": "stress_hypoxia",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "data_supports_cell_intrinsic_cause"
                    ],
                    "canonical": "data_supports_cell_intrinsic_cause"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "HIF_hypoxia_program_specific"
                    ],
                    "canonical": "HIF_hypoxia_program_specific"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "robust_across_methods",
                      "robust_after_decontamination"
                    ],
                    "canonical": "robust_across_methods"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CXCL9_CXCL10": {
                "op": "in",
                "value": [
                  "immune_response_tumor_state"
                ]
              },
              "CCL5": {
                "op": "in",
                "value": [
                  "shared_across_tumor_states",
                  "partner_lymphocyte_derived",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "gate_conjugate_gate_confounder",
            "role": "gate",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "partner_lymphocyte_capture"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "antigen_presentation_decomposition",
            "kind": "dict",
            "expected": {
              "MHC_class_I": {
                "op": "in",
                "value": [
                  "co_regulated_with_dominant_program",
                  "tumor_intrinsic_enriched_independent"
                ]
              },
              "MHC_class_II": {
                "op": "in",
                "value": [
                  "partner_cell_attributable",
                  "flat_not_enriched",
                  "method_sensitive_no_robust_call",
                  "co_regulated_with_dominant_program"
                ]
              }
            }
          },
          {
            "name": "cf_non_robust_state_claimed_enriched",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "transitory_melanocytic",
              "intermediate_melanocytic",
              "melanocytic",
              "MITF_high",
              "neural_crest_like",
              "mitotic",
              "stress_p53",
              "undifferentiated",
              "dedifferentiated",
              "mesenchymal_tumor",
              "AXL_high_tumor",
              "invasive_tumor",
              "senescence_like",
              "EMT_transition",
              "cancer_stem_cell_like",
              "oxidative_stress_tumor"
            ]
          },
          {
            "name": "cf_framework_generic_or_offtissue_substitution",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "Tsoi_undifferentiated",
              "Tsoi_neural_crest_like",
              "Tsoi_transitory",
              "Tsoi_melanocytic",
              "Rambow_NCSC",
              "Wouters_mesenchymal",
              "Pozniak_TCF4_dependent",
              "tumor_cell",
              "melanoma_cell",
              "malignant_cell",
              "tumor_subpopulation",
              "keratinocyte",
              "hepatocyte",
              "lung_adenocarcinoma_cell",
              "colon_adenocarcinoma_cell",
              "Tirosh_cell_cycle_program",
              "Jerby_Arnon_program",
              "glioblastoma_cell",
              "breast_carcinoma_cell"
            ]
          },
          {
            "name": "cf_immune_response_direction_committed",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_immune_robustness_recall",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_ambiguous_hedge",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].causation_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_stress_generic_stress_response_evidence",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_conjugate_confounder_contamination_denied",
            "role": "hard_fail",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected"
            ]
          },
          {
            "name": "cf_ccl5_recall_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CCL5",
            "kind": "avoid",
            "expected": [
              "immune_response_tumor_state"
            ]
          },
          {
            "name": "cf_cxcl9_cxcl10_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CXCL9_CXCL10",
            "kind": "avoid",
            "expected": [
              "stress_hypoxia_tumor_state"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-6-claude-code",
      "modelName": "claude-opus-4-6",
      "harness": "claude-code",
      "task": "10x_5p_melanoma_cluster_tumor_state_causation_v5",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1836,
      "cost": 2.3689885000000004,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "enriched_tumor_states",
            "kind": "list",
            "matchKey": "tumor_state",
            "k": 3,
            "expected": [
              {
                "key": "immune_response",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "direction_unresolved_cross_sectional"
                    ],
                    "canonical": "direction_unresolved_cross_sectional"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "IFN_response_axis",
                      "broad_antigen_presentation_MHC"
                    ],
                    "canonical": "IFN_response_axis"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "enrichment_diminished_by_decontamination",
                      "threshold_sensitive"
                    ],
                    "canonical": "enrichment_diminished_by_decontamination"
                  }
                }
              },
              {
                "key": "stress_hypoxia",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "data_supports_cell_intrinsic_cause"
                    ],
                    "canonical": "data_supports_cell_intrinsic_cause"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "HIF_hypoxia_program_specific"
                    ],
                    "canonical": "HIF_hypoxia_program_specific"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "robust_across_methods",
                      "robust_after_decontamination"
                    ],
                    "canonical": "robust_across_methods"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CXCL9_CXCL10": {
                "op": "in",
                "value": [
                  "immune_response_tumor_state"
                ]
              },
              "CCL5": {
                "op": "in",
                "value": [
                  "shared_across_tumor_states",
                  "partner_lymphocyte_derived",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "gate_conjugate_gate_confounder",
            "role": "gate",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "partner_lymphocyte_capture"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "antigen_presentation_decomposition",
            "kind": "dict",
            "expected": {
              "MHC_class_I": {
                "op": "in",
                "value": [
                  "co_regulated_with_dominant_program",
                  "tumor_intrinsic_enriched_independent"
                ]
              },
              "MHC_class_II": {
                "op": "in",
                "value": [
                  "partner_cell_attributable",
                  "flat_not_enriched",
                  "method_sensitive_no_robust_call",
                  "co_regulated_with_dominant_program"
                ]
              }
            }
          },
          {
            "name": "cf_non_robust_state_claimed_enriched",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "transitory_melanocytic",
              "intermediate_melanocytic",
              "melanocytic",
              "MITF_high",
              "neural_crest_like",
              "mitotic",
              "stress_p53",
              "undifferentiated",
              "dedifferentiated",
              "mesenchymal_tumor",
              "AXL_high_tumor",
              "invasive_tumor",
              "senescence_like",
              "EMT_transition",
              "cancer_stem_cell_like",
              "oxidative_stress_tumor"
            ]
          },
          {
            "name": "cf_framework_generic_or_offtissue_substitution",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "Tsoi_undifferentiated",
              "Tsoi_neural_crest_like",
              "Tsoi_transitory",
              "Tsoi_melanocytic",
              "Rambow_NCSC",
              "Wouters_mesenchymal",
              "Pozniak_TCF4_dependent",
              "tumor_cell",
              "melanoma_cell",
              "malignant_cell",
              "tumor_subpopulation",
              "keratinocyte",
              "hepatocyte",
              "lung_adenocarcinoma_cell",
              "colon_adenocarcinoma_cell",
              "Tirosh_cell_cycle_program",
              "Jerby_Arnon_program",
              "glioblastoma_cell",
              "breast_carcinoma_cell"
            ]
          },
          {
            "name": "cf_immune_response_direction_committed",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_immune_robustness_recall",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_ambiguous_hedge",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].causation_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_stress_generic_stress_response_evidence",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_conjugate_confounder_contamination_denied",
            "role": "hard_fail",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected"
            ]
          },
          {
            "name": "cf_ccl5_recall_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CCL5",
            "kind": "avoid",
            "expected": [
              "immune_response_tumor_state"
            ]
          },
          {
            "name": "cf_cxcl9_cxcl10_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CXCL9_CXCL10",
            "kind": "avoid",
            "expected": [
              "stress_hypoxia_tumor_state"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-6-pi",
      "modelName": "claude-opus-4-6",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_tumor_state_causation_v5",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 1986,
      "cost": 2.049194,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "enriched_tumor_states",
            "kind": "list",
            "matchKey": "tumor_state",
            "k": 3,
            "expected": [
              {
                "key": "immune_response",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "direction_unresolved_cross_sectional"
                    ],
                    "canonical": "direction_unresolved_cross_sectional"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "IFN_response_axis",
                      "broad_antigen_presentation_MHC"
                    ],
                    "canonical": "IFN_response_axis"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "enrichment_diminished_by_decontamination",
                      "threshold_sensitive"
                    ],
                    "canonical": "enrichment_diminished_by_decontamination"
                  }
                }
              },
              {
                "key": "stress_hypoxia",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "data_supports_cell_intrinsic_cause"
                    ],
                    "canonical": "data_supports_cell_intrinsic_cause"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "HIF_hypoxia_program_specific"
                    ],
                    "canonical": "HIF_hypoxia_program_specific"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "robust_across_methods",
                      "robust_after_decontamination"
                    ],
                    "canonical": "robust_across_methods"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CXCL9_CXCL10": {
                "op": "in",
                "value": [
                  "immune_response_tumor_state"
                ]
              },
              "CCL5": {
                "op": "in",
                "value": [
                  "shared_across_tumor_states",
                  "partner_lymphocyte_derived",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "gate_conjugate_gate_confounder",
            "role": "gate",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "partner_lymphocyte_capture"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "antigen_presentation_decomposition",
            "kind": "dict",
            "expected": {
              "MHC_class_I": {
                "op": "in",
                "value": [
                  "co_regulated_with_dominant_program",
                  "tumor_intrinsic_enriched_independent"
                ]
              },
              "MHC_class_II": {
                "op": "in",
                "value": [
                  "partner_cell_attributable",
                  "flat_not_enriched",
                  "method_sensitive_no_robust_call",
                  "co_regulated_with_dominant_program"
                ]
              }
            }
          },
          {
            "name": "cf_non_robust_state_claimed_enriched",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "transitory_melanocytic",
              "intermediate_melanocytic",
              "melanocytic",
              "MITF_high",
              "neural_crest_like",
              "mitotic",
              "stress_p53",
              "undifferentiated",
              "dedifferentiated",
              "mesenchymal_tumor",
              "AXL_high_tumor",
              "invasive_tumor",
              "senescence_like",
              "EMT_transition",
              "cancer_stem_cell_like",
              "oxidative_stress_tumor"
            ]
          },
          {
            "name": "cf_framework_generic_or_offtissue_substitution",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "Tsoi_undifferentiated",
              "Tsoi_neural_crest_like",
              "Tsoi_transitory",
              "Tsoi_melanocytic",
              "Rambow_NCSC",
              "Wouters_mesenchymal",
              "Pozniak_TCF4_dependent",
              "tumor_cell",
              "melanoma_cell",
              "malignant_cell",
              "tumor_subpopulation",
              "keratinocyte",
              "hepatocyte",
              "lung_adenocarcinoma_cell",
              "colon_adenocarcinoma_cell",
              "Tirosh_cell_cycle_program",
              "Jerby_Arnon_program",
              "glioblastoma_cell",
              "breast_carcinoma_cell"
            ]
          },
          {
            "name": "cf_immune_response_direction_committed",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_immune_robustness_recall",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_ambiguous_hedge",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].causation_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_stress_generic_stress_response_evidence",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_conjugate_confounder_contamination_denied",
            "role": "hard_fail",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected"
            ]
          },
          {
            "name": "cf_ccl5_recall_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CCL5",
            "kind": "avoid",
            "expected": [
              "immune_response_tumor_state"
            ]
          },
          {
            "name": "cf_cxcl9_cxcl10_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CXCL9_CXCL10",
            "kind": "avoid",
            "expected": [
              "stress_hypoxia_tumor_state"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-6-pi",
      "modelName": "claude-opus-4-6",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_tumor_state_causation_v5",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 3558,
      "cost": 2.76632475,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "enriched_tumor_states",
            "kind": "list",
            "matchKey": "tumor_state",
            "k": 3,
            "expected": [
              {
                "key": "immune_response",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "direction_unresolved_cross_sectional"
                    ],
                    "canonical": "direction_unresolved_cross_sectional"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "IFN_response_axis",
                      "broad_antigen_presentation_MHC"
                    ],
                    "canonical": "IFN_response_axis"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "enrichment_diminished_by_decontamination",
                      "threshold_sensitive"
                    ],
                    "canonical": "enrichment_diminished_by_decontamination"
                  }
                }
              },
              {
                "key": "stress_hypoxia",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "data_supports_cell_intrinsic_cause"
                    ],
                    "canonical": "data_supports_cell_intrinsic_cause"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "HIF_hypoxia_program_specific"
                    ],
                    "canonical": "HIF_hypoxia_program_specific"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "robust_across_methods",
                      "robust_after_decontamination"
                    ],
                    "canonical": "robust_across_methods"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CXCL9_CXCL10": {
                "op": "in",
                "value": [
                  "immune_response_tumor_state"
                ]
              },
              "CCL5": {
                "op": "in",
                "value": [
                  "shared_across_tumor_states",
                  "partner_lymphocyte_derived",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "gate_conjugate_gate_confounder",
            "role": "gate",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "partner_lymphocyte_capture"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "antigen_presentation_decomposition",
            "kind": "dict",
            "expected": {
              "MHC_class_I": {
                "op": "in",
                "value": [
                  "co_regulated_with_dominant_program",
                  "tumor_intrinsic_enriched_independent"
                ]
              },
              "MHC_class_II": {
                "op": "in",
                "value": [
                  "partner_cell_attributable",
                  "flat_not_enriched",
                  "method_sensitive_no_robust_call",
                  "co_regulated_with_dominant_program"
                ]
              }
            }
          },
          {
            "name": "cf_non_robust_state_claimed_enriched",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "transitory_melanocytic",
              "intermediate_melanocytic",
              "melanocytic",
              "MITF_high",
              "neural_crest_like",
              "mitotic",
              "stress_p53",
              "undifferentiated",
              "dedifferentiated",
              "mesenchymal_tumor",
              "AXL_high_tumor",
              "invasive_tumor",
              "senescence_like",
              "EMT_transition",
              "cancer_stem_cell_like",
              "oxidative_stress_tumor"
            ]
          },
          {
            "name": "cf_framework_generic_or_offtissue_substitution",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "Tsoi_undifferentiated",
              "Tsoi_neural_crest_like",
              "Tsoi_transitory",
              "Tsoi_melanocytic",
              "Rambow_NCSC",
              "Wouters_mesenchymal",
              "Pozniak_TCF4_dependent",
              "tumor_cell",
              "melanoma_cell",
              "malignant_cell",
              "tumor_subpopulation",
              "keratinocyte",
              "hepatocyte",
              "lung_adenocarcinoma_cell",
              "colon_adenocarcinoma_cell",
              "Tirosh_cell_cycle_program",
              "Jerby_Arnon_program",
              "glioblastoma_cell",
              "breast_carcinoma_cell"
            ]
          },
          {
            "name": "cf_immune_response_direction_committed",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_immune_robustness_recall",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_ambiguous_hedge",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].causation_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_stress_generic_stress_response_evidence",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_conjugate_confounder_contamination_denied",
            "role": "hard_fail",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected"
            ]
          },
          {
            "name": "cf_ccl5_recall_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CCL5",
            "kind": "avoid",
            "expected": [
              "immune_response_tumor_state"
            ]
          },
          {
            "name": "cf_cxcl9_cxcl10_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CXCL9_CXCL10",
            "kind": "avoid",
            "expected": [
              "stress_hypoxia_tumor_state"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-6-pi",
      "modelName": "claude-opus-4-6",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_tumor_state_causation_v5",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 2227,
      "cost": 3.1417900000000003,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "enriched_tumor_states",
            "kind": "list",
            "matchKey": "tumor_state",
            "k": 3,
            "expected": [
              {
                "key": "immune_response",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "direction_unresolved_cross_sectional"
                    ],
                    "canonical": "direction_unresolved_cross_sectional"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "IFN_response_axis",
                      "broad_antigen_presentation_MHC"
                    ],
                    "canonical": "IFN_response_axis"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "enrichment_diminished_by_decontamination",
                      "threshold_sensitive"
                    ],
                    "canonical": "enrichment_diminished_by_decontamination"
                  }
                }
              },
              {
                "key": "stress_hypoxia",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "data_supports_cell_intrinsic_cause"
                    ],
                    "canonical": "data_supports_cell_intrinsic_cause"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "HIF_hypoxia_program_specific"
                    ],
                    "canonical": "HIF_hypoxia_program_specific"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "robust_across_methods",
                      "robust_after_decontamination"
                    ],
                    "canonical": "robust_across_methods"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CXCL9_CXCL10": {
                "op": "in",
                "value": [
                  "immune_response_tumor_state"
                ]
              },
              "CCL5": {
                "op": "in",
                "value": [
                  "shared_across_tumor_states",
                  "partner_lymphocyte_derived",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "gate_conjugate_gate_confounder",
            "role": "gate",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "partner_lymphocyte_capture"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "antigen_presentation_decomposition",
            "kind": "dict",
            "expected": {
              "MHC_class_I": {
                "op": "in",
                "value": [
                  "co_regulated_with_dominant_program",
                  "tumor_intrinsic_enriched_independent"
                ]
              },
              "MHC_class_II": {
                "op": "in",
                "value": [
                  "partner_cell_attributable",
                  "flat_not_enriched",
                  "method_sensitive_no_robust_call",
                  "co_regulated_with_dominant_program"
                ]
              }
            }
          },
          {
            "name": "cf_non_robust_state_claimed_enriched",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "transitory_melanocytic",
              "intermediate_melanocytic",
              "melanocytic",
              "MITF_high",
              "neural_crest_like",
              "mitotic",
              "stress_p53",
              "undifferentiated",
              "dedifferentiated",
              "mesenchymal_tumor",
              "AXL_high_tumor",
              "invasive_tumor",
              "senescence_like",
              "EMT_transition",
              "cancer_stem_cell_like",
              "oxidative_stress_tumor"
            ]
          },
          {
            "name": "cf_framework_generic_or_offtissue_substitution",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "Tsoi_undifferentiated",
              "Tsoi_neural_crest_like",
              "Tsoi_transitory",
              "Tsoi_melanocytic",
              "Rambow_NCSC",
              "Wouters_mesenchymal",
              "Pozniak_TCF4_dependent",
              "tumor_cell",
              "melanoma_cell",
              "malignant_cell",
              "tumor_subpopulation",
              "keratinocyte",
              "hepatocyte",
              "lung_adenocarcinoma_cell",
              "colon_adenocarcinoma_cell",
              "Tirosh_cell_cycle_program",
              "Jerby_Arnon_program",
              "glioblastoma_cell",
              "breast_carcinoma_cell"
            ]
          },
          {
            "name": "cf_immune_response_direction_committed",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_immune_robustness_recall",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_ambiguous_hedge",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].causation_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_stress_generic_stress_response_evidence",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_conjugate_confounder_contamination_denied",
            "role": "hard_fail",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected"
            ]
          },
          {
            "name": "cf_ccl5_recall_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CCL5",
            "kind": "avoid",
            "expected": [
              "immune_response_tumor_state"
            ]
          },
          {
            "name": "cf_cxcl9_cxcl10_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CXCL9_CXCL10",
            "kind": "avoid",
            "expected": [
              "stress_hypoxia_tumor_state"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-7-claude-code",
      "modelName": "claude-opus-4-7",
      "harness": "claude-code",
      "task": "10x_5p_melanoma_cluster_tumor_state_causation_v5",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 1483,
      "cost": 4.45837975,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "enriched_tumor_states",
            "kind": "list",
            "matchKey": "tumor_state",
            "k": 3,
            "expected": [
              {
                "key": "immune_response",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "direction_unresolved_cross_sectional"
                    ],
                    "canonical": "direction_unresolved_cross_sectional"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "IFN_response_axis",
                      "broad_antigen_presentation_MHC"
                    ],
                    "canonical": "IFN_response_axis"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "enrichment_diminished_by_decontamination",
                      "threshold_sensitive"
                    ],
                    "canonical": "enrichment_diminished_by_decontamination"
                  }
                }
              },
              {
                "key": "stress_hypoxia",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "data_supports_cell_intrinsic_cause"
                    ],
                    "canonical": "data_supports_cell_intrinsic_cause"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "HIF_hypoxia_program_specific"
                    ],
                    "canonical": "HIF_hypoxia_program_specific"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "robust_across_methods",
                      "robust_after_decontamination"
                    ],
                    "canonical": "robust_across_methods"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CXCL9_CXCL10": {
                "op": "in",
                "value": [
                  "immune_response_tumor_state"
                ]
              },
              "CCL5": {
                "op": "in",
                "value": [
                  "shared_across_tumor_states",
                  "partner_lymphocyte_derived",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "gate_conjugate_gate_confounder",
            "role": "gate",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "partner_lymphocyte_capture"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "antigen_presentation_decomposition",
            "kind": "dict",
            "expected": {
              "MHC_class_I": {
                "op": "in",
                "value": [
                  "co_regulated_with_dominant_program",
                  "tumor_intrinsic_enriched_independent"
                ]
              },
              "MHC_class_II": {
                "op": "in",
                "value": [
                  "partner_cell_attributable",
                  "flat_not_enriched",
                  "method_sensitive_no_robust_call",
                  "co_regulated_with_dominant_program"
                ]
              }
            }
          },
          {
            "name": "cf_non_robust_state_claimed_enriched",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "transitory_melanocytic",
              "intermediate_melanocytic",
              "melanocytic",
              "MITF_high",
              "neural_crest_like",
              "mitotic",
              "stress_p53",
              "undifferentiated",
              "dedifferentiated",
              "mesenchymal_tumor",
              "AXL_high_tumor",
              "invasive_tumor",
              "senescence_like",
              "EMT_transition",
              "cancer_stem_cell_like",
              "oxidative_stress_tumor"
            ]
          },
          {
            "name": "cf_framework_generic_or_offtissue_substitution",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "Tsoi_undifferentiated",
              "Tsoi_neural_crest_like",
              "Tsoi_transitory",
              "Tsoi_melanocytic",
              "Rambow_NCSC",
              "Wouters_mesenchymal",
              "Pozniak_TCF4_dependent",
              "tumor_cell",
              "melanoma_cell",
              "malignant_cell",
              "tumor_subpopulation",
              "keratinocyte",
              "hepatocyte",
              "lung_adenocarcinoma_cell",
              "colon_adenocarcinoma_cell",
              "Tirosh_cell_cycle_program",
              "Jerby_Arnon_program",
              "glioblastoma_cell",
              "breast_carcinoma_cell"
            ]
          },
          {
            "name": "cf_immune_response_direction_committed",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_immune_robustness_recall",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_ambiguous_hedge",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].causation_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_stress_generic_stress_response_evidence",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_conjugate_confounder_contamination_denied",
            "role": "hard_fail",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected"
            ]
          },
          {
            "name": "cf_ccl5_recall_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CCL5",
            "kind": "avoid",
            "expected": [
              "immune_response_tumor_state"
            ]
          },
          {
            "name": "cf_cxcl9_cxcl10_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CXCL9_CXCL10",
            "kind": "avoid",
            "expected": [
              "stress_hypoxia_tumor_state"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-7-claude-code",
      "modelName": "claude-opus-4-7",
      "harness": "claude-code",
      "task": "10x_5p_melanoma_cluster_tumor_state_causation_v5",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 1626,
      "cost": 4.139787499999999,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "enriched_tumor_states",
            "kind": "list",
            "matchKey": "tumor_state",
            "k": 3,
            "expected": [
              {
                "key": "immune_response",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "direction_unresolved_cross_sectional"
                    ],
                    "canonical": "direction_unresolved_cross_sectional"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "IFN_response_axis",
                      "broad_antigen_presentation_MHC"
                    ],
                    "canonical": "IFN_response_axis"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "enrichment_diminished_by_decontamination",
                      "threshold_sensitive"
                    ],
                    "canonical": "enrichment_diminished_by_decontamination"
                  }
                }
              },
              {
                "key": "stress_hypoxia",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "data_supports_cell_intrinsic_cause"
                    ],
                    "canonical": "data_supports_cell_intrinsic_cause"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "HIF_hypoxia_program_specific"
                    ],
                    "canonical": "HIF_hypoxia_program_specific"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "robust_across_methods",
                      "robust_after_decontamination"
                    ],
                    "canonical": "robust_across_methods"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CXCL9_CXCL10": {
                "op": "in",
                "value": [
                  "immune_response_tumor_state"
                ]
              },
              "CCL5": {
                "op": "in",
                "value": [
                  "shared_across_tumor_states",
                  "partner_lymphocyte_derived",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "gate_conjugate_gate_confounder",
            "role": "gate",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "partner_lymphocyte_capture"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "antigen_presentation_decomposition",
            "kind": "dict",
            "expected": {
              "MHC_class_I": {
                "op": "in",
                "value": [
                  "co_regulated_with_dominant_program",
                  "tumor_intrinsic_enriched_independent"
                ]
              },
              "MHC_class_II": {
                "op": "in",
                "value": [
                  "partner_cell_attributable",
                  "flat_not_enriched",
                  "method_sensitive_no_robust_call",
                  "co_regulated_with_dominant_program"
                ]
              }
            }
          },
          {
            "name": "cf_non_robust_state_claimed_enriched",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "transitory_melanocytic",
              "intermediate_melanocytic",
              "melanocytic",
              "MITF_high",
              "neural_crest_like",
              "mitotic",
              "stress_p53",
              "undifferentiated",
              "dedifferentiated",
              "mesenchymal_tumor",
              "AXL_high_tumor",
              "invasive_tumor",
              "senescence_like",
              "EMT_transition",
              "cancer_stem_cell_like",
              "oxidative_stress_tumor"
            ]
          },
          {
            "name": "cf_framework_generic_or_offtissue_substitution",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "Tsoi_undifferentiated",
              "Tsoi_neural_crest_like",
              "Tsoi_transitory",
              "Tsoi_melanocytic",
              "Rambow_NCSC",
              "Wouters_mesenchymal",
              "Pozniak_TCF4_dependent",
              "tumor_cell",
              "melanoma_cell",
              "malignant_cell",
              "tumor_subpopulation",
              "keratinocyte",
              "hepatocyte",
              "lung_adenocarcinoma_cell",
              "colon_adenocarcinoma_cell",
              "Tirosh_cell_cycle_program",
              "Jerby_Arnon_program",
              "glioblastoma_cell",
              "breast_carcinoma_cell"
            ]
          },
          {
            "name": "cf_immune_response_direction_committed",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_immune_robustness_recall",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_ambiguous_hedge",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].causation_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_stress_generic_stress_response_evidence",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_conjugate_confounder_contamination_denied",
            "role": "hard_fail",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected"
            ]
          },
          {
            "name": "cf_ccl5_recall_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CCL5",
            "kind": "avoid",
            "expected": [
              "immune_response_tumor_state"
            ]
          },
          {
            "name": "cf_cxcl9_cxcl10_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CXCL9_CXCL10",
            "kind": "avoid",
            "expected": [
              "stress_hypoxia_tumor_state"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-7-claude-code",
      "modelName": "claude-opus-4-7",
      "harness": "claude-code",
      "task": "10x_5p_melanoma_cluster_tumor_state_causation_v5",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 2140,
      "cost": 6.39238725,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "enriched_tumor_states",
            "kind": "list",
            "matchKey": "tumor_state",
            "k": 3,
            "expected": [
              {
                "key": "immune_response",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "direction_unresolved_cross_sectional"
                    ],
                    "canonical": "direction_unresolved_cross_sectional"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "IFN_response_axis",
                      "broad_antigen_presentation_MHC"
                    ],
                    "canonical": "IFN_response_axis"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "enrichment_diminished_by_decontamination",
                      "threshold_sensitive"
                    ],
                    "canonical": "enrichment_diminished_by_decontamination"
                  }
                }
              },
              {
                "key": "stress_hypoxia",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "data_supports_cell_intrinsic_cause"
                    ],
                    "canonical": "data_supports_cell_intrinsic_cause"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "HIF_hypoxia_program_specific"
                    ],
                    "canonical": "HIF_hypoxia_program_specific"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "robust_across_methods",
                      "robust_after_decontamination"
                    ],
                    "canonical": "robust_across_methods"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CXCL9_CXCL10": {
                "op": "in",
                "value": [
                  "immune_response_tumor_state"
                ]
              },
              "CCL5": {
                "op": "in",
                "value": [
                  "shared_across_tumor_states",
                  "partner_lymphocyte_derived",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "gate_conjugate_gate_confounder",
            "role": "gate",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "partner_lymphocyte_capture"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "antigen_presentation_decomposition",
            "kind": "dict",
            "expected": {
              "MHC_class_I": {
                "op": "in",
                "value": [
                  "co_regulated_with_dominant_program",
                  "tumor_intrinsic_enriched_independent"
                ]
              },
              "MHC_class_II": {
                "op": "in",
                "value": [
                  "partner_cell_attributable",
                  "flat_not_enriched",
                  "method_sensitive_no_robust_call",
                  "co_regulated_with_dominant_program"
                ]
              }
            }
          },
          {
            "name": "cf_non_robust_state_claimed_enriched",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "transitory_melanocytic",
              "intermediate_melanocytic",
              "melanocytic",
              "MITF_high",
              "neural_crest_like",
              "mitotic",
              "stress_p53",
              "undifferentiated",
              "dedifferentiated",
              "mesenchymal_tumor",
              "AXL_high_tumor",
              "invasive_tumor",
              "senescence_like",
              "EMT_transition",
              "cancer_stem_cell_like",
              "oxidative_stress_tumor"
            ]
          },
          {
            "name": "cf_framework_generic_or_offtissue_substitution",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "Tsoi_undifferentiated",
              "Tsoi_neural_crest_like",
              "Tsoi_transitory",
              "Tsoi_melanocytic",
              "Rambow_NCSC",
              "Wouters_mesenchymal",
              "Pozniak_TCF4_dependent",
              "tumor_cell",
              "melanoma_cell",
              "malignant_cell",
              "tumor_subpopulation",
              "keratinocyte",
              "hepatocyte",
              "lung_adenocarcinoma_cell",
              "colon_adenocarcinoma_cell",
              "Tirosh_cell_cycle_program",
              "Jerby_Arnon_program",
              "glioblastoma_cell",
              "breast_carcinoma_cell"
            ]
          },
          {
            "name": "cf_immune_response_direction_committed",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_immune_robustness_recall",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_ambiguous_hedge",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].causation_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_stress_generic_stress_response_evidence",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_conjugate_confounder_contamination_denied",
            "role": "hard_fail",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected"
            ]
          },
          {
            "name": "cf_ccl5_recall_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CCL5",
            "kind": "avoid",
            "expected": [
              "immune_response_tumor_state"
            ]
          },
          {
            "name": "cf_cxcl9_cxcl10_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CXCL9_CXCL10",
            "kind": "avoid",
            "expected": [
              "stress_hypoxia_tumor_state"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-7-pi",
      "modelName": "claude-opus-4-7",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_tumor_state_causation_v5",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 2375,
      "cost": 3.90136925,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "enriched_tumor_states",
            "kind": "list",
            "matchKey": "tumor_state",
            "k": 3,
            "expected": [
              {
                "key": "immune_response",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "direction_unresolved_cross_sectional"
                    ],
                    "canonical": "direction_unresolved_cross_sectional"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "IFN_response_axis",
                      "broad_antigen_presentation_MHC"
                    ],
                    "canonical": "IFN_response_axis"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "enrichment_diminished_by_decontamination",
                      "threshold_sensitive"
                    ],
                    "canonical": "enrichment_diminished_by_decontamination"
                  }
                }
              },
              {
                "key": "stress_hypoxia",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "data_supports_cell_intrinsic_cause"
                    ],
                    "canonical": "data_supports_cell_intrinsic_cause"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "HIF_hypoxia_program_specific"
                    ],
                    "canonical": "HIF_hypoxia_program_specific"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "robust_across_methods",
                      "robust_after_decontamination"
                    ],
                    "canonical": "robust_across_methods"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CXCL9_CXCL10": {
                "op": "in",
                "value": [
                  "immune_response_tumor_state"
                ]
              },
              "CCL5": {
                "op": "in",
                "value": [
                  "shared_across_tumor_states",
                  "partner_lymphocyte_derived",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "gate_conjugate_gate_confounder",
            "role": "gate",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "partner_lymphocyte_capture"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "antigen_presentation_decomposition",
            "kind": "dict",
            "expected": {
              "MHC_class_I": {
                "op": "in",
                "value": [
                  "co_regulated_with_dominant_program",
                  "tumor_intrinsic_enriched_independent"
                ]
              },
              "MHC_class_II": {
                "op": "in",
                "value": [
                  "partner_cell_attributable",
                  "flat_not_enriched",
                  "method_sensitive_no_robust_call",
                  "co_regulated_with_dominant_program"
                ]
              }
            }
          },
          {
            "name": "cf_non_robust_state_claimed_enriched",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "transitory_melanocytic",
              "intermediate_melanocytic",
              "melanocytic",
              "MITF_high",
              "neural_crest_like",
              "mitotic",
              "stress_p53",
              "undifferentiated",
              "dedifferentiated",
              "mesenchymal_tumor",
              "AXL_high_tumor",
              "invasive_tumor",
              "senescence_like",
              "EMT_transition",
              "cancer_stem_cell_like",
              "oxidative_stress_tumor"
            ]
          },
          {
            "name": "cf_framework_generic_or_offtissue_substitution",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "Tsoi_undifferentiated",
              "Tsoi_neural_crest_like",
              "Tsoi_transitory",
              "Tsoi_melanocytic",
              "Rambow_NCSC",
              "Wouters_mesenchymal",
              "Pozniak_TCF4_dependent",
              "tumor_cell",
              "melanoma_cell",
              "malignant_cell",
              "tumor_subpopulation",
              "keratinocyte",
              "hepatocyte",
              "lung_adenocarcinoma_cell",
              "colon_adenocarcinoma_cell",
              "Tirosh_cell_cycle_program",
              "Jerby_Arnon_program",
              "glioblastoma_cell",
              "breast_carcinoma_cell"
            ]
          },
          {
            "name": "cf_immune_response_direction_committed",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_immune_robustness_recall",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_ambiguous_hedge",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].causation_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_stress_generic_stress_response_evidence",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_conjugate_confounder_contamination_denied",
            "role": "hard_fail",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected"
            ]
          },
          {
            "name": "cf_ccl5_recall_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CCL5",
            "kind": "avoid",
            "expected": [
              "immune_response_tumor_state"
            ]
          },
          {
            "name": "cf_cxcl9_cxcl10_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CXCL9_CXCL10",
            "kind": "avoid",
            "expected": [
              "stress_hypoxia_tumor_state"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-7-pi",
      "modelName": "claude-opus-4-7",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_tumor_state_causation_v5",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 3517,
      "cost": 3.6113607500000002,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "enriched_tumor_states",
            "kind": "list",
            "matchKey": "tumor_state",
            "k": 3,
            "expected": [
              {
                "key": "immune_response",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "direction_unresolved_cross_sectional"
                    ],
                    "canonical": "direction_unresolved_cross_sectional"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "IFN_response_axis",
                      "broad_antigen_presentation_MHC"
                    ],
                    "canonical": "IFN_response_axis"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "enrichment_diminished_by_decontamination",
                      "threshold_sensitive"
                    ],
                    "canonical": "enrichment_diminished_by_decontamination"
                  }
                }
              },
              {
                "key": "stress_hypoxia",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "data_supports_cell_intrinsic_cause"
                    ],
                    "canonical": "data_supports_cell_intrinsic_cause"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "HIF_hypoxia_program_specific"
                    ],
                    "canonical": "HIF_hypoxia_program_specific"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "robust_across_methods",
                      "robust_after_decontamination"
                    ],
                    "canonical": "robust_across_methods"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CXCL9_CXCL10": {
                "op": "in",
                "value": [
                  "immune_response_tumor_state"
                ]
              },
              "CCL5": {
                "op": "in",
                "value": [
                  "shared_across_tumor_states",
                  "partner_lymphocyte_derived",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "gate_conjugate_gate_confounder",
            "role": "gate",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "partner_lymphocyte_capture"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "antigen_presentation_decomposition",
            "kind": "dict",
            "expected": {
              "MHC_class_I": {
                "op": "in",
                "value": [
                  "co_regulated_with_dominant_program",
                  "tumor_intrinsic_enriched_independent"
                ]
              },
              "MHC_class_II": {
                "op": "in",
                "value": [
                  "partner_cell_attributable",
                  "flat_not_enriched",
                  "method_sensitive_no_robust_call",
                  "co_regulated_with_dominant_program"
                ]
              }
            }
          },
          {
            "name": "cf_non_robust_state_claimed_enriched",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "transitory_melanocytic",
              "intermediate_melanocytic",
              "melanocytic",
              "MITF_high",
              "neural_crest_like",
              "mitotic",
              "stress_p53",
              "undifferentiated",
              "dedifferentiated",
              "mesenchymal_tumor",
              "AXL_high_tumor",
              "invasive_tumor",
              "senescence_like",
              "EMT_transition",
              "cancer_stem_cell_like",
              "oxidative_stress_tumor"
            ]
          },
          {
            "name": "cf_framework_generic_or_offtissue_substitution",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "Tsoi_undifferentiated",
              "Tsoi_neural_crest_like",
              "Tsoi_transitory",
              "Tsoi_melanocytic",
              "Rambow_NCSC",
              "Wouters_mesenchymal",
              "Pozniak_TCF4_dependent",
              "tumor_cell",
              "melanoma_cell",
              "malignant_cell",
              "tumor_subpopulation",
              "keratinocyte",
              "hepatocyte",
              "lung_adenocarcinoma_cell",
              "colon_adenocarcinoma_cell",
              "Tirosh_cell_cycle_program",
              "Jerby_Arnon_program",
              "glioblastoma_cell",
              "breast_carcinoma_cell"
            ]
          },
          {
            "name": "cf_immune_response_direction_committed",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_immune_robustness_recall",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_ambiguous_hedge",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].causation_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_stress_generic_stress_response_evidence",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_conjugate_confounder_contamination_denied",
            "role": "hard_fail",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected"
            ]
          },
          {
            "name": "cf_ccl5_recall_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CCL5",
            "kind": "avoid",
            "expected": [
              "immune_response_tumor_state"
            ]
          },
          {
            "name": "cf_cxcl9_cxcl10_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CXCL9_CXCL10",
            "kind": "avoid",
            "expected": [
              "stress_hypoxia_tumor_state"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-7-pi",
      "modelName": "claude-opus-4-7",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_tumor_state_causation_v5",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1909,
      "cost": 3.310835,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "enriched_tumor_states",
            "kind": "list",
            "matchKey": "tumor_state",
            "k": 3,
            "expected": [
              {
                "key": "immune_response",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "direction_unresolved_cross_sectional"
                    ],
                    "canonical": "direction_unresolved_cross_sectional"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "IFN_response_axis",
                      "broad_antigen_presentation_MHC"
                    ],
                    "canonical": "IFN_response_axis"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "enrichment_diminished_by_decontamination",
                      "threshold_sensitive"
                    ],
                    "canonical": "enrichment_diminished_by_decontamination"
                  }
                }
              },
              {
                "key": "stress_hypoxia",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "data_supports_cell_intrinsic_cause"
                    ],
                    "canonical": "data_supports_cell_intrinsic_cause"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "HIF_hypoxia_program_specific"
                    ],
                    "canonical": "HIF_hypoxia_program_specific"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "robust_across_methods",
                      "robust_after_decontamination"
                    ],
                    "canonical": "robust_across_methods"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CXCL9_CXCL10": {
                "op": "in",
                "value": [
                  "immune_response_tumor_state"
                ]
              },
              "CCL5": {
                "op": "in",
                "value": [
                  "shared_across_tumor_states",
                  "partner_lymphocyte_derived",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "gate_conjugate_gate_confounder",
            "role": "gate",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "partner_lymphocyte_capture"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "antigen_presentation_decomposition",
            "kind": "dict",
            "expected": {
              "MHC_class_I": {
                "op": "in",
                "value": [
                  "co_regulated_with_dominant_program",
                  "tumor_intrinsic_enriched_independent"
                ]
              },
              "MHC_class_II": {
                "op": "in",
                "value": [
                  "partner_cell_attributable",
                  "flat_not_enriched",
                  "method_sensitive_no_robust_call",
                  "co_regulated_with_dominant_program"
                ]
              }
            }
          },
          {
            "name": "cf_non_robust_state_claimed_enriched",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "transitory_melanocytic",
              "intermediate_melanocytic",
              "melanocytic",
              "MITF_high",
              "neural_crest_like",
              "mitotic",
              "stress_p53",
              "undifferentiated",
              "dedifferentiated",
              "mesenchymal_tumor",
              "AXL_high_tumor",
              "invasive_tumor",
              "senescence_like",
              "EMT_transition",
              "cancer_stem_cell_like",
              "oxidative_stress_tumor"
            ]
          },
          {
            "name": "cf_framework_generic_or_offtissue_substitution",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "Tsoi_undifferentiated",
              "Tsoi_neural_crest_like",
              "Tsoi_transitory",
              "Tsoi_melanocytic",
              "Rambow_NCSC",
              "Wouters_mesenchymal",
              "Pozniak_TCF4_dependent",
              "tumor_cell",
              "melanoma_cell",
              "malignant_cell",
              "tumor_subpopulation",
              "keratinocyte",
              "hepatocyte",
              "lung_adenocarcinoma_cell",
              "colon_adenocarcinoma_cell",
              "Tirosh_cell_cycle_program",
              "Jerby_Arnon_program",
              "glioblastoma_cell",
              "breast_carcinoma_cell"
            ]
          },
          {
            "name": "cf_immune_response_direction_committed",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_immune_robustness_recall",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_ambiguous_hedge",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].causation_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_stress_generic_stress_response_evidence",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_conjugate_confounder_contamination_denied",
            "role": "hard_fail",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected"
            ]
          },
          {
            "name": "cf_ccl5_recall_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CCL5",
            "kind": "avoid",
            "expected": [
              "immune_response_tumor_state"
            ]
          },
          {
            "name": "cf_cxcl9_cxcl10_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CXCL9_CXCL10",
            "kind": "avoid",
            "expected": [
              "stress_hypoxia_tumor_state"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-8-claude-code",
      "modelName": "claude-opus-4-8",
      "harness": "claude-code",
      "task": "10x_5p_melanoma_cluster_tumor_state_causation_v5",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 1555,
      "cost": 4.1471055,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "enriched_tumor_states",
            "kind": "list",
            "matchKey": "tumor_state",
            "k": 3,
            "expected": [
              {
                "key": "immune_response",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "direction_unresolved_cross_sectional"
                    ],
                    "canonical": "direction_unresolved_cross_sectional"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "IFN_response_axis",
                      "broad_antigen_presentation_MHC"
                    ],
                    "canonical": "IFN_response_axis"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "enrichment_diminished_by_decontamination",
                      "threshold_sensitive"
                    ],
                    "canonical": "enrichment_diminished_by_decontamination"
                  }
                }
              },
              {
                "key": "stress_hypoxia",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "data_supports_cell_intrinsic_cause"
                    ],
                    "canonical": "data_supports_cell_intrinsic_cause"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "HIF_hypoxia_program_specific"
                    ],
                    "canonical": "HIF_hypoxia_program_specific"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "robust_across_methods",
                      "robust_after_decontamination"
                    ],
                    "canonical": "robust_across_methods"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CXCL9_CXCL10": {
                "op": "in",
                "value": [
                  "immune_response_tumor_state"
                ]
              },
              "CCL5": {
                "op": "in",
                "value": [
                  "shared_across_tumor_states",
                  "partner_lymphocyte_derived",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "gate_conjugate_gate_confounder",
            "role": "gate",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "partner_lymphocyte_capture"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "antigen_presentation_decomposition",
            "kind": "dict",
            "expected": {
              "MHC_class_I": {
                "op": "in",
                "value": [
                  "co_regulated_with_dominant_program",
                  "tumor_intrinsic_enriched_independent"
                ]
              },
              "MHC_class_II": {
                "op": "in",
                "value": [
                  "partner_cell_attributable",
                  "flat_not_enriched",
                  "method_sensitive_no_robust_call",
                  "co_regulated_with_dominant_program"
                ]
              }
            }
          },
          {
            "name": "cf_non_robust_state_claimed_enriched",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "transitory_melanocytic",
              "intermediate_melanocytic",
              "melanocytic",
              "MITF_high",
              "neural_crest_like",
              "mitotic",
              "stress_p53",
              "undifferentiated",
              "dedifferentiated",
              "mesenchymal_tumor",
              "AXL_high_tumor",
              "invasive_tumor",
              "senescence_like",
              "EMT_transition",
              "cancer_stem_cell_like",
              "oxidative_stress_tumor"
            ]
          },
          {
            "name": "cf_framework_generic_or_offtissue_substitution",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "Tsoi_undifferentiated",
              "Tsoi_neural_crest_like",
              "Tsoi_transitory",
              "Tsoi_melanocytic",
              "Rambow_NCSC",
              "Wouters_mesenchymal",
              "Pozniak_TCF4_dependent",
              "tumor_cell",
              "melanoma_cell",
              "malignant_cell",
              "tumor_subpopulation",
              "keratinocyte",
              "hepatocyte",
              "lung_adenocarcinoma_cell",
              "colon_adenocarcinoma_cell",
              "Tirosh_cell_cycle_program",
              "Jerby_Arnon_program",
              "glioblastoma_cell",
              "breast_carcinoma_cell"
            ]
          },
          {
            "name": "cf_immune_response_direction_committed",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_immune_robustness_recall",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_ambiguous_hedge",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].causation_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_stress_generic_stress_response_evidence",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_conjugate_confounder_contamination_denied",
            "role": "hard_fail",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected"
            ]
          },
          {
            "name": "cf_ccl5_recall_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CCL5",
            "kind": "avoid",
            "expected": [
              "immune_response_tumor_state"
            ]
          },
          {
            "name": "cf_cxcl9_cxcl10_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CXCL9_CXCL10",
            "kind": "avoid",
            "expected": [
              "stress_hypoxia_tumor_state"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-8-claude-code",
      "modelName": "claude-opus-4-8",
      "harness": "claude-code",
      "task": "10x_5p_melanoma_cluster_tumor_state_causation_v5",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 2101,
      "cost": 6.0296295,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "enriched_tumor_states",
            "kind": "list",
            "matchKey": "tumor_state",
            "k": 3,
            "expected": [
              {
                "key": "immune_response",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "direction_unresolved_cross_sectional"
                    ],
                    "canonical": "direction_unresolved_cross_sectional"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "IFN_response_axis",
                      "broad_antigen_presentation_MHC"
                    ],
                    "canonical": "IFN_response_axis"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "enrichment_diminished_by_decontamination",
                      "threshold_sensitive"
                    ],
                    "canonical": "enrichment_diminished_by_decontamination"
                  }
                }
              },
              {
                "key": "stress_hypoxia",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "data_supports_cell_intrinsic_cause"
                    ],
                    "canonical": "data_supports_cell_intrinsic_cause"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "HIF_hypoxia_program_specific"
                    ],
                    "canonical": "HIF_hypoxia_program_specific"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "robust_across_methods",
                      "robust_after_decontamination"
                    ],
                    "canonical": "robust_across_methods"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CXCL9_CXCL10": {
                "op": "in",
                "value": [
                  "immune_response_tumor_state"
                ]
              },
              "CCL5": {
                "op": "in",
                "value": [
                  "shared_across_tumor_states",
                  "partner_lymphocyte_derived",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "gate_conjugate_gate_confounder",
            "role": "gate",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "partner_lymphocyte_capture"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "antigen_presentation_decomposition",
            "kind": "dict",
            "expected": {
              "MHC_class_I": {
                "op": "in",
                "value": [
                  "co_regulated_with_dominant_program",
                  "tumor_intrinsic_enriched_independent"
                ]
              },
              "MHC_class_II": {
                "op": "in",
                "value": [
                  "partner_cell_attributable",
                  "flat_not_enriched",
                  "method_sensitive_no_robust_call",
                  "co_regulated_with_dominant_program"
                ]
              }
            }
          },
          {
            "name": "cf_non_robust_state_claimed_enriched",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "transitory_melanocytic",
              "intermediate_melanocytic",
              "melanocytic",
              "MITF_high",
              "neural_crest_like",
              "mitotic",
              "stress_p53",
              "undifferentiated",
              "dedifferentiated",
              "mesenchymal_tumor",
              "AXL_high_tumor",
              "invasive_tumor",
              "senescence_like",
              "EMT_transition",
              "cancer_stem_cell_like",
              "oxidative_stress_tumor"
            ]
          },
          {
            "name": "cf_framework_generic_or_offtissue_substitution",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "Tsoi_undifferentiated",
              "Tsoi_neural_crest_like",
              "Tsoi_transitory",
              "Tsoi_melanocytic",
              "Rambow_NCSC",
              "Wouters_mesenchymal",
              "Pozniak_TCF4_dependent",
              "tumor_cell",
              "melanoma_cell",
              "malignant_cell",
              "tumor_subpopulation",
              "keratinocyte",
              "hepatocyte",
              "lung_adenocarcinoma_cell",
              "colon_adenocarcinoma_cell",
              "Tirosh_cell_cycle_program",
              "Jerby_Arnon_program",
              "glioblastoma_cell",
              "breast_carcinoma_cell"
            ]
          },
          {
            "name": "cf_immune_response_direction_committed",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_immune_robustness_recall",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_ambiguous_hedge",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].causation_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_stress_generic_stress_response_evidence",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_conjugate_confounder_contamination_denied",
            "role": "hard_fail",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected"
            ]
          },
          {
            "name": "cf_ccl5_recall_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CCL5",
            "kind": "avoid",
            "expected": [
              "immune_response_tumor_state"
            ]
          },
          {
            "name": "cf_cxcl9_cxcl10_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CXCL9_CXCL10",
            "kind": "avoid",
            "expected": [
              "stress_hypoxia_tumor_state"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-8-claude-code",
      "modelName": "claude-opus-4-8",
      "harness": "claude-code",
      "task": "10x_5p_melanoma_cluster_tumor_state_causation_v5",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 2116,
      "cost": 6.070954249999999,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "enriched_tumor_states",
            "kind": "list",
            "matchKey": "tumor_state",
            "k": 3,
            "expected": [
              {
                "key": "immune_response",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "direction_unresolved_cross_sectional"
                    ],
                    "canonical": "direction_unresolved_cross_sectional"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "IFN_response_axis",
                      "broad_antigen_presentation_MHC"
                    ],
                    "canonical": "IFN_response_axis"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "enrichment_diminished_by_decontamination",
                      "threshold_sensitive"
                    ],
                    "canonical": "enrichment_diminished_by_decontamination"
                  }
                }
              },
              {
                "key": "stress_hypoxia",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "data_supports_cell_intrinsic_cause"
                    ],
                    "canonical": "data_supports_cell_intrinsic_cause"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "HIF_hypoxia_program_specific"
                    ],
                    "canonical": "HIF_hypoxia_program_specific"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "robust_across_methods",
                      "robust_after_decontamination"
                    ],
                    "canonical": "robust_across_methods"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CXCL9_CXCL10": {
                "op": "in",
                "value": [
                  "immune_response_tumor_state"
                ]
              },
              "CCL5": {
                "op": "in",
                "value": [
                  "shared_across_tumor_states",
                  "partner_lymphocyte_derived",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "gate_conjugate_gate_confounder",
            "role": "gate",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "partner_lymphocyte_capture"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "antigen_presentation_decomposition",
            "kind": "dict",
            "expected": {
              "MHC_class_I": {
                "op": "in",
                "value": [
                  "co_regulated_with_dominant_program",
                  "tumor_intrinsic_enriched_independent"
                ]
              },
              "MHC_class_II": {
                "op": "in",
                "value": [
                  "partner_cell_attributable",
                  "flat_not_enriched",
                  "method_sensitive_no_robust_call",
                  "co_regulated_with_dominant_program"
                ]
              }
            }
          },
          {
            "name": "cf_non_robust_state_claimed_enriched",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "transitory_melanocytic",
              "intermediate_melanocytic",
              "melanocytic",
              "MITF_high",
              "neural_crest_like",
              "mitotic",
              "stress_p53",
              "undifferentiated",
              "dedifferentiated",
              "mesenchymal_tumor",
              "AXL_high_tumor",
              "invasive_tumor",
              "senescence_like",
              "EMT_transition",
              "cancer_stem_cell_like",
              "oxidative_stress_tumor"
            ]
          },
          {
            "name": "cf_framework_generic_or_offtissue_substitution",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "Tsoi_undifferentiated",
              "Tsoi_neural_crest_like",
              "Tsoi_transitory",
              "Tsoi_melanocytic",
              "Rambow_NCSC",
              "Wouters_mesenchymal",
              "Pozniak_TCF4_dependent",
              "tumor_cell",
              "melanoma_cell",
              "malignant_cell",
              "tumor_subpopulation",
              "keratinocyte",
              "hepatocyte",
              "lung_adenocarcinoma_cell",
              "colon_adenocarcinoma_cell",
              "Tirosh_cell_cycle_program",
              "Jerby_Arnon_program",
              "glioblastoma_cell",
              "breast_carcinoma_cell"
            ]
          },
          {
            "name": "cf_immune_response_direction_committed",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_immune_robustness_recall",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_ambiguous_hedge",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].causation_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_stress_generic_stress_response_evidence",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_conjugate_confounder_contamination_denied",
            "role": "hard_fail",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected"
            ]
          },
          {
            "name": "cf_ccl5_recall_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CCL5",
            "kind": "avoid",
            "expected": [
              "immune_response_tumor_state"
            ]
          },
          {
            "name": "cf_cxcl9_cxcl10_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CXCL9_CXCL10",
            "kind": "avoid",
            "expected": [
              "stress_hypoxia_tumor_state"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-8-pi",
      "modelName": "claude-opus-4-8",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_tumor_state_causation_v5",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 1959,
      "cost": 3.5734482499999998,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "enriched_tumor_states",
            "kind": "list",
            "matchKey": "tumor_state",
            "k": 3,
            "expected": [
              {
                "key": "immune_response",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "direction_unresolved_cross_sectional"
                    ],
                    "canonical": "direction_unresolved_cross_sectional"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "IFN_response_axis",
                      "broad_antigen_presentation_MHC"
                    ],
                    "canonical": "IFN_response_axis"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "enrichment_diminished_by_decontamination",
                      "threshold_sensitive"
                    ],
                    "canonical": "enrichment_diminished_by_decontamination"
                  }
                }
              },
              {
                "key": "stress_hypoxia",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "data_supports_cell_intrinsic_cause"
                    ],
                    "canonical": "data_supports_cell_intrinsic_cause"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "HIF_hypoxia_program_specific"
                    ],
                    "canonical": "HIF_hypoxia_program_specific"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "robust_across_methods",
                      "robust_after_decontamination"
                    ],
                    "canonical": "robust_across_methods"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CXCL9_CXCL10": {
                "op": "in",
                "value": [
                  "immune_response_tumor_state"
                ]
              },
              "CCL5": {
                "op": "in",
                "value": [
                  "shared_across_tumor_states",
                  "partner_lymphocyte_derived",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "gate_conjugate_gate_confounder",
            "role": "gate",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "partner_lymphocyte_capture"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "antigen_presentation_decomposition",
            "kind": "dict",
            "expected": {
              "MHC_class_I": {
                "op": "in",
                "value": [
                  "co_regulated_with_dominant_program",
                  "tumor_intrinsic_enriched_independent"
                ]
              },
              "MHC_class_II": {
                "op": "in",
                "value": [
                  "partner_cell_attributable",
                  "flat_not_enriched",
                  "method_sensitive_no_robust_call",
                  "co_regulated_with_dominant_program"
                ]
              }
            }
          },
          {
            "name": "cf_non_robust_state_claimed_enriched",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "transitory_melanocytic",
              "intermediate_melanocytic",
              "melanocytic",
              "MITF_high",
              "neural_crest_like",
              "mitotic",
              "stress_p53",
              "undifferentiated",
              "dedifferentiated",
              "mesenchymal_tumor",
              "AXL_high_tumor",
              "invasive_tumor",
              "senescence_like",
              "EMT_transition",
              "cancer_stem_cell_like",
              "oxidative_stress_tumor"
            ]
          },
          {
            "name": "cf_framework_generic_or_offtissue_substitution",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "Tsoi_undifferentiated",
              "Tsoi_neural_crest_like",
              "Tsoi_transitory",
              "Tsoi_melanocytic",
              "Rambow_NCSC",
              "Wouters_mesenchymal",
              "Pozniak_TCF4_dependent",
              "tumor_cell",
              "melanoma_cell",
              "malignant_cell",
              "tumor_subpopulation",
              "keratinocyte",
              "hepatocyte",
              "lung_adenocarcinoma_cell",
              "colon_adenocarcinoma_cell",
              "Tirosh_cell_cycle_program",
              "Jerby_Arnon_program",
              "glioblastoma_cell",
              "breast_carcinoma_cell"
            ]
          },
          {
            "name": "cf_immune_response_direction_committed",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_immune_robustness_recall",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_ambiguous_hedge",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].causation_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_stress_generic_stress_response_evidence",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_conjugate_confounder_contamination_denied",
            "role": "hard_fail",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected"
            ]
          },
          {
            "name": "cf_ccl5_recall_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CCL5",
            "kind": "avoid",
            "expected": [
              "immune_response_tumor_state"
            ]
          },
          {
            "name": "cf_cxcl9_cxcl10_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CXCL9_CXCL10",
            "kind": "avoid",
            "expected": [
              "stress_hypoxia_tumor_state"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-8-pi",
      "modelName": "claude-opus-4-8",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_tumor_state_causation_v5",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 2185,
      "cost": 4.937422249999999,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "enriched_tumor_states",
            "kind": "list",
            "matchKey": "tumor_state",
            "k": 3,
            "expected": [
              {
                "key": "immune_response",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "direction_unresolved_cross_sectional"
                    ],
                    "canonical": "direction_unresolved_cross_sectional"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "IFN_response_axis",
                      "broad_antigen_presentation_MHC"
                    ],
                    "canonical": "IFN_response_axis"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "enrichment_diminished_by_decontamination",
                      "threshold_sensitive"
                    ],
                    "canonical": "enrichment_diminished_by_decontamination"
                  }
                }
              },
              {
                "key": "stress_hypoxia",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "data_supports_cell_intrinsic_cause"
                    ],
                    "canonical": "data_supports_cell_intrinsic_cause"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "HIF_hypoxia_program_specific"
                    ],
                    "canonical": "HIF_hypoxia_program_specific"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "robust_across_methods",
                      "robust_after_decontamination"
                    ],
                    "canonical": "robust_across_methods"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CXCL9_CXCL10": {
                "op": "in",
                "value": [
                  "immune_response_tumor_state"
                ]
              },
              "CCL5": {
                "op": "in",
                "value": [
                  "shared_across_tumor_states",
                  "partner_lymphocyte_derived",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "gate_conjugate_gate_confounder",
            "role": "gate",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "partner_lymphocyte_capture"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "antigen_presentation_decomposition",
            "kind": "dict",
            "expected": {
              "MHC_class_I": {
                "op": "in",
                "value": [
                  "co_regulated_with_dominant_program",
                  "tumor_intrinsic_enriched_independent"
                ]
              },
              "MHC_class_II": {
                "op": "in",
                "value": [
                  "partner_cell_attributable",
                  "flat_not_enriched",
                  "method_sensitive_no_robust_call",
                  "co_regulated_with_dominant_program"
                ]
              }
            }
          },
          {
            "name": "cf_non_robust_state_claimed_enriched",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "transitory_melanocytic",
              "intermediate_melanocytic",
              "melanocytic",
              "MITF_high",
              "neural_crest_like",
              "mitotic",
              "stress_p53",
              "undifferentiated",
              "dedifferentiated",
              "mesenchymal_tumor",
              "AXL_high_tumor",
              "invasive_tumor",
              "senescence_like",
              "EMT_transition",
              "cancer_stem_cell_like",
              "oxidative_stress_tumor"
            ]
          },
          {
            "name": "cf_framework_generic_or_offtissue_substitution",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "Tsoi_undifferentiated",
              "Tsoi_neural_crest_like",
              "Tsoi_transitory",
              "Tsoi_melanocytic",
              "Rambow_NCSC",
              "Wouters_mesenchymal",
              "Pozniak_TCF4_dependent",
              "tumor_cell",
              "melanoma_cell",
              "malignant_cell",
              "tumor_subpopulation",
              "keratinocyte",
              "hepatocyte",
              "lung_adenocarcinoma_cell",
              "colon_adenocarcinoma_cell",
              "Tirosh_cell_cycle_program",
              "Jerby_Arnon_program",
              "glioblastoma_cell",
              "breast_carcinoma_cell"
            ]
          },
          {
            "name": "cf_immune_response_direction_committed",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_immune_robustness_recall",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_ambiguous_hedge",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].causation_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_stress_generic_stress_response_evidence",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_conjugate_confounder_contamination_denied",
            "role": "hard_fail",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected"
            ]
          },
          {
            "name": "cf_ccl5_recall_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CCL5",
            "kind": "avoid",
            "expected": [
              "immune_response_tumor_state"
            ]
          },
          {
            "name": "cf_cxcl9_cxcl10_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CXCL9_CXCL10",
            "kind": "avoid",
            "expected": [
              "stress_hypoxia_tumor_state"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-8-pi",
      "modelName": "claude-opus-4-8",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_tumor_state_causation_v5",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 2252,
      "cost": 3.73460325,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "enriched_tumor_states",
            "kind": "list",
            "matchKey": "tumor_state",
            "k": 3,
            "expected": [
              {
                "key": "immune_response",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "direction_unresolved_cross_sectional"
                    ],
                    "canonical": "direction_unresolved_cross_sectional"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "IFN_response_axis",
                      "broad_antigen_presentation_MHC"
                    ],
                    "canonical": "IFN_response_axis"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "enrichment_diminished_by_decontamination",
                      "threshold_sensitive"
                    ],
                    "canonical": "enrichment_diminished_by_decontamination"
                  }
                }
              },
              {
                "key": "stress_hypoxia",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "data_supports_cell_intrinsic_cause"
                    ],
                    "canonical": "data_supports_cell_intrinsic_cause"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "HIF_hypoxia_program_specific"
                    ],
                    "canonical": "HIF_hypoxia_program_specific"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "robust_across_methods",
                      "robust_after_decontamination"
                    ],
                    "canonical": "robust_across_methods"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CXCL9_CXCL10": {
                "op": "in",
                "value": [
                  "immune_response_tumor_state"
                ]
              },
              "CCL5": {
                "op": "in",
                "value": [
                  "shared_across_tumor_states",
                  "partner_lymphocyte_derived",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "gate_conjugate_gate_confounder",
            "role": "gate",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "partner_lymphocyte_capture"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "antigen_presentation_decomposition",
            "kind": "dict",
            "expected": {
              "MHC_class_I": {
                "op": "in",
                "value": [
                  "co_regulated_with_dominant_program",
                  "tumor_intrinsic_enriched_independent"
                ]
              },
              "MHC_class_II": {
                "op": "in",
                "value": [
                  "partner_cell_attributable",
                  "flat_not_enriched",
                  "method_sensitive_no_robust_call",
                  "co_regulated_with_dominant_program"
                ]
              }
            }
          },
          {
            "name": "cf_non_robust_state_claimed_enriched",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "transitory_melanocytic",
              "intermediate_melanocytic",
              "melanocytic",
              "MITF_high",
              "neural_crest_like",
              "mitotic",
              "stress_p53",
              "undifferentiated",
              "dedifferentiated",
              "mesenchymal_tumor",
              "AXL_high_tumor",
              "invasive_tumor",
              "senescence_like",
              "EMT_transition",
              "cancer_stem_cell_like",
              "oxidative_stress_tumor"
            ]
          },
          {
            "name": "cf_framework_generic_or_offtissue_substitution",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "Tsoi_undifferentiated",
              "Tsoi_neural_crest_like",
              "Tsoi_transitory",
              "Tsoi_melanocytic",
              "Rambow_NCSC",
              "Wouters_mesenchymal",
              "Pozniak_TCF4_dependent",
              "tumor_cell",
              "melanoma_cell",
              "malignant_cell",
              "tumor_subpopulation",
              "keratinocyte",
              "hepatocyte",
              "lung_adenocarcinoma_cell",
              "colon_adenocarcinoma_cell",
              "Tirosh_cell_cycle_program",
              "Jerby_Arnon_program",
              "glioblastoma_cell",
              "breast_carcinoma_cell"
            ]
          },
          {
            "name": "cf_immune_response_direction_committed",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_immune_robustness_recall",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_ambiguous_hedge",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].causation_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_stress_generic_stress_response_evidence",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_conjugate_confounder_contamination_denied",
            "role": "hard_fail",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected"
            ]
          },
          {
            "name": "cf_ccl5_recall_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CCL5",
            "kind": "avoid",
            "expected": [
              "immune_response_tumor_state"
            ]
          },
          {
            "name": "cf_cxcl9_cxcl10_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CXCL9_CXCL10",
            "kind": "avoid",
            "expected": [
              "stress_hypoxia_tumor_state"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-4-6-pi",
      "modelName": "claude-sonnet-4-6",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_tumor_state_causation_v5",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 2853,
      "cost": 2.6462545499999997,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "enriched_tumor_states",
            "kind": "list",
            "matchKey": "tumor_state",
            "k": 3,
            "expected": [
              {
                "key": "immune_response",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "direction_unresolved_cross_sectional"
                    ],
                    "canonical": "direction_unresolved_cross_sectional"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "IFN_response_axis",
                      "broad_antigen_presentation_MHC"
                    ],
                    "canonical": "IFN_response_axis"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "enrichment_diminished_by_decontamination",
                      "threshold_sensitive"
                    ],
                    "canonical": "enrichment_diminished_by_decontamination"
                  }
                }
              },
              {
                "key": "stress_hypoxia",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "data_supports_cell_intrinsic_cause"
                    ],
                    "canonical": "data_supports_cell_intrinsic_cause"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "HIF_hypoxia_program_specific"
                    ],
                    "canonical": "HIF_hypoxia_program_specific"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "robust_across_methods",
                      "robust_after_decontamination"
                    ],
                    "canonical": "robust_across_methods"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CXCL9_CXCL10": {
                "op": "in",
                "value": [
                  "immune_response_tumor_state"
                ]
              },
              "CCL5": {
                "op": "in",
                "value": [
                  "shared_across_tumor_states",
                  "partner_lymphocyte_derived",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "gate_conjugate_gate_confounder",
            "role": "gate",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "partner_lymphocyte_capture"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "antigen_presentation_decomposition",
            "kind": "dict",
            "expected": {
              "MHC_class_I": {
                "op": "in",
                "value": [
                  "co_regulated_with_dominant_program",
                  "tumor_intrinsic_enriched_independent"
                ]
              },
              "MHC_class_II": {
                "op": "in",
                "value": [
                  "partner_cell_attributable",
                  "flat_not_enriched",
                  "method_sensitive_no_robust_call",
                  "co_regulated_with_dominant_program"
                ]
              }
            }
          },
          {
            "name": "cf_non_robust_state_claimed_enriched",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "transitory_melanocytic",
              "intermediate_melanocytic",
              "melanocytic",
              "MITF_high",
              "neural_crest_like",
              "mitotic",
              "stress_p53",
              "undifferentiated",
              "dedifferentiated",
              "mesenchymal_tumor",
              "AXL_high_tumor",
              "invasive_tumor",
              "senescence_like",
              "EMT_transition",
              "cancer_stem_cell_like",
              "oxidative_stress_tumor"
            ]
          },
          {
            "name": "cf_framework_generic_or_offtissue_substitution",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "Tsoi_undifferentiated",
              "Tsoi_neural_crest_like",
              "Tsoi_transitory",
              "Tsoi_melanocytic",
              "Rambow_NCSC",
              "Wouters_mesenchymal",
              "Pozniak_TCF4_dependent",
              "tumor_cell",
              "melanoma_cell",
              "malignant_cell",
              "tumor_subpopulation",
              "keratinocyte",
              "hepatocyte",
              "lung_adenocarcinoma_cell",
              "colon_adenocarcinoma_cell",
              "Tirosh_cell_cycle_program",
              "Jerby_Arnon_program",
              "glioblastoma_cell",
              "breast_carcinoma_cell"
            ]
          },
          {
            "name": "cf_immune_response_direction_committed",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_immune_robustness_recall",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_ambiguous_hedge",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].causation_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_stress_generic_stress_response_evidence",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_conjugate_confounder_contamination_denied",
            "role": "hard_fail",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected"
            ]
          },
          {
            "name": "cf_ccl5_recall_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CCL5",
            "kind": "avoid",
            "expected": [
              "immune_response_tumor_state"
            ]
          },
          {
            "name": "cf_cxcl9_cxcl10_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CXCL9_CXCL10",
            "kind": "avoid",
            "expected": [
              "stress_hypoxia_tumor_state"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-4-6-pi",
      "modelName": "claude-sonnet-4-6",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_tumor_state_causation_v5",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 2953,
      "cost": 2.38495785,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "enriched_tumor_states",
            "kind": "list",
            "matchKey": "tumor_state",
            "k": 3,
            "expected": [
              {
                "key": "immune_response",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "direction_unresolved_cross_sectional"
                    ],
                    "canonical": "direction_unresolved_cross_sectional"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "IFN_response_axis",
                      "broad_antigen_presentation_MHC"
                    ],
                    "canonical": "IFN_response_axis"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "enrichment_diminished_by_decontamination",
                      "threshold_sensitive"
                    ],
                    "canonical": "enrichment_diminished_by_decontamination"
                  }
                }
              },
              {
                "key": "stress_hypoxia",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "data_supports_cell_intrinsic_cause"
                    ],
                    "canonical": "data_supports_cell_intrinsic_cause"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "HIF_hypoxia_program_specific"
                    ],
                    "canonical": "HIF_hypoxia_program_specific"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "robust_across_methods",
                      "robust_after_decontamination"
                    ],
                    "canonical": "robust_across_methods"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CXCL9_CXCL10": {
                "op": "in",
                "value": [
                  "immune_response_tumor_state"
                ]
              },
              "CCL5": {
                "op": "in",
                "value": [
                  "shared_across_tumor_states",
                  "partner_lymphocyte_derived",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "gate_conjugate_gate_confounder",
            "role": "gate",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "partner_lymphocyte_capture"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "antigen_presentation_decomposition",
            "kind": "dict",
            "expected": {
              "MHC_class_I": {
                "op": "in",
                "value": [
                  "co_regulated_with_dominant_program",
                  "tumor_intrinsic_enriched_independent"
                ]
              },
              "MHC_class_II": {
                "op": "in",
                "value": [
                  "partner_cell_attributable",
                  "flat_not_enriched",
                  "method_sensitive_no_robust_call",
                  "co_regulated_with_dominant_program"
                ]
              }
            }
          },
          {
            "name": "cf_non_robust_state_claimed_enriched",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "transitory_melanocytic",
              "intermediate_melanocytic",
              "melanocytic",
              "MITF_high",
              "neural_crest_like",
              "mitotic",
              "stress_p53",
              "undifferentiated",
              "dedifferentiated",
              "mesenchymal_tumor",
              "AXL_high_tumor",
              "invasive_tumor",
              "senescence_like",
              "EMT_transition",
              "cancer_stem_cell_like",
              "oxidative_stress_tumor"
            ]
          },
          {
            "name": "cf_framework_generic_or_offtissue_substitution",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "Tsoi_undifferentiated",
              "Tsoi_neural_crest_like",
              "Tsoi_transitory",
              "Tsoi_melanocytic",
              "Rambow_NCSC",
              "Wouters_mesenchymal",
              "Pozniak_TCF4_dependent",
              "tumor_cell",
              "melanoma_cell",
              "malignant_cell",
              "tumor_subpopulation",
              "keratinocyte",
              "hepatocyte",
              "lung_adenocarcinoma_cell",
              "colon_adenocarcinoma_cell",
              "Tirosh_cell_cycle_program",
              "Jerby_Arnon_program",
              "glioblastoma_cell",
              "breast_carcinoma_cell"
            ]
          },
          {
            "name": "cf_immune_response_direction_committed",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_immune_robustness_recall",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_ambiguous_hedge",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].causation_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_stress_generic_stress_response_evidence",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_conjugate_confounder_contamination_denied",
            "role": "hard_fail",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected"
            ]
          },
          {
            "name": "cf_ccl5_recall_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CCL5",
            "kind": "avoid",
            "expected": [
              "immune_response_tumor_state"
            ]
          },
          {
            "name": "cf_cxcl9_cxcl10_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CXCL9_CXCL10",
            "kind": "avoid",
            "expected": [
              "stress_hypoxia_tumor_state"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-4-6-pi",
      "modelName": "claude-sonnet-4-6",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_tumor_state_causation_v5",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 2711,
      "cost": 2.5535451,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "enriched_tumor_states",
            "kind": "list",
            "matchKey": "tumor_state",
            "k": 3,
            "expected": [
              {
                "key": "immune_response",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "direction_unresolved_cross_sectional"
                    ],
                    "canonical": "direction_unresolved_cross_sectional"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "IFN_response_axis",
                      "broad_antigen_presentation_MHC"
                    ],
                    "canonical": "IFN_response_axis"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "enrichment_diminished_by_decontamination",
                      "threshold_sensitive"
                    ],
                    "canonical": "enrichment_diminished_by_decontamination"
                  }
                }
              },
              {
                "key": "stress_hypoxia",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "data_supports_cell_intrinsic_cause"
                    ],
                    "canonical": "data_supports_cell_intrinsic_cause"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "HIF_hypoxia_program_specific"
                    ],
                    "canonical": "HIF_hypoxia_program_specific"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "robust_across_methods",
                      "robust_after_decontamination"
                    ],
                    "canonical": "robust_across_methods"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CXCL9_CXCL10": {
                "op": "in",
                "value": [
                  "immune_response_tumor_state"
                ]
              },
              "CCL5": {
                "op": "in",
                "value": [
                  "shared_across_tumor_states",
                  "partner_lymphocyte_derived",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "gate_conjugate_gate_confounder",
            "role": "gate",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "partner_lymphocyte_capture"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "antigen_presentation_decomposition",
            "kind": "dict",
            "expected": {
              "MHC_class_I": {
                "op": "in",
                "value": [
                  "co_regulated_with_dominant_program",
                  "tumor_intrinsic_enriched_independent"
                ]
              },
              "MHC_class_II": {
                "op": "in",
                "value": [
                  "partner_cell_attributable",
                  "flat_not_enriched",
                  "method_sensitive_no_robust_call",
                  "co_regulated_with_dominant_program"
                ]
              }
            }
          },
          {
            "name": "cf_non_robust_state_claimed_enriched",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "transitory_melanocytic",
              "intermediate_melanocytic",
              "melanocytic",
              "MITF_high",
              "neural_crest_like",
              "mitotic",
              "stress_p53",
              "undifferentiated",
              "dedifferentiated",
              "mesenchymal_tumor",
              "AXL_high_tumor",
              "invasive_tumor",
              "senescence_like",
              "EMT_transition",
              "cancer_stem_cell_like",
              "oxidative_stress_tumor"
            ]
          },
          {
            "name": "cf_framework_generic_or_offtissue_substitution",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "Tsoi_undifferentiated",
              "Tsoi_neural_crest_like",
              "Tsoi_transitory",
              "Tsoi_melanocytic",
              "Rambow_NCSC",
              "Wouters_mesenchymal",
              "Pozniak_TCF4_dependent",
              "tumor_cell",
              "melanoma_cell",
              "malignant_cell",
              "tumor_subpopulation",
              "keratinocyte",
              "hepatocyte",
              "lung_adenocarcinoma_cell",
              "colon_adenocarcinoma_cell",
              "Tirosh_cell_cycle_program",
              "Jerby_Arnon_program",
              "glioblastoma_cell",
              "breast_carcinoma_cell"
            ]
          },
          {
            "name": "cf_immune_response_direction_committed",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_immune_robustness_recall",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_ambiguous_hedge",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].causation_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_stress_generic_stress_response_evidence",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_conjugate_confounder_contamination_denied",
            "role": "hard_fail",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected"
            ]
          },
          {
            "name": "cf_ccl5_recall_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CCL5",
            "kind": "avoid",
            "expected": [
              "immune_response_tumor_state"
            ]
          },
          {
            "name": "cf_cxcl9_cxcl10_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CXCL9_CXCL10",
            "kind": "avoid",
            "expected": [
              "stress_hypoxia_tumor_state"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gemini-3-1-pro-preview-pi",
      "modelName": "gemini-3.1-pro-preview",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_tumor_state_causation_v5",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 1290,
      "cost": 0.6324261999999999,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "enriched_tumor_states",
            "kind": "list",
            "matchKey": "tumor_state",
            "k": 3,
            "expected": [
              {
                "key": "immune_response",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "direction_unresolved_cross_sectional"
                    ],
                    "canonical": "direction_unresolved_cross_sectional"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "IFN_response_axis",
                      "broad_antigen_presentation_MHC"
                    ],
                    "canonical": "IFN_response_axis"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "enrichment_diminished_by_decontamination",
                      "threshold_sensitive"
                    ],
                    "canonical": "enrichment_diminished_by_decontamination"
                  }
                }
              },
              {
                "key": "stress_hypoxia",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "data_supports_cell_intrinsic_cause"
                    ],
                    "canonical": "data_supports_cell_intrinsic_cause"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "HIF_hypoxia_program_specific"
                    ],
                    "canonical": "HIF_hypoxia_program_specific"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "robust_across_methods",
                      "robust_after_decontamination"
                    ],
                    "canonical": "robust_across_methods"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CXCL9_CXCL10": {
                "op": "in",
                "value": [
                  "immune_response_tumor_state"
                ]
              },
              "CCL5": {
                "op": "in",
                "value": [
                  "shared_across_tumor_states",
                  "partner_lymphocyte_derived",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "gate_conjugate_gate_confounder",
            "role": "gate",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "partner_lymphocyte_capture"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "antigen_presentation_decomposition",
            "kind": "dict",
            "expected": {
              "MHC_class_I": {
                "op": "in",
                "value": [
                  "co_regulated_with_dominant_program",
                  "tumor_intrinsic_enriched_independent"
                ]
              },
              "MHC_class_II": {
                "op": "in",
                "value": [
                  "partner_cell_attributable",
                  "flat_not_enriched",
                  "method_sensitive_no_robust_call",
                  "co_regulated_with_dominant_program"
                ]
              }
            }
          },
          {
            "name": "cf_non_robust_state_claimed_enriched",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "transitory_melanocytic",
              "intermediate_melanocytic",
              "melanocytic",
              "MITF_high",
              "neural_crest_like",
              "mitotic",
              "stress_p53",
              "undifferentiated",
              "dedifferentiated",
              "mesenchymal_tumor",
              "AXL_high_tumor",
              "invasive_tumor",
              "senescence_like",
              "EMT_transition",
              "cancer_stem_cell_like",
              "oxidative_stress_tumor"
            ]
          },
          {
            "name": "cf_framework_generic_or_offtissue_substitution",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "Tsoi_undifferentiated",
              "Tsoi_neural_crest_like",
              "Tsoi_transitory",
              "Tsoi_melanocytic",
              "Rambow_NCSC",
              "Wouters_mesenchymal",
              "Pozniak_TCF4_dependent",
              "tumor_cell",
              "melanoma_cell",
              "malignant_cell",
              "tumor_subpopulation",
              "keratinocyte",
              "hepatocyte",
              "lung_adenocarcinoma_cell",
              "colon_adenocarcinoma_cell",
              "Tirosh_cell_cycle_program",
              "Jerby_Arnon_program",
              "glioblastoma_cell",
              "breast_carcinoma_cell"
            ]
          },
          {
            "name": "cf_immune_response_direction_committed",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_immune_robustness_recall",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_ambiguous_hedge",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].causation_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_stress_generic_stress_response_evidence",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_conjugate_confounder_contamination_denied",
            "role": "hard_fail",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected"
            ]
          },
          {
            "name": "cf_ccl5_recall_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CCL5",
            "kind": "avoid",
            "expected": [
              "immune_response_tumor_state"
            ]
          },
          {
            "name": "cf_cxcl9_cxcl10_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CXCL9_CXCL10",
            "kind": "avoid",
            "expected": [
              "stress_hypoxia_tumor_state"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gemini-3-1-pro-preview-pi",
      "modelName": "gemini-3.1-pro-preview",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_tumor_state_causation_v5",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 2221,
      "cost": 1.21627,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "enriched_tumor_states",
            "kind": "list",
            "matchKey": "tumor_state",
            "k": 3,
            "expected": [
              {
                "key": "immune_response",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "direction_unresolved_cross_sectional"
                    ],
                    "canonical": "direction_unresolved_cross_sectional"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "IFN_response_axis",
                      "broad_antigen_presentation_MHC"
                    ],
                    "canonical": "IFN_response_axis"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "enrichment_diminished_by_decontamination",
                      "threshold_sensitive"
                    ],
                    "canonical": "enrichment_diminished_by_decontamination"
                  }
                }
              },
              {
                "key": "stress_hypoxia",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "data_supports_cell_intrinsic_cause"
                    ],
                    "canonical": "data_supports_cell_intrinsic_cause"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "HIF_hypoxia_program_specific"
                    ],
                    "canonical": "HIF_hypoxia_program_specific"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "robust_across_methods",
                      "robust_after_decontamination"
                    ],
                    "canonical": "robust_across_methods"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CXCL9_CXCL10": {
                "op": "in",
                "value": [
                  "immune_response_tumor_state"
                ]
              },
              "CCL5": {
                "op": "in",
                "value": [
                  "shared_across_tumor_states",
                  "partner_lymphocyte_derived",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "gate_conjugate_gate_confounder",
            "role": "gate",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "partner_lymphocyte_capture"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "antigen_presentation_decomposition",
            "kind": "dict",
            "expected": {
              "MHC_class_I": {
                "op": "in",
                "value": [
                  "co_regulated_with_dominant_program",
                  "tumor_intrinsic_enriched_independent"
                ]
              },
              "MHC_class_II": {
                "op": "in",
                "value": [
                  "partner_cell_attributable",
                  "flat_not_enriched",
                  "method_sensitive_no_robust_call",
                  "co_regulated_with_dominant_program"
                ]
              }
            }
          },
          {
            "name": "cf_non_robust_state_claimed_enriched",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "transitory_melanocytic",
              "intermediate_melanocytic",
              "melanocytic",
              "MITF_high",
              "neural_crest_like",
              "mitotic",
              "stress_p53",
              "undifferentiated",
              "dedifferentiated",
              "mesenchymal_tumor",
              "AXL_high_tumor",
              "invasive_tumor",
              "senescence_like",
              "EMT_transition",
              "cancer_stem_cell_like",
              "oxidative_stress_tumor"
            ]
          },
          {
            "name": "cf_framework_generic_or_offtissue_substitution",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "Tsoi_undifferentiated",
              "Tsoi_neural_crest_like",
              "Tsoi_transitory",
              "Tsoi_melanocytic",
              "Rambow_NCSC",
              "Wouters_mesenchymal",
              "Pozniak_TCF4_dependent",
              "tumor_cell",
              "melanoma_cell",
              "malignant_cell",
              "tumor_subpopulation",
              "keratinocyte",
              "hepatocyte",
              "lung_adenocarcinoma_cell",
              "colon_adenocarcinoma_cell",
              "Tirosh_cell_cycle_program",
              "Jerby_Arnon_program",
              "glioblastoma_cell",
              "breast_carcinoma_cell"
            ]
          },
          {
            "name": "cf_immune_response_direction_committed",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_immune_robustness_recall",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_ambiguous_hedge",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].causation_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_stress_generic_stress_response_evidence",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_conjugate_confounder_contamination_denied",
            "role": "hard_fail",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected"
            ]
          },
          {
            "name": "cf_ccl5_recall_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CCL5",
            "kind": "avoid",
            "expected": [
              "immune_response_tumor_state"
            ]
          },
          {
            "name": "cf_cxcl9_cxcl10_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CXCL9_CXCL10",
            "kind": "avoid",
            "expected": [
              "stress_hypoxia_tumor_state"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gemini-3-1-pro-preview-pi",
      "modelName": "gemini-3.1-pro-preview",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_tumor_state_causation_v5",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 3304,
      "cost": 0.7400798000000002,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "enriched_tumor_states",
            "kind": "list",
            "matchKey": "tumor_state",
            "k": 3,
            "expected": [
              {
                "key": "immune_response",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "direction_unresolved_cross_sectional"
                    ],
                    "canonical": "direction_unresolved_cross_sectional"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "IFN_response_axis",
                      "broad_antigen_presentation_MHC"
                    ],
                    "canonical": "IFN_response_axis"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "enrichment_diminished_by_decontamination",
                      "threshold_sensitive"
                    ],
                    "canonical": "enrichment_diminished_by_decontamination"
                  }
                }
              },
              {
                "key": "stress_hypoxia",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "data_supports_cell_intrinsic_cause"
                    ],
                    "canonical": "data_supports_cell_intrinsic_cause"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "HIF_hypoxia_program_specific"
                    ],
                    "canonical": "HIF_hypoxia_program_specific"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "robust_across_methods",
                      "robust_after_decontamination"
                    ],
                    "canonical": "robust_across_methods"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CXCL9_CXCL10": {
                "op": "in",
                "value": [
                  "immune_response_tumor_state"
                ]
              },
              "CCL5": {
                "op": "in",
                "value": [
                  "shared_across_tumor_states",
                  "partner_lymphocyte_derived",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "gate_conjugate_gate_confounder",
            "role": "gate",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "partner_lymphocyte_capture"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "antigen_presentation_decomposition",
            "kind": "dict",
            "expected": {
              "MHC_class_I": {
                "op": "in",
                "value": [
                  "co_regulated_with_dominant_program",
                  "tumor_intrinsic_enriched_independent"
                ]
              },
              "MHC_class_II": {
                "op": "in",
                "value": [
                  "partner_cell_attributable",
                  "flat_not_enriched",
                  "method_sensitive_no_robust_call",
                  "co_regulated_with_dominant_program"
                ]
              }
            }
          },
          {
            "name": "cf_non_robust_state_claimed_enriched",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "transitory_melanocytic",
              "intermediate_melanocytic",
              "melanocytic",
              "MITF_high",
              "neural_crest_like",
              "mitotic",
              "stress_p53",
              "undifferentiated",
              "dedifferentiated",
              "mesenchymal_tumor",
              "AXL_high_tumor",
              "invasive_tumor",
              "senescence_like",
              "EMT_transition",
              "cancer_stem_cell_like",
              "oxidative_stress_tumor"
            ]
          },
          {
            "name": "cf_framework_generic_or_offtissue_substitution",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "Tsoi_undifferentiated",
              "Tsoi_neural_crest_like",
              "Tsoi_transitory",
              "Tsoi_melanocytic",
              "Rambow_NCSC",
              "Wouters_mesenchymal",
              "Pozniak_TCF4_dependent",
              "tumor_cell",
              "melanoma_cell",
              "malignant_cell",
              "tumor_subpopulation",
              "keratinocyte",
              "hepatocyte",
              "lung_adenocarcinoma_cell",
              "colon_adenocarcinoma_cell",
              "Tirosh_cell_cycle_program",
              "Jerby_Arnon_program",
              "glioblastoma_cell",
              "breast_carcinoma_cell"
            ]
          },
          {
            "name": "cf_immune_response_direction_committed",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_immune_robustness_recall",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_ambiguous_hedge",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].causation_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_stress_generic_stress_response_evidence",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_conjugate_confounder_contamination_denied",
            "role": "hard_fail",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected"
            ]
          },
          {
            "name": "cf_ccl5_recall_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CCL5",
            "kind": "avoid",
            "expected": [
              "immune_response_tumor_state"
            ]
          },
          {
            "name": "cf_cxcl9_cxcl10_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CXCL9_CXCL10",
            "kind": "avoid",
            "expected": [
              "stress_hypoxia_tumor_state"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gemini-3-5-flash-pi",
      "modelName": "gemini-3.5-flash",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_tumor_state_causation_v5",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 1530,
      "cost": 1.3162995,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "enriched_tumor_states",
            "kind": "list",
            "matchKey": "tumor_state",
            "k": 3,
            "expected": [
              {
                "key": "immune_response",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "direction_unresolved_cross_sectional"
                    ],
                    "canonical": "direction_unresolved_cross_sectional"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "IFN_response_axis",
                      "broad_antigen_presentation_MHC"
                    ],
                    "canonical": "IFN_response_axis"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "enrichment_diminished_by_decontamination",
                      "threshold_sensitive"
                    ],
                    "canonical": "enrichment_diminished_by_decontamination"
                  }
                }
              },
              {
                "key": "stress_hypoxia",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "data_supports_cell_intrinsic_cause"
                    ],
                    "canonical": "data_supports_cell_intrinsic_cause"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "HIF_hypoxia_program_specific"
                    ],
                    "canonical": "HIF_hypoxia_program_specific"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "robust_across_methods",
                      "robust_after_decontamination"
                    ],
                    "canonical": "robust_across_methods"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CXCL9_CXCL10": {
                "op": "in",
                "value": [
                  "immune_response_tumor_state"
                ]
              },
              "CCL5": {
                "op": "in",
                "value": [
                  "shared_across_tumor_states",
                  "partner_lymphocyte_derived",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "gate_conjugate_gate_confounder",
            "role": "gate",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "partner_lymphocyte_capture"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "antigen_presentation_decomposition",
            "kind": "dict",
            "expected": {
              "MHC_class_I": {
                "op": "in",
                "value": [
                  "co_regulated_with_dominant_program",
                  "tumor_intrinsic_enriched_independent"
                ]
              },
              "MHC_class_II": {
                "op": "in",
                "value": [
                  "partner_cell_attributable",
                  "flat_not_enriched",
                  "method_sensitive_no_robust_call",
                  "co_regulated_with_dominant_program"
                ]
              }
            }
          },
          {
            "name": "cf_non_robust_state_claimed_enriched",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "transitory_melanocytic",
              "intermediate_melanocytic",
              "melanocytic",
              "MITF_high",
              "neural_crest_like",
              "mitotic",
              "stress_p53",
              "undifferentiated",
              "dedifferentiated",
              "mesenchymal_tumor",
              "AXL_high_tumor",
              "invasive_tumor",
              "senescence_like",
              "EMT_transition",
              "cancer_stem_cell_like",
              "oxidative_stress_tumor"
            ]
          },
          {
            "name": "cf_framework_generic_or_offtissue_substitution",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "Tsoi_undifferentiated",
              "Tsoi_neural_crest_like",
              "Tsoi_transitory",
              "Tsoi_melanocytic",
              "Rambow_NCSC",
              "Wouters_mesenchymal",
              "Pozniak_TCF4_dependent",
              "tumor_cell",
              "melanoma_cell",
              "malignant_cell",
              "tumor_subpopulation",
              "keratinocyte",
              "hepatocyte",
              "lung_adenocarcinoma_cell",
              "colon_adenocarcinoma_cell",
              "Tirosh_cell_cycle_program",
              "Jerby_Arnon_program",
              "glioblastoma_cell",
              "breast_carcinoma_cell"
            ]
          },
          {
            "name": "cf_immune_response_direction_committed",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_immune_robustness_recall",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_ambiguous_hedge",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].causation_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_stress_generic_stress_response_evidence",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_conjugate_confounder_contamination_denied",
            "role": "hard_fail",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected"
            ]
          },
          {
            "name": "cf_ccl5_recall_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CCL5",
            "kind": "avoid",
            "expected": [
              "immune_response_tumor_state"
            ]
          },
          {
            "name": "cf_cxcl9_cxcl10_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CXCL9_CXCL10",
            "kind": "avoid",
            "expected": [
              "stress_hypoxia_tumor_state"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gemini-3-5-flash-pi",
      "modelName": "gemini-3.5-flash",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_tumor_state_causation_v5",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 2056,
      "cost": 1.6278288000000007,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "enriched_tumor_states",
            "kind": "list",
            "matchKey": "tumor_state",
            "k": 3,
            "expected": [
              {
                "key": "immune_response",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "direction_unresolved_cross_sectional"
                    ],
                    "canonical": "direction_unresolved_cross_sectional"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "IFN_response_axis",
                      "broad_antigen_presentation_MHC"
                    ],
                    "canonical": "IFN_response_axis"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "enrichment_diminished_by_decontamination",
                      "threshold_sensitive"
                    ],
                    "canonical": "enrichment_diminished_by_decontamination"
                  }
                }
              },
              {
                "key": "stress_hypoxia",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "data_supports_cell_intrinsic_cause"
                    ],
                    "canonical": "data_supports_cell_intrinsic_cause"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "HIF_hypoxia_program_specific"
                    ],
                    "canonical": "HIF_hypoxia_program_specific"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "robust_across_methods",
                      "robust_after_decontamination"
                    ],
                    "canonical": "robust_across_methods"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CXCL9_CXCL10": {
                "op": "in",
                "value": [
                  "immune_response_tumor_state"
                ]
              },
              "CCL5": {
                "op": "in",
                "value": [
                  "shared_across_tumor_states",
                  "partner_lymphocyte_derived",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "gate_conjugate_gate_confounder",
            "role": "gate",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "partner_lymphocyte_capture"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "antigen_presentation_decomposition",
            "kind": "dict",
            "expected": {
              "MHC_class_I": {
                "op": "in",
                "value": [
                  "co_regulated_with_dominant_program",
                  "tumor_intrinsic_enriched_independent"
                ]
              },
              "MHC_class_II": {
                "op": "in",
                "value": [
                  "partner_cell_attributable",
                  "flat_not_enriched",
                  "method_sensitive_no_robust_call",
                  "co_regulated_with_dominant_program"
                ]
              }
            }
          },
          {
            "name": "cf_non_robust_state_claimed_enriched",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "transitory_melanocytic",
              "intermediate_melanocytic",
              "melanocytic",
              "MITF_high",
              "neural_crest_like",
              "mitotic",
              "stress_p53",
              "undifferentiated",
              "dedifferentiated",
              "mesenchymal_tumor",
              "AXL_high_tumor",
              "invasive_tumor",
              "senescence_like",
              "EMT_transition",
              "cancer_stem_cell_like",
              "oxidative_stress_tumor"
            ]
          },
          {
            "name": "cf_framework_generic_or_offtissue_substitution",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "Tsoi_undifferentiated",
              "Tsoi_neural_crest_like",
              "Tsoi_transitory",
              "Tsoi_melanocytic",
              "Rambow_NCSC",
              "Wouters_mesenchymal",
              "Pozniak_TCF4_dependent",
              "tumor_cell",
              "melanoma_cell",
              "malignant_cell",
              "tumor_subpopulation",
              "keratinocyte",
              "hepatocyte",
              "lung_adenocarcinoma_cell",
              "colon_adenocarcinoma_cell",
              "Tirosh_cell_cycle_program",
              "Jerby_Arnon_program",
              "glioblastoma_cell",
              "breast_carcinoma_cell"
            ]
          },
          {
            "name": "cf_immune_response_direction_committed",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_immune_robustness_recall",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_ambiguous_hedge",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].causation_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_stress_generic_stress_response_evidence",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_conjugate_confounder_contamination_denied",
            "role": "hard_fail",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected"
            ]
          },
          {
            "name": "cf_ccl5_recall_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CCL5",
            "kind": "avoid",
            "expected": [
              "immune_response_tumor_state"
            ]
          },
          {
            "name": "cf_cxcl9_cxcl10_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CXCL9_CXCL10",
            "kind": "avoid",
            "expected": [
              "stress_hypoxia_tumor_state"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gemini-3-5-flash-pi",
      "modelName": "gemini-3.5-flash",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_tumor_state_causation_v5",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1907,
      "cost": 0.8726769000000002,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "enriched_tumor_states",
            "kind": "list",
            "matchKey": "tumor_state",
            "k": 3,
            "expected": [
              {
                "key": "immune_response",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "direction_unresolved_cross_sectional"
                    ],
                    "canonical": "direction_unresolved_cross_sectional"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "IFN_response_axis",
                      "broad_antigen_presentation_MHC"
                    ],
                    "canonical": "IFN_response_axis"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "enrichment_diminished_by_decontamination",
                      "threshold_sensitive"
                    ],
                    "canonical": "enrichment_diminished_by_decontamination"
                  }
                }
              },
              {
                "key": "stress_hypoxia",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "data_supports_cell_intrinsic_cause"
                    ],
                    "canonical": "data_supports_cell_intrinsic_cause"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "HIF_hypoxia_program_specific"
                    ],
                    "canonical": "HIF_hypoxia_program_specific"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "robust_across_methods",
                      "robust_after_decontamination"
                    ],
                    "canonical": "robust_across_methods"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CXCL9_CXCL10": {
                "op": "in",
                "value": [
                  "immune_response_tumor_state"
                ]
              },
              "CCL5": {
                "op": "in",
                "value": [
                  "shared_across_tumor_states",
                  "partner_lymphocyte_derived",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "gate_conjugate_gate_confounder",
            "role": "gate",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "partner_lymphocyte_capture"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "antigen_presentation_decomposition",
            "kind": "dict",
            "expected": {
              "MHC_class_I": {
                "op": "in",
                "value": [
                  "co_regulated_with_dominant_program",
                  "tumor_intrinsic_enriched_independent"
                ]
              },
              "MHC_class_II": {
                "op": "in",
                "value": [
                  "partner_cell_attributable",
                  "flat_not_enriched",
                  "method_sensitive_no_robust_call",
                  "co_regulated_with_dominant_program"
                ]
              }
            }
          },
          {
            "name": "cf_non_robust_state_claimed_enriched",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "transitory_melanocytic",
              "intermediate_melanocytic",
              "melanocytic",
              "MITF_high",
              "neural_crest_like",
              "mitotic",
              "stress_p53",
              "undifferentiated",
              "dedifferentiated",
              "mesenchymal_tumor",
              "AXL_high_tumor",
              "invasive_tumor",
              "senescence_like",
              "EMT_transition",
              "cancer_stem_cell_like",
              "oxidative_stress_tumor"
            ]
          },
          {
            "name": "cf_framework_generic_or_offtissue_substitution",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "Tsoi_undifferentiated",
              "Tsoi_neural_crest_like",
              "Tsoi_transitory",
              "Tsoi_melanocytic",
              "Rambow_NCSC",
              "Wouters_mesenchymal",
              "Pozniak_TCF4_dependent",
              "tumor_cell",
              "melanoma_cell",
              "malignant_cell",
              "tumor_subpopulation",
              "keratinocyte",
              "hepatocyte",
              "lung_adenocarcinoma_cell",
              "colon_adenocarcinoma_cell",
              "Tirosh_cell_cycle_program",
              "Jerby_Arnon_program",
              "glioblastoma_cell",
              "breast_carcinoma_cell"
            ]
          },
          {
            "name": "cf_immune_response_direction_committed",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_immune_robustness_recall",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_ambiguous_hedge",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].causation_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_stress_generic_stress_response_evidence",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_conjugate_confounder_contamination_denied",
            "role": "hard_fail",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected"
            ]
          },
          {
            "name": "cf_ccl5_recall_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CCL5",
            "kind": "avoid",
            "expected": [
              "immune_response_tumor_state"
            ]
          },
          {
            "name": "cf_cxcl9_cxcl10_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CXCL9_CXCL10",
            "kind": "avoid",
            "expected": [
              "stress_hypoxia_tumor_state"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-glm-5p2-pi",
      "modelName": "glm-5p2",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_tumor_state_causation_v5",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 4931,
      "cost": 3.8467468200000003,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "enriched_tumor_states",
            "kind": "list",
            "matchKey": "tumor_state",
            "k": 3,
            "expected": [
              {
                "key": "immune_response",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "direction_unresolved_cross_sectional"
                    ],
                    "canonical": "direction_unresolved_cross_sectional"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "IFN_response_axis",
                      "broad_antigen_presentation_MHC"
                    ],
                    "canonical": "IFN_response_axis"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "enrichment_diminished_by_decontamination",
                      "threshold_sensitive"
                    ],
                    "canonical": "enrichment_diminished_by_decontamination"
                  }
                }
              },
              {
                "key": "stress_hypoxia",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "data_supports_cell_intrinsic_cause"
                    ],
                    "canonical": "data_supports_cell_intrinsic_cause"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "HIF_hypoxia_program_specific"
                    ],
                    "canonical": "HIF_hypoxia_program_specific"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "robust_across_methods",
                      "robust_after_decontamination"
                    ],
                    "canonical": "robust_across_methods"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CXCL9_CXCL10": {
                "op": "in",
                "value": [
                  "immune_response_tumor_state"
                ]
              },
              "CCL5": {
                "op": "in",
                "value": [
                  "shared_across_tumor_states",
                  "partner_lymphocyte_derived",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "gate_conjugate_gate_confounder",
            "role": "gate",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "partner_lymphocyte_capture"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "antigen_presentation_decomposition",
            "kind": "dict",
            "expected": {
              "MHC_class_I": {
                "op": "in",
                "value": [
                  "co_regulated_with_dominant_program",
                  "tumor_intrinsic_enriched_independent"
                ]
              },
              "MHC_class_II": {
                "op": "in",
                "value": [
                  "partner_cell_attributable",
                  "flat_not_enriched",
                  "method_sensitive_no_robust_call",
                  "co_regulated_with_dominant_program"
                ]
              }
            }
          },
          {
            "name": "cf_non_robust_state_claimed_enriched",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "transitory_melanocytic",
              "intermediate_melanocytic",
              "melanocytic",
              "MITF_high",
              "neural_crest_like",
              "mitotic",
              "stress_p53",
              "undifferentiated",
              "dedifferentiated",
              "mesenchymal_tumor",
              "AXL_high_tumor",
              "invasive_tumor",
              "senescence_like",
              "EMT_transition",
              "cancer_stem_cell_like",
              "oxidative_stress_tumor"
            ]
          },
          {
            "name": "cf_framework_generic_or_offtissue_substitution",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "Tsoi_undifferentiated",
              "Tsoi_neural_crest_like",
              "Tsoi_transitory",
              "Tsoi_melanocytic",
              "Rambow_NCSC",
              "Wouters_mesenchymal",
              "Pozniak_TCF4_dependent",
              "tumor_cell",
              "melanoma_cell",
              "malignant_cell",
              "tumor_subpopulation",
              "keratinocyte",
              "hepatocyte",
              "lung_adenocarcinoma_cell",
              "colon_adenocarcinoma_cell",
              "Tirosh_cell_cycle_program",
              "Jerby_Arnon_program",
              "glioblastoma_cell",
              "breast_carcinoma_cell"
            ]
          },
          {
            "name": "cf_immune_response_direction_committed",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_immune_robustness_recall",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_ambiguous_hedge",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].causation_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_stress_generic_stress_response_evidence",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_conjugate_confounder_contamination_denied",
            "role": "hard_fail",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected"
            ]
          },
          {
            "name": "cf_ccl5_recall_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CCL5",
            "kind": "avoid",
            "expected": [
              "immune_response_tumor_state"
            ]
          },
          {
            "name": "cf_cxcl9_cxcl10_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CXCL9_CXCL10",
            "kind": "avoid",
            "expected": [
              "stress_hypoxia_tumor_state"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-glm-5p2-pi",
      "modelName": "glm-5p2",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_tumor_state_causation_v5",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 4376,
      "cost": 5.73953692,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "enriched_tumor_states",
            "kind": "list",
            "matchKey": "tumor_state",
            "k": 3,
            "expected": [
              {
                "key": "immune_response",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "direction_unresolved_cross_sectional"
                    ],
                    "canonical": "direction_unresolved_cross_sectional"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "IFN_response_axis",
                      "broad_antigen_presentation_MHC"
                    ],
                    "canonical": "IFN_response_axis"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "enrichment_diminished_by_decontamination",
                      "threshold_sensitive"
                    ],
                    "canonical": "enrichment_diminished_by_decontamination"
                  }
                }
              },
              {
                "key": "stress_hypoxia",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "data_supports_cell_intrinsic_cause"
                    ],
                    "canonical": "data_supports_cell_intrinsic_cause"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "HIF_hypoxia_program_specific"
                    ],
                    "canonical": "HIF_hypoxia_program_specific"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "robust_across_methods",
                      "robust_after_decontamination"
                    ],
                    "canonical": "robust_across_methods"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CXCL9_CXCL10": {
                "op": "in",
                "value": [
                  "immune_response_tumor_state"
                ]
              },
              "CCL5": {
                "op": "in",
                "value": [
                  "shared_across_tumor_states",
                  "partner_lymphocyte_derived",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "gate_conjugate_gate_confounder",
            "role": "gate",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "partner_lymphocyte_capture"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "antigen_presentation_decomposition",
            "kind": "dict",
            "expected": {
              "MHC_class_I": {
                "op": "in",
                "value": [
                  "co_regulated_with_dominant_program",
                  "tumor_intrinsic_enriched_independent"
                ]
              },
              "MHC_class_II": {
                "op": "in",
                "value": [
                  "partner_cell_attributable",
                  "flat_not_enriched",
                  "method_sensitive_no_robust_call",
                  "co_regulated_with_dominant_program"
                ]
              }
            }
          },
          {
            "name": "cf_non_robust_state_claimed_enriched",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "transitory_melanocytic",
              "intermediate_melanocytic",
              "melanocytic",
              "MITF_high",
              "neural_crest_like",
              "mitotic",
              "stress_p53",
              "undifferentiated",
              "dedifferentiated",
              "mesenchymal_tumor",
              "AXL_high_tumor",
              "invasive_tumor",
              "senescence_like",
              "EMT_transition",
              "cancer_stem_cell_like",
              "oxidative_stress_tumor"
            ]
          },
          {
            "name": "cf_framework_generic_or_offtissue_substitution",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "Tsoi_undifferentiated",
              "Tsoi_neural_crest_like",
              "Tsoi_transitory",
              "Tsoi_melanocytic",
              "Rambow_NCSC",
              "Wouters_mesenchymal",
              "Pozniak_TCF4_dependent",
              "tumor_cell",
              "melanoma_cell",
              "malignant_cell",
              "tumor_subpopulation",
              "keratinocyte",
              "hepatocyte",
              "lung_adenocarcinoma_cell",
              "colon_adenocarcinoma_cell",
              "Tirosh_cell_cycle_program",
              "Jerby_Arnon_program",
              "glioblastoma_cell",
              "breast_carcinoma_cell"
            ]
          },
          {
            "name": "cf_immune_response_direction_committed",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_immune_robustness_recall",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_ambiguous_hedge",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].causation_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_stress_generic_stress_response_evidence",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_conjugate_confounder_contamination_denied",
            "role": "hard_fail",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected"
            ]
          },
          {
            "name": "cf_ccl5_recall_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CCL5",
            "kind": "avoid",
            "expected": [
              "immune_response_tumor_state"
            ]
          },
          {
            "name": "cf_cxcl9_cxcl10_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CXCL9_CXCL10",
            "kind": "avoid",
            "expected": [
              "stress_hypoxia_tumor_state"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-glm-5p2-pi",
      "modelName": "glm-5p2",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_tumor_state_causation_v5",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 4364,
      "cost": 3.895923539999999,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "enriched_tumor_states",
            "kind": "list",
            "matchKey": "tumor_state",
            "k": 3,
            "expected": [
              {
                "key": "immune_response",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "direction_unresolved_cross_sectional"
                    ],
                    "canonical": "direction_unresolved_cross_sectional"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "IFN_response_axis",
                      "broad_antigen_presentation_MHC"
                    ],
                    "canonical": "IFN_response_axis"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "enrichment_diminished_by_decontamination",
                      "threshold_sensitive"
                    ],
                    "canonical": "enrichment_diminished_by_decontamination"
                  }
                }
              },
              {
                "key": "stress_hypoxia",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "data_supports_cell_intrinsic_cause"
                    ],
                    "canonical": "data_supports_cell_intrinsic_cause"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "HIF_hypoxia_program_specific"
                    ],
                    "canonical": "HIF_hypoxia_program_specific"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "robust_across_methods",
                      "robust_after_decontamination"
                    ],
                    "canonical": "robust_across_methods"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CXCL9_CXCL10": {
                "op": "in",
                "value": [
                  "immune_response_tumor_state"
                ]
              },
              "CCL5": {
                "op": "in",
                "value": [
                  "shared_across_tumor_states",
                  "partner_lymphocyte_derived",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "gate_conjugate_gate_confounder",
            "role": "gate",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "partner_lymphocyte_capture"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "antigen_presentation_decomposition",
            "kind": "dict",
            "expected": {
              "MHC_class_I": {
                "op": "in",
                "value": [
                  "co_regulated_with_dominant_program",
                  "tumor_intrinsic_enriched_independent"
                ]
              },
              "MHC_class_II": {
                "op": "in",
                "value": [
                  "partner_cell_attributable",
                  "flat_not_enriched",
                  "method_sensitive_no_robust_call",
                  "co_regulated_with_dominant_program"
                ]
              }
            }
          },
          {
            "name": "cf_non_robust_state_claimed_enriched",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "transitory_melanocytic",
              "intermediate_melanocytic",
              "melanocytic",
              "MITF_high",
              "neural_crest_like",
              "mitotic",
              "stress_p53",
              "undifferentiated",
              "dedifferentiated",
              "mesenchymal_tumor",
              "AXL_high_tumor",
              "invasive_tumor",
              "senescence_like",
              "EMT_transition",
              "cancer_stem_cell_like",
              "oxidative_stress_tumor"
            ]
          },
          {
            "name": "cf_framework_generic_or_offtissue_substitution",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "Tsoi_undifferentiated",
              "Tsoi_neural_crest_like",
              "Tsoi_transitory",
              "Tsoi_melanocytic",
              "Rambow_NCSC",
              "Wouters_mesenchymal",
              "Pozniak_TCF4_dependent",
              "tumor_cell",
              "melanoma_cell",
              "malignant_cell",
              "tumor_subpopulation",
              "keratinocyte",
              "hepatocyte",
              "lung_adenocarcinoma_cell",
              "colon_adenocarcinoma_cell",
              "Tirosh_cell_cycle_program",
              "Jerby_Arnon_program",
              "glioblastoma_cell",
              "breast_carcinoma_cell"
            ]
          },
          {
            "name": "cf_immune_response_direction_committed",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_immune_robustness_recall",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_ambiguous_hedge",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].causation_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_stress_generic_stress_response_evidence",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_conjugate_confounder_contamination_denied",
            "role": "hard_fail",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected"
            ]
          },
          {
            "name": "cf_ccl5_recall_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CCL5",
            "kind": "avoid",
            "expected": [
              "immune_response_tumor_state"
            ]
          },
          {
            "name": "cf_cxcl9_cxcl10_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CXCL9_CXCL10",
            "kind": "avoid",
            "expected": [
              "stress_hypoxia_tumor_state"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-4-codex",
      "modelName": "gpt-5.4",
      "harness": "openai-codex",
      "task": "10x_5p_melanoma_cluster_tumor_state_causation_v5",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 1204,
      "cost": 2.2088615,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "enriched_tumor_states",
            "kind": "list",
            "matchKey": "tumor_state",
            "k": 3,
            "expected": [
              {
                "key": "immune_response",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "direction_unresolved_cross_sectional"
                    ],
                    "canonical": "direction_unresolved_cross_sectional"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "IFN_response_axis",
                      "broad_antigen_presentation_MHC"
                    ],
                    "canonical": "IFN_response_axis"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "enrichment_diminished_by_decontamination",
                      "threshold_sensitive"
                    ],
                    "canonical": "enrichment_diminished_by_decontamination"
                  }
                }
              },
              {
                "key": "stress_hypoxia",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "data_supports_cell_intrinsic_cause"
                    ],
                    "canonical": "data_supports_cell_intrinsic_cause"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "HIF_hypoxia_program_specific"
                    ],
                    "canonical": "HIF_hypoxia_program_specific"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "robust_across_methods",
                      "robust_after_decontamination"
                    ],
                    "canonical": "robust_across_methods"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CXCL9_CXCL10": {
                "op": "in",
                "value": [
                  "immune_response_tumor_state"
                ]
              },
              "CCL5": {
                "op": "in",
                "value": [
                  "shared_across_tumor_states",
                  "partner_lymphocyte_derived",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "gate_conjugate_gate_confounder",
            "role": "gate",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "partner_lymphocyte_capture"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "antigen_presentation_decomposition",
            "kind": "dict",
            "expected": {
              "MHC_class_I": {
                "op": "in",
                "value": [
                  "co_regulated_with_dominant_program",
                  "tumor_intrinsic_enriched_independent"
                ]
              },
              "MHC_class_II": {
                "op": "in",
                "value": [
                  "partner_cell_attributable",
                  "flat_not_enriched",
                  "method_sensitive_no_robust_call",
                  "co_regulated_with_dominant_program"
                ]
              }
            }
          },
          {
            "name": "cf_non_robust_state_claimed_enriched",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "transitory_melanocytic",
              "intermediate_melanocytic",
              "melanocytic",
              "MITF_high",
              "neural_crest_like",
              "mitotic",
              "stress_p53",
              "undifferentiated",
              "dedifferentiated",
              "mesenchymal_tumor",
              "AXL_high_tumor",
              "invasive_tumor",
              "senescence_like",
              "EMT_transition",
              "cancer_stem_cell_like",
              "oxidative_stress_tumor"
            ]
          },
          {
            "name": "cf_framework_generic_or_offtissue_substitution",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "Tsoi_undifferentiated",
              "Tsoi_neural_crest_like",
              "Tsoi_transitory",
              "Tsoi_melanocytic",
              "Rambow_NCSC",
              "Wouters_mesenchymal",
              "Pozniak_TCF4_dependent",
              "tumor_cell",
              "melanoma_cell",
              "malignant_cell",
              "tumor_subpopulation",
              "keratinocyte",
              "hepatocyte",
              "lung_adenocarcinoma_cell",
              "colon_adenocarcinoma_cell",
              "Tirosh_cell_cycle_program",
              "Jerby_Arnon_program",
              "glioblastoma_cell",
              "breast_carcinoma_cell"
            ]
          },
          {
            "name": "cf_immune_response_direction_committed",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_immune_robustness_recall",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_ambiguous_hedge",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].causation_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_stress_generic_stress_response_evidence",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_conjugate_confounder_contamination_denied",
            "role": "hard_fail",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected"
            ]
          },
          {
            "name": "cf_ccl5_recall_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CCL5",
            "kind": "avoid",
            "expected": [
              "immune_response_tumor_state"
            ]
          },
          {
            "name": "cf_cxcl9_cxcl10_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CXCL9_CXCL10",
            "kind": "avoid",
            "expected": [
              "stress_hypoxia_tumor_state"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-4-codex",
      "modelName": "gpt-5.4",
      "harness": "openai-codex",
      "task": "10x_5p_melanoma_cluster_tumor_state_causation_v5",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 1341,
      "cost": 2.650178,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "enriched_tumor_states",
            "kind": "list",
            "matchKey": "tumor_state",
            "k": 3,
            "expected": [
              {
                "key": "immune_response",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "direction_unresolved_cross_sectional"
                    ],
                    "canonical": "direction_unresolved_cross_sectional"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "IFN_response_axis",
                      "broad_antigen_presentation_MHC"
                    ],
                    "canonical": "IFN_response_axis"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "enrichment_diminished_by_decontamination",
                      "threshold_sensitive"
                    ],
                    "canonical": "enrichment_diminished_by_decontamination"
                  }
                }
              },
              {
                "key": "stress_hypoxia",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "data_supports_cell_intrinsic_cause"
                    ],
                    "canonical": "data_supports_cell_intrinsic_cause"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "HIF_hypoxia_program_specific"
                    ],
                    "canonical": "HIF_hypoxia_program_specific"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "robust_across_methods",
                      "robust_after_decontamination"
                    ],
                    "canonical": "robust_across_methods"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CXCL9_CXCL10": {
                "op": "in",
                "value": [
                  "immune_response_tumor_state"
                ]
              },
              "CCL5": {
                "op": "in",
                "value": [
                  "shared_across_tumor_states",
                  "partner_lymphocyte_derived",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "gate_conjugate_gate_confounder",
            "role": "gate",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "partner_lymphocyte_capture"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "antigen_presentation_decomposition",
            "kind": "dict",
            "expected": {
              "MHC_class_I": {
                "op": "in",
                "value": [
                  "co_regulated_with_dominant_program",
                  "tumor_intrinsic_enriched_independent"
                ]
              },
              "MHC_class_II": {
                "op": "in",
                "value": [
                  "partner_cell_attributable",
                  "flat_not_enriched",
                  "method_sensitive_no_robust_call",
                  "co_regulated_with_dominant_program"
                ]
              }
            }
          },
          {
            "name": "cf_non_robust_state_claimed_enriched",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "transitory_melanocytic",
              "intermediate_melanocytic",
              "melanocytic",
              "MITF_high",
              "neural_crest_like",
              "mitotic",
              "stress_p53",
              "undifferentiated",
              "dedifferentiated",
              "mesenchymal_tumor",
              "AXL_high_tumor",
              "invasive_tumor",
              "senescence_like",
              "EMT_transition",
              "cancer_stem_cell_like",
              "oxidative_stress_tumor"
            ]
          },
          {
            "name": "cf_framework_generic_or_offtissue_substitution",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "Tsoi_undifferentiated",
              "Tsoi_neural_crest_like",
              "Tsoi_transitory",
              "Tsoi_melanocytic",
              "Rambow_NCSC",
              "Wouters_mesenchymal",
              "Pozniak_TCF4_dependent",
              "tumor_cell",
              "melanoma_cell",
              "malignant_cell",
              "tumor_subpopulation",
              "keratinocyte",
              "hepatocyte",
              "lung_adenocarcinoma_cell",
              "colon_adenocarcinoma_cell",
              "Tirosh_cell_cycle_program",
              "Jerby_Arnon_program",
              "glioblastoma_cell",
              "breast_carcinoma_cell"
            ]
          },
          {
            "name": "cf_immune_response_direction_committed",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_immune_robustness_recall",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_ambiguous_hedge",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].causation_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_stress_generic_stress_response_evidence",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_conjugate_confounder_contamination_denied",
            "role": "hard_fail",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected"
            ]
          },
          {
            "name": "cf_ccl5_recall_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CCL5",
            "kind": "avoid",
            "expected": [
              "immune_response_tumor_state"
            ]
          },
          {
            "name": "cf_cxcl9_cxcl10_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CXCL9_CXCL10",
            "kind": "avoid",
            "expected": [
              "stress_hypoxia_tumor_state"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-4-codex",
      "modelName": "gpt-5.4",
      "harness": "openai-codex",
      "task": "10x_5p_melanoma_cluster_tumor_state_causation_v5",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 849,
      "cost": 2.2990915000000003,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "enriched_tumor_states",
            "kind": "list",
            "matchKey": "tumor_state",
            "k": 3,
            "expected": [
              {
                "key": "immune_response",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "direction_unresolved_cross_sectional"
                    ],
                    "canonical": "direction_unresolved_cross_sectional"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "IFN_response_axis",
                      "broad_antigen_presentation_MHC"
                    ],
                    "canonical": "IFN_response_axis"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "enrichment_diminished_by_decontamination",
                      "threshold_sensitive"
                    ],
                    "canonical": "enrichment_diminished_by_decontamination"
                  }
                }
              },
              {
                "key": "stress_hypoxia",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "data_supports_cell_intrinsic_cause"
                    ],
                    "canonical": "data_supports_cell_intrinsic_cause"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "HIF_hypoxia_program_specific"
                    ],
                    "canonical": "HIF_hypoxia_program_specific"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "robust_across_methods",
                      "robust_after_decontamination"
                    ],
                    "canonical": "robust_across_methods"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CXCL9_CXCL10": {
                "op": "in",
                "value": [
                  "immune_response_tumor_state"
                ]
              },
              "CCL5": {
                "op": "in",
                "value": [
                  "shared_across_tumor_states",
                  "partner_lymphocyte_derived",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "gate_conjugate_gate_confounder",
            "role": "gate",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "partner_lymphocyte_capture"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "antigen_presentation_decomposition",
            "kind": "dict",
            "expected": {
              "MHC_class_I": {
                "op": "in",
                "value": [
                  "co_regulated_with_dominant_program",
                  "tumor_intrinsic_enriched_independent"
                ]
              },
              "MHC_class_II": {
                "op": "in",
                "value": [
                  "partner_cell_attributable",
                  "flat_not_enriched",
                  "method_sensitive_no_robust_call",
                  "co_regulated_with_dominant_program"
                ]
              }
            }
          },
          {
            "name": "cf_non_robust_state_claimed_enriched",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "transitory_melanocytic",
              "intermediate_melanocytic",
              "melanocytic",
              "MITF_high",
              "neural_crest_like",
              "mitotic",
              "stress_p53",
              "undifferentiated",
              "dedifferentiated",
              "mesenchymal_tumor",
              "AXL_high_tumor",
              "invasive_tumor",
              "senescence_like",
              "EMT_transition",
              "cancer_stem_cell_like",
              "oxidative_stress_tumor"
            ]
          },
          {
            "name": "cf_framework_generic_or_offtissue_substitution",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "Tsoi_undifferentiated",
              "Tsoi_neural_crest_like",
              "Tsoi_transitory",
              "Tsoi_melanocytic",
              "Rambow_NCSC",
              "Wouters_mesenchymal",
              "Pozniak_TCF4_dependent",
              "tumor_cell",
              "melanoma_cell",
              "malignant_cell",
              "tumor_subpopulation",
              "keratinocyte",
              "hepatocyte",
              "lung_adenocarcinoma_cell",
              "colon_adenocarcinoma_cell",
              "Tirosh_cell_cycle_program",
              "Jerby_Arnon_program",
              "glioblastoma_cell",
              "breast_carcinoma_cell"
            ]
          },
          {
            "name": "cf_immune_response_direction_committed",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_immune_robustness_recall",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_ambiguous_hedge",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].causation_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_stress_generic_stress_response_evidence",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_conjugate_confounder_contamination_denied",
            "role": "hard_fail",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected"
            ]
          },
          {
            "name": "cf_ccl5_recall_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CCL5",
            "kind": "avoid",
            "expected": [
              "immune_response_tumor_state"
            ]
          },
          {
            "name": "cf_cxcl9_cxcl10_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CXCL9_CXCL10",
            "kind": "avoid",
            "expected": [
              "stress_hypoxia_tumor_state"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-4-pi",
      "modelName": "gpt-5.4",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_tumor_state_causation_v5",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 3283,
      "cost": 1.5254645,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "enriched_tumor_states",
            "kind": "list",
            "matchKey": "tumor_state",
            "k": 3,
            "expected": [
              {
                "key": "immune_response",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "direction_unresolved_cross_sectional"
                    ],
                    "canonical": "direction_unresolved_cross_sectional"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "IFN_response_axis",
                      "broad_antigen_presentation_MHC"
                    ],
                    "canonical": "IFN_response_axis"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "enrichment_diminished_by_decontamination",
                      "threshold_sensitive"
                    ],
                    "canonical": "enrichment_diminished_by_decontamination"
                  }
                }
              },
              {
                "key": "stress_hypoxia",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "data_supports_cell_intrinsic_cause"
                    ],
                    "canonical": "data_supports_cell_intrinsic_cause"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "HIF_hypoxia_program_specific"
                    ],
                    "canonical": "HIF_hypoxia_program_specific"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "robust_across_methods",
                      "robust_after_decontamination"
                    ],
                    "canonical": "robust_across_methods"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CXCL9_CXCL10": {
                "op": "in",
                "value": [
                  "immune_response_tumor_state"
                ]
              },
              "CCL5": {
                "op": "in",
                "value": [
                  "shared_across_tumor_states",
                  "partner_lymphocyte_derived",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "gate_conjugate_gate_confounder",
            "role": "gate",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "partner_lymphocyte_capture"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "antigen_presentation_decomposition",
            "kind": "dict",
            "expected": {
              "MHC_class_I": {
                "op": "in",
                "value": [
                  "co_regulated_with_dominant_program",
                  "tumor_intrinsic_enriched_independent"
                ]
              },
              "MHC_class_II": {
                "op": "in",
                "value": [
                  "partner_cell_attributable",
                  "flat_not_enriched",
                  "method_sensitive_no_robust_call",
                  "co_regulated_with_dominant_program"
                ]
              }
            }
          },
          {
            "name": "cf_non_robust_state_claimed_enriched",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "transitory_melanocytic",
              "intermediate_melanocytic",
              "melanocytic",
              "MITF_high",
              "neural_crest_like",
              "mitotic",
              "stress_p53",
              "undifferentiated",
              "dedifferentiated",
              "mesenchymal_tumor",
              "AXL_high_tumor",
              "invasive_tumor",
              "senescence_like",
              "EMT_transition",
              "cancer_stem_cell_like",
              "oxidative_stress_tumor"
            ]
          },
          {
            "name": "cf_framework_generic_or_offtissue_substitution",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "Tsoi_undifferentiated",
              "Tsoi_neural_crest_like",
              "Tsoi_transitory",
              "Tsoi_melanocytic",
              "Rambow_NCSC",
              "Wouters_mesenchymal",
              "Pozniak_TCF4_dependent",
              "tumor_cell",
              "melanoma_cell",
              "malignant_cell",
              "tumor_subpopulation",
              "keratinocyte",
              "hepatocyte",
              "lung_adenocarcinoma_cell",
              "colon_adenocarcinoma_cell",
              "Tirosh_cell_cycle_program",
              "Jerby_Arnon_program",
              "glioblastoma_cell",
              "breast_carcinoma_cell"
            ]
          },
          {
            "name": "cf_immune_response_direction_committed",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_immune_robustness_recall",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_ambiguous_hedge",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].causation_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_stress_generic_stress_response_evidence",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_conjugate_confounder_contamination_denied",
            "role": "hard_fail",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected"
            ]
          },
          {
            "name": "cf_ccl5_recall_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CCL5",
            "kind": "avoid",
            "expected": [
              "immune_response_tumor_state"
            ]
          },
          {
            "name": "cf_cxcl9_cxcl10_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CXCL9_CXCL10",
            "kind": "avoid",
            "expected": [
              "stress_hypoxia_tumor_state"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-4-pi",
      "modelName": "gpt-5.4",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_tumor_state_causation_v5",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 4052,
      "cost": 1.1826955,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "enriched_tumor_states",
            "kind": "list",
            "matchKey": "tumor_state",
            "k": 3,
            "expected": [
              {
                "key": "immune_response",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "direction_unresolved_cross_sectional"
                    ],
                    "canonical": "direction_unresolved_cross_sectional"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "IFN_response_axis",
                      "broad_antigen_presentation_MHC"
                    ],
                    "canonical": "IFN_response_axis"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "enrichment_diminished_by_decontamination",
                      "threshold_sensitive"
                    ],
                    "canonical": "enrichment_diminished_by_decontamination"
                  }
                }
              },
              {
                "key": "stress_hypoxia",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "data_supports_cell_intrinsic_cause"
                    ],
                    "canonical": "data_supports_cell_intrinsic_cause"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "HIF_hypoxia_program_specific"
                    ],
                    "canonical": "HIF_hypoxia_program_specific"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "robust_across_methods",
                      "robust_after_decontamination"
                    ],
                    "canonical": "robust_across_methods"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CXCL9_CXCL10": {
                "op": "in",
                "value": [
                  "immune_response_tumor_state"
                ]
              },
              "CCL5": {
                "op": "in",
                "value": [
                  "shared_across_tumor_states",
                  "partner_lymphocyte_derived",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "gate_conjugate_gate_confounder",
            "role": "gate",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "partner_lymphocyte_capture"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "antigen_presentation_decomposition",
            "kind": "dict",
            "expected": {
              "MHC_class_I": {
                "op": "in",
                "value": [
                  "co_regulated_with_dominant_program",
                  "tumor_intrinsic_enriched_independent"
                ]
              },
              "MHC_class_II": {
                "op": "in",
                "value": [
                  "partner_cell_attributable",
                  "flat_not_enriched",
                  "method_sensitive_no_robust_call",
                  "co_regulated_with_dominant_program"
                ]
              }
            }
          },
          {
            "name": "cf_non_robust_state_claimed_enriched",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "transitory_melanocytic",
              "intermediate_melanocytic",
              "melanocytic",
              "MITF_high",
              "neural_crest_like",
              "mitotic",
              "stress_p53",
              "undifferentiated",
              "dedifferentiated",
              "mesenchymal_tumor",
              "AXL_high_tumor",
              "invasive_tumor",
              "senescence_like",
              "EMT_transition",
              "cancer_stem_cell_like",
              "oxidative_stress_tumor"
            ]
          },
          {
            "name": "cf_framework_generic_or_offtissue_substitution",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "Tsoi_undifferentiated",
              "Tsoi_neural_crest_like",
              "Tsoi_transitory",
              "Tsoi_melanocytic",
              "Rambow_NCSC",
              "Wouters_mesenchymal",
              "Pozniak_TCF4_dependent",
              "tumor_cell",
              "melanoma_cell",
              "malignant_cell",
              "tumor_subpopulation",
              "keratinocyte",
              "hepatocyte",
              "lung_adenocarcinoma_cell",
              "colon_adenocarcinoma_cell",
              "Tirosh_cell_cycle_program",
              "Jerby_Arnon_program",
              "glioblastoma_cell",
              "breast_carcinoma_cell"
            ]
          },
          {
            "name": "cf_immune_response_direction_committed",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_immune_robustness_recall",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_ambiguous_hedge",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].causation_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_stress_generic_stress_response_evidence",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_conjugate_confounder_contamination_denied",
            "role": "hard_fail",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected"
            ]
          },
          {
            "name": "cf_ccl5_recall_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CCL5",
            "kind": "avoid",
            "expected": [
              "immune_response_tumor_state"
            ]
          },
          {
            "name": "cf_cxcl9_cxcl10_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CXCL9_CXCL10",
            "kind": "avoid",
            "expected": [
              "stress_hypoxia_tumor_state"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-4-pi",
      "modelName": "gpt-5.4",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_tumor_state_causation_v5",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 2078,
      "cost": 1.6036540000000001,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "enriched_tumor_states",
            "kind": "list",
            "matchKey": "tumor_state",
            "k": 3,
            "expected": [
              {
                "key": "immune_response",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "direction_unresolved_cross_sectional"
                    ],
                    "canonical": "direction_unresolved_cross_sectional"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "IFN_response_axis",
                      "broad_antigen_presentation_MHC"
                    ],
                    "canonical": "IFN_response_axis"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "enrichment_diminished_by_decontamination",
                      "threshold_sensitive"
                    ],
                    "canonical": "enrichment_diminished_by_decontamination"
                  }
                }
              },
              {
                "key": "stress_hypoxia",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "data_supports_cell_intrinsic_cause"
                    ],
                    "canonical": "data_supports_cell_intrinsic_cause"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "HIF_hypoxia_program_specific"
                    ],
                    "canonical": "HIF_hypoxia_program_specific"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "robust_across_methods",
                      "robust_after_decontamination"
                    ],
                    "canonical": "robust_across_methods"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CXCL9_CXCL10": {
                "op": "in",
                "value": [
                  "immune_response_tumor_state"
                ]
              },
              "CCL5": {
                "op": "in",
                "value": [
                  "shared_across_tumor_states",
                  "partner_lymphocyte_derived",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "gate_conjugate_gate_confounder",
            "role": "gate",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "partner_lymphocyte_capture"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "antigen_presentation_decomposition",
            "kind": "dict",
            "expected": {
              "MHC_class_I": {
                "op": "in",
                "value": [
                  "co_regulated_with_dominant_program",
                  "tumor_intrinsic_enriched_independent"
                ]
              },
              "MHC_class_II": {
                "op": "in",
                "value": [
                  "partner_cell_attributable",
                  "flat_not_enriched",
                  "method_sensitive_no_robust_call",
                  "co_regulated_with_dominant_program"
                ]
              }
            }
          },
          {
            "name": "cf_non_robust_state_claimed_enriched",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "transitory_melanocytic",
              "intermediate_melanocytic",
              "melanocytic",
              "MITF_high",
              "neural_crest_like",
              "mitotic",
              "stress_p53",
              "undifferentiated",
              "dedifferentiated",
              "mesenchymal_tumor",
              "AXL_high_tumor",
              "invasive_tumor",
              "senescence_like",
              "EMT_transition",
              "cancer_stem_cell_like",
              "oxidative_stress_tumor"
            ]
          },
          {
            "name": "cf_framework_generic_or_offtissue_substitution",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "Tsoi_undifferentiated",
              "Tsoi_neural_crest_like",
              "Tsoi_transitory",
              "Tsoi_melanocytic",
              "Rambow_NCSC",
              "Wouters_mesenchymal",
              "Pozniak_TCF4_dependent",
              "tumor_cell",
              "melanoma_cell",
              "malignant_cell",
              "tumor_subpopulation",
              "keratinocyte",
              "hepatocyte",
              "lung_adenocarcinoma_cell",
              "colon_adenocarcinoma_cell",
              "Tirosh_cell_cycle_program",
              "Jerby_Arnon_program",
              "glioblastoma_cell",
              "breast_carcinoma_cell"
            ]
          },
          {
            "name": "cf_immune_response_direction_committed",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_immune_robustness_recall",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_ambiguous_hedge",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].causation_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_stress_generic_stress_response_evidence",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_conjugate_confounder_contamination_denied",
            "role": "hard_fail",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected"
            ]
          },
          {
            "name": "cf_ccl5_recall_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CCL5",
            "kind": "avoid",
            "expected": [
              "immune_response_tumor_state"
            ]
          },
          {
            "name": "cf_cxcl9_cxcl10_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CXCL9_CXCL10",
            "kind": "avoid",
            "expected": [
              "stress_hypoxia_tumor_state"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-5-codex",
      "modelName": "gpt-5.5",
      "harness": "openai-codex",
      "task": "10x_5p_melanoma_cluster_tumor_state_causation_v5",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 882,
      "cost": 3.507356,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "enriched_tumor_states",
            "kind": "list",
            "matchKey": "tumor_state",
            "k": 3,
            "expected": [
              {
                "key": "immune_response",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "direction_unresolved_cross_sectional"
                    ],
                    "canonical": "direction_unresolved_cross_sectional"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "IFN_response_axis",
                      "broad_antigen_presentation_MHC"
                    ],
                    "canonical": "IFN_response_axis"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "enrichment_diminished_by_decontamination",
                      "threshold_sensitive"
                    ],
                    "canonical": "enrichment_diminished_by_decontamination"
                  }
                }
              },
              {
                "key": "stress_hypoxia",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "data_supports_cell_intrinsic_cause"
                    ],
                    "canonical": "data_supports_cell_intrinsic_cause"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "HIF_hypoxia_program_specific"
                    ],
                    "canonical": "HIF_hypoxia_program_specific"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "robust_across_methods",
                      "robust_after_decontamination"
                    ],
                    "canonical": "robust_across_methods"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CXCL9_CXCL10": {
                "op": "in",
                "value": [
                  "immune_response_tumor_state"
                ]
              },
              "CCL5": {
                "op": "in",
                "value": [
                  "shared_across_tumor_states",
                  "partner_lymphocyte_derived",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "gate_conjugate_gate_confounder",
            "role": "gate",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "partner_lymphocyte_capture"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "antigen_presentation_decomposition",
            "kind": "dict",
            "expected": {
              "MHC_class_I": {
                "op": "in",
                "value": [
                  "co_regulated_with_dominant_program",
                  "tumor_intrinsic_enriched_independent"
                ]
              },
              "MHC_class_II": {
                "op": "in",
                "value": [
                  "partner_cell_attributable",
                  "flat_not_enriched",
                  "method_sensitive_no_robust_call",
                  "co_regulated_with_dominant_program"
                ]
              }
            }
          },
          {
            "name": "cf_non_robust_state_claimed_enriched",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "transitory_melanocytic",
              "intermediate_melanocytic",
              "melanocytic",
              "MITF_high",
              "neural_crest_like",
              "mitotic",
              "stress_p53",
              "undifferentiated",
              "dedifferentiated",
              "mesenchymal_tumor",
              "AXL_high_tumor",
              "invasive_tumor",
              "senescence_like",
              "EMT_transition",
              "cancer_stem_cell_like",
              "oxidative_stress_tumor"
            ]
          },
          {
            "name": "cf_framework_generic_or_offtissue_substitution",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "Tsoi_undifferentiated",
              "Tsoi_neural_crest_like",
              "Tsoi_transitory",
              "Tsoi_melanocytic",
              "Rambow_NCSC",
              "Wouters_mesenchymal",
              "Pozniak_TCF4_dependent",
              "tumor_cell",
              "melanoma_cell",
              "malignant_cell",
              "tumor_subpopulation",
              "keratinocyte",
              "hepatocyte",
              "lung_adenocarcinoma_cell",
              "colon_adenocarcinoma_cell",
              "Tirosh_cell_cycle_program",
              "Jerby_Arnon_program",
              "glioblastoma_cell",
              "breast_carcinoma_cell"
            ]
          },
          {
            "name": "cf_immune_response_direction_committed",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_immune_robustness_recall",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_ambiguous_hedge",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].causation_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_stress_generic_stress_response_evidence",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_conjugate_confounder_contamination_denied",
            "role": "hard_fail",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected"
            ]
          },
          {
            "name": "cf_ccl5_recall_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CCL5",
            "kind": "avoid",
            "expected": [
              "immune_response_tumor_state"
            ]
          },
          {
            "name": "cf_cxcl9_cxcl10_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CXCL9_CXCL10",
            "kind": "avoid",
            "expected": [
              "stress_hypoxia_tumor_state"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-5-codex",
      "modelName": "gpt-5.5",
      "harness": "openai-codex",
      "task": "10x_5p_melanoma_cluster_tumor_state_causation_v5",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 862,
      "cost": 2.766508,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "enriched_tumor_states",
            "kind": "list",
            "matchKey": "tumor_state",
            "k": 3,
            "expected": [
              {
                "key": "immune_response",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "direction_unresolved_cross_sectional"
                    ],
                    "canonical": "direction_unresolved_cross_sectional"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "IFN_response_axis",
                      "broad_antigen_presentation_MHC"
                    ],
                    "canonical": "IFN_response_axis"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "enrichment_diminished_by_decontamination",
                      "threshold_sensitive"
                    ],
                    "canonical": "enrichment_diminished_by_decontamination"
                  }
                }
              },
              {
                "key": "stress_hypoxia",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "data_supports_cell_intrinsic_cause"
                    ],
                    "canonical": "data_supports_cell_intrinsic_cause"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "HIF_hypoxia_program_specific"
                    ],
                    "canonical": "HIF_hypoxia_program_specific"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "robust_across_methods",
                      "robust_after_decontamination"
                    ],
                    "canonical": "robust_across_methods"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CXCL9_CXCL10": {
                "op": "in",
                "value": [
                  "immune_response_tumor_state"
                ]
              },
              "CCL5": {
                "op": "in",
                "value": [
                  "shared_across_tumor_states",
                  "partner_lymphocyte_derived",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "gate_conjugate_gate_confounder",
            "role": "gate",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "partner_lymphocyte_capture"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "antigen_presentation_decomposition",
            "kind": "dict",
            "expected": {
              "MHC_class_I": {
                "op": "in",
                "value": [
                  "co_regulated_with_dominant_program",
                  "tumor_intrinsic_enriched_independent"
                ]
              },
              "MHC_class_II": {
                "op": "in",
                "value": [
                  "partner_cell_attributable",
                  "flat_not_enriched",
                  "method_sensitive_no_robust_call",
                  "co_regulated_with_dominant_program"
                ]
              }
            }
          },
          {
            "name": "cf_non_robust_state_claimed_enriched",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "transitory_melanocytic",
              "intermediate_melanocytic",
              "melanocytic",
              "MITF_high",
              "neural_crest_like",
              "mitotic",
              "stress_p53",
              "undifferentiated",
              "dedifferentiated",
              "mesenchymal_tumor",
              "AXL_high_tumor",
              "invasive_tumor",
              "senescence_like",
              "EMT_transition",
              "cancer_stem_cell_like",
              "oxidative_stress_tumor"
            ]
          },
          {
            "name": "cf_framework_generic_or_offtissue_substitution",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "Tsoi_undifferentiated",
              "Tsoi_neural_crest_like",
              "Tsoi_transitory",
              "Tsoi_melanocytic",
              "Rambow_NCSC",
              "Wouters_mesenchymal",
              "Pozniak_TCF4_dependent",
              "tumor_cell",
              "melanoma_cell",
              "malignant_cell",
              "tumor_subpopulation",
              "keratinocyte",
              "hepatocyte",
              "lung_adenocarcinoma_cell",
              "colon_adenocarcinoma_cell",
              "Tirosh_cell_cycle_program",
              "Jerby_Arnon_program",
              "glioblastoma_cell",
              "breast_carcinoma_cell"
            ]
          },
          {
            "name": "cf_immune_response_direction_committed",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_immune_robustness_recall",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_ambiguous_hedge",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].causation_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_stress_generic_stress_response_evidence",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_conjugate_confounder_contamination_denied",
            "role": "hard_fail",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected"
            ]
          },
          {
            "name": "cf_ccl5_recall_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CCL5",
            "kind": "avoid",
            "expected": [
              "immune_response_tumor_state"
            ]
          },
          {
            "name": "cf_cxcl9_cxcl10_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CXCL9_CXCL10",
            "kind": "avoid",
            "expected": [
              "stress_hypoxia_tumor_state"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-5-codex",
      "modelName": "gpt-5.5",
      "harness": "openai-codex",
      "task": "10x_5p_melanoma_cluster_tumor_state_causation_v5",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 471,
      "cost": 2.070474,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "enriched_tumor_states",
            "kind": "list",
            "matchKey": "tumor_state",
            "k": 3,
            "expected": [
              {
                "key": "immune_response",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "direction_unresolved_cross_sectional"
                    ],
                    "canonical": "direction_unresolved_cross_sectional"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "IFN_response_axis",
                      "broad_antigen_presentation_MHC"
                    ],
                    "canonical": "IFN_response_axis"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "enrichment_diminished_by_decontamination",
                      "threshold_sensitive"
                    ],
                    "canonical": "enrichment_diminished_by_decontamination"
                  }
                }
              },
              {
                "key": "stress_hypoxia",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "data_supports_cell_intrinsic_cause"
                    ],
                    "canonical": "data_supports_cell_intrinsic_cause"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "HIF_hypoxia_program_specific"
                    ],
                    "canonical": "HIF_hypoxia_program_specific"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "robust_across_methods",
                      "robust_after_decontamination"
                    ],
                    "canonical": "robust_across_methods"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CXCL9_CXCL10": {
                "op": "in",
                "value": [
                  "immune_response_tumor_state"
                ]
              },
              "CCL5": {
                "op": "in",
                "value": [
                  "shared_across_tumor_states",
                  "partner_lymphocyte_derived",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "gate_conjugate_gate_confounder",
            "role": "gate",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "partner_lymphocyte_capture"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "antigen_presentation_decomposition",
            "kind": "dict",
            "expected": {
              "MHC_class_I": {
                "op": "in",
                "value": [
                  "co_regulated_with_dominant_program",
                  "tumor_intrinsic_enriched_independent"
                ]
              },
              "MHC_class_II": {
                "op": "in",
                "value": [
                  "partner_cell_attributable",
                  "flat_not_enriched",
                  "method_sensitive_no_robust_call",
                  "co_regulated_with_dominant_program"
                ]
              }
            }
          },
          {
            "name": "cf_non_robust_state_claimed_enriched",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "transitory_melanocytic",
              "intermediate_melanocytic",
              "melanocytic",
              "MITF_high",
              "neural_crest_like",
              "mitotic",
              "stress_p53",
              "undifferentiated",
              "dedifferentiated",
              "mesenchymal_tumor",
              "AXL_high_tumor",
              "invasive_tumor",
              "senescence_like",
              "EMT_transition",
              "cancer_stem_cell_like",
              "oxidative_stress_tumor"
            ]
          },
          {
            "name": "cf_framework_generic_or_offtissue_substitution",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "Tsoi_undifferentiated",
              "Tsoi_neural_crest_like",
              "Tsoi_transitory",
              "Tsoi_melanocytic",
              "Rambow_NCSC",
              "Wouters_mesenchymal",
              "Pozniak_TCF4_dependent",
              "tumor_cell",
              "melanoma_cell",
              "malignant_cell",
              "tumor_subpopulation",
              "keratinocyte",
              "hepatocyte",
              "lung_adenocarcinoma_cell",
              "colon_adenocarcinoma_cell",
              "Tirosh_cell_cycle_program",
              "Jerby_Arnon_program",
              "glioblastoma_cell",
              "breast_carcinoma_cell"
            ]
          },
          {
            "name": "cf_immune_response_direction_committed",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_immune_robustness_recall",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_ambiguous_hedge",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].causation_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_stress_generic_stress_response_evidence",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_conjugate_confounder_contamination_denied",
            "role": "hard_fail",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected"
            ]
          },
          {
            "name": "cf_ccl5_recall_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CCL5",
            "kind": "avoid",
            "expected": [
              "immune_response_tumor_state"
            ]
          },
          {
            "name": "cf_cxcl9_cxcl10_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CXCL9_CXCL10",
            "kind": "avoid",
            "expected": [
              "stress_hypoxia_tumor_state"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-5-pi",
      "modelName": "gpt-5.5",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_tumor_state_causation_v5",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 855,
      "cost": 2.022617,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "enriched_tumor_states",
            "kind": "list",
            "matchKey": "tumor_state",
            "k": 3,
            "expected": [
              {
                "key": "immune_response",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "direction_unresolved_cross_sectional"
                    ],
                    "canonical": "direction_unresolved_cross_sectional"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "IFN_response_axis",
                      "broad_antigen_presentation_MHC"
                    ],
                    "canonical": "IFN_response_axis"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "enrichment_diminished_by_decontamination",
                      "threshold_sensitive"
                    ],
                    "canonical": "enrichment_diminished_by_decontamination"
                  }
                }
              },
              {
                "key": "stress_hypoxia",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "data_supports_cell_intrinsic_cause"
                    ],
                    "canonical": "data_supports_cell_intrinsic_cause"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "HIF_hypoxia_program_specific"
                    ],
                    "canonical": "HIF_hypoxia_program_specific"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "robust_across_methods",
                      "robust_after_decontamination"
                    ],
                    "canonical": "robust_across_methods"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CXCL9_CXCL10": {
                "op": "in",
                "value": [
                  "immune_response_tumor_state"
                ]
              },
              "CCL5": {
                "op": "in",
                "value": [
                  "shared_across_tumor_states",
                  "partner_lymphocyte_derived",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "gate_conjugate_gate_confounder",
            "role": "gate",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "partner_lymphocyte_capture"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "antigen_presentation_decomposition",
            "kind": "dict",
            "expected": {
              "MHC_class_I": {
                "op": "in",
                "value": [
                  "co_regulated_with_dominant_program",
                  "tumor_intrinsic_enriched_independent"
                ]
              },
              "MHC_class_II": {
                "op": "in",
                "value": [
                  "partner_cell_attributable",
                  "flat_not_enriched",
                  "method_sensitive_no_robust_call",
                  "co_regulated_with_dominant_program"
                ]
              }
            }
          },
          {
            "name": "cf_non_robust_state_claimed_enriched",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "transitory_melanocytic",
              "intermediate_melanocytic",
              "melanocytic",
              "MITF_high",
              "neural_crest_like",
              "mitotic",
              "stress_p53",
              "undifferentiated",
              "dedifferentiated",
              "mesenchymal_tumor",
              "AXL_high_tumor",
              "invasive_tumor",
              "senescence_like",
              "EMT_transition",
              "cancer_stem_cell_like",
              "oxidative_stress_tumor"
            ]
          },
          {
            "name": "cf_framework_generic_or_offtissue_substitution",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "Tsoi_undifferentiated",
              "Tsoi_neural_crest_like",
              "Tsoi_transitory",
              "Tsoi_melanocytic",
              "Rambow_NCSC",
              "Wouters_mesenchymal",
              "Pozniak_TCF4_dependent",
              "tumor_cell",
              "melanoma_cell",
              "malignant_cell",
              "tumor_subpopulation",
              "keratinocyte",
              "hepatocyte",
              "lung_adenocarcinoma_cell",
              "colon_adenocarcinoma_cell",
              "Tirosh_cell_cycle_program",
              "Jerby_Arnon_program",
              "glioblastoma_cell",
              "breast_carcinoma_cell"
            ]
          },
          {
            "name": "cf_immune_response_direction_committed",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_immune_robustness_recall",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_ambiguous_hedge",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].causation_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_stress_generic_stress_response_evidence",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_conjugate_confounder_contamination_denied",
            "role": "hard_fail",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected"
            ]
          },
          {
            "name": "cf_ccl5_recall_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CCL5",
            "kind": "avoid",
            "expected": [
              "immune_response_tumor_state"
            ]
          },
          {
            "name": "cf_cxcl9_cxcl10_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CXCL9_CXCL10",
            "kind": "avoid",
            "expected": [
              "stress_hypoxia_tumor_state"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-5-pi",
      "modelName": "gpt-5.5",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_tumor_state_causation_v5",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 1253,
      "cost": 2.179504,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "enriched_tumor_states",
            "kind": "list",
            "matchKey": "tumor_state",
            "k": 3,
            "expected": [
              {
                "key": "immune_response",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "direction_unresolved_cross_sectional"
                    ],
                    "canonical": "direction_unresolved_cross_sectional"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "IFN_response_axis",
                      "broad_antigen_presentation_MHC"
                    ],
                    "canonical": "IFN_response_axis"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "enrichment_diminished_by_decontamination",
                      "threshold_sensitive"
                    ],
                    "canonical": "enrichment_diminished_by_decontamination"
                  }
                }
              },
              {
                "key": "stress_hypoxia",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "data_supports_cell_intrinsic_cause"
                    ],
                    "canonical": "data_supports_cell_intrinsic_cause"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "HIF_hypoxia_program_specific"
                    ],
                    "canonical": "HIF_hypoxia_program_specific"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "robust_across_methods",
                      "robust_after_decontamination"
                    ],
                    "canonical": "robust_across_methods"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CXCL9_CXCL10": {
                "op": "in",
                "value": [
                  "immune_response_tumor_state"
                ]
              },
              "CCL5": {
                "op": "in",
                "value": [
                  "shared_across_tumor_states",
                  "partner_lymphocyte_derived",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "gate_conjugate_gate_confounder",
            "role": "gate",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "partner_lymphocyte_capture"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "antigen_presentation_decomposition",
            "kind": "dict",
            "expected": {
              "MHC_class_I": {
                "op": "in",
                "value": [
                  "co_regulated_with_dominant_program",
                  "tumor_intrinsic_enriched_independent"
                ]
              },
              "MHC_class_II": {
                "op": "in",
                "value": [
                  "partner_cell_attributable",
                  "flat_not_enriched",
                  "method_sensitive_no_robust_call",
                  "co_regulated_with_dominant_program"
                ]
              }
            }
          },
          {
            "name": "cf_non_robust_state_claimed_enriched",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "transitory_melanocytic",
              "intermediate_melanocytic",
              "melanocytic",
              "MITF_high",
              "neural_crest_like",
              "mitotic",
              "stress_p53",
              "undifferentiated",
              "dedifferentiated",
              "mesenchymal_tumor",
              "AXL_high_tumor",
              "invasive_tumor",
              "senescence_like",
              "EMT_transition",
              "cancer_stem_cell_like",
              "oxidative_stress_tumor"
            ]
          },
          {
            "name": "cf_framework_generic_or_offtissue_substitution",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "Tsoi_undifferentiated",
              "Tsoi_neural_crest_like",
              "Tsoi_transitory",
              "Tsoi_melanocytic",
              "Rambow_NCSC",
              "Wouters_mesenchymal",
              "Pozniak_TCF4_dependent",
              "tumor_cell",
              "melanoma_cell",
              "malignant_cell",
              "tumor_subpopulation",
              "keratinocyte",
              "hepatocyte",
              "lung_adenocarcinoma_cell",
              "colon_adenocarcinoma_cell",
              "Tirosh_cell_cycle_program",
              "Jerby_Arnon_program",
              "glioblastoma_cell",
              "breast_carcinoma_cell"
            ]
          },
          {
            "name": "cf_immune_response_direction_committed",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_immune_robustness_recall",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_ambiguous_hedge",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].causation_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_stress_generic_stress_response_evidence",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_conjugate_confounder_contamination_denied",
            "role": "hard_fail",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected"
            ]
          },
          {
            "name": "cf_ccl5_recall_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CCL5",
            "kind": "avoid",
            "expected": [
              "immune_response_tumor_state"
            ]
          },
          {
            "name": "cf_cxcl9_cxcl10_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CXCL9_CXCL10",
            "kind": "avoid",
            "expected": [
              "stress_hypoxia_tumor_state"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-5-pi",
      "modelName": "gpt-5.5",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_tumor_state_causation_v5",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1540,
      "cost": 1.6685139999999998,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "enriched_tumor_states",
            "kind": "list",
            "matchKey": "tumor_state",
            "k": 3,
            "expected": [
              {
                "key": "immune_response",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "direction_unresolved_cross_sectional"
                    ],
                    "canonical": "direction_unresolved_cross_sectional"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "IFN_response_axis",
                      "broad_antigen_presentation_MHC"
                    ],
                    "canonical": "IFN_response_axis"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "enrichment_diminished_by_decontamination",
                      "threshold_sensitive"
                    ],
                    "canonical": "enrichment_diminished_by_decontamination"
                  }
                }
              },
              {
                "key": "stress_hypoxia",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "data_supports_cell_intrinsic_cause"
                    ],
                    "canonical": "data_supports_cell_intrinsic_cause"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "HIF_hypoxia_program_specific"
                    ],
                    "canonical": "HIF_hypoxia_program_specific"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "robust_across_methods",
                      "robust_after_decontamination"
                    ],
                    "canonical": "robust_across_methods"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CXCL9_CXCL10": {
                "op": "in",
                "value": [
                  "immune_response_tumor_state"
                ]
              },
              "CCL5": {
                "op": "in",
                "value": [
                  "shared_across_tumor_states",
                  "partner_lymphocyte_derived",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "gate_conjugate_gate_confounder",
            "role": "gate",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "partner_lymphocyte_capture"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "antigen_presentation_decomposition",
            "kind": "dict",
            "expected": {
              "MHC_class_I": {
                "op": "in",
                "value": [
                  "co_regulated_with_dominant_program",
                  "tumor_intrinsic_enriched_independent"
                ]
              },
              "MHC_class_II": {
                "op": "in",
                "value": [
                  "partner_cell_attributable",
                  "flat_not_enriched",
                  "method_sensitive_no_robust_call",
                  "co_regulated_with_dominant_program"
                ]
              }
            }
          },
          {
            "name": "cf_non_robust_state_claimed_enriched",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "transitory_melanocytic",
              "intermediate_melanocytic",
              "melanocytic",
              "MITF_high",
              "neural_crest_like",
              "mitotic",
              "stress_p53",
              "undifferentiated",
              "dedifferentiated",
              "mesenchymal_tumor",
              "AXL_high_tumor",
              "invasive_tumor",
              "senescence_like",
              "EMT_transition",
              "cancer_stem_cell_like",
              "oxidative_stress_tumor"
            ]
          },
          {
            "name": "cf_framework_generic_or_offtissue_substitution",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "Tsoi_undifferentiated",
              "Tsoi_neural_crest_like",
              "Tsoi_transitory",
              "Tsoi_melanocytic",
              "Rambow_NCSC",
              "Wouters_mesenchymal",
              "Pozniak_TCF4_dependent",
              "tumor_cell",
              "melanoma_cell",
              "malignant_cell",
              "tumor_subpopulation",
              "keratinocyte",
              "hepatocyte",
              "lung_adenocarcinoma_cell",
              "colon_adenocarcinoma_cell",
              "Tirosh_cell_cycle_program",
              "Jerby_Arnon_program",
              "glioblastoma_cell",
              "breast_carcinoma_cell"
            ]
          },
          {
            "name": "cf_immune_response_direction_committed",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_immune_robustness_recall",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_ambiguous_hedge",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].causation_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_stress_generic_stress_response_evidence",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_conjugate_confounder_contamination_denied",
            "role": "hard_fail",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected"
            ]
          },
          {
            "name": "cf_ccl5_recall_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CCL5",
            "kind": "avoid",
            "expected": [
              "immune_response_tumor_state"
            ]
          },
          {
            "name": "cf_cxcl9_cxcl10_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CXCL9_CXCL10",
            "kind": "avoid",
            "expected": [
              "stress_hypoxia_tumor_state"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-20-beta-reasoning-pi",
      "modelName": "grok-4.20-beta-0309-reasoning",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_tumor_state_causation_v5",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 597,
      "cost": 0.0364504,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "enriched_tumor_states",
            "kind": "list",
            "matchKey": "tumor_state",
            "k": 3,
            "expected": [
              {
                "key": "immune_response",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "direction_unresolved_cross_sectional"
                    ],
                    "canonical": "direction_unresolved_cross_sectional"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "IFN_response_axis",
                      "broad_antigen_presentation_MHC"
                    ],
                    "canonical": "IFN_response_axis"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "enrichment_diminished_by_decontamination",
                      "threshold_sensitive"
                    ],
                    "canonical": "enrichment_diminished_by_decontamination"
                  }
                }
              },
              {
                "key": "stress_hypoxia",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "data_supports_cell_intrinsic_cause"
                    ],
                    "canonical": "data_supports_cell_intrinsic_cause"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "HIF_hypoxia_program_specific"
                    ],
                    "canonical": "HIF_hypoxia_program_specific"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "robust_across_methods",
                      "robust_after_decontamination"
                    ],
                    "canonical": "robust_across_methods"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CXCL9_CXCL10": {
                "op": "in",
                "value": [
                  "immune_response_tumor_state"
                ]
              },
              "CCL5": {
                "op": "in",
                "value": [
                  "shared_across_tumor_states",
                  "partner_lymphocyte_derived",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "gate_conjugate_gate_confounder",
            "role": "gate",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "partner_lymphocyte_capture"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "antigen_presentation_decomposition",
            "kind": "dict",
            "expected": {
              "MHC_class_I": {
                "op": "in",
                "value": [
                  "co_regulated_with_dominant_program",
                  "tumor_intrinsic_enriched_independent"
                ]
              },
              "MHC_class_II": {
                "op": "in",
                "value": [
                  "partner_cell_attributable",
                  "flat_not_enriched",
                  "method_sensitive_no_robust_call",
                  "co_regulated_with_dominant_program"
                ]
              }
            }
          },
          {
            "name": "cf_non_robust_state_claimed_enriched",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "transitory_melanocytic",
              "intermediate_melanocytic",
              "melanocytic",
              "MITF_high",
              "neural_crest_like",
              "mitotic",
              "stress_p53",
              "undifferentiated",
              "dedifferentiated",
              "mesenchymal_tumor",
              "AXL_high_tumor",
              "invasive_tumor",
              "senescence_like",
              "EMT_transition",
              "cancer_stem_cell_like",
              "oxidative_stress_tumor"
            ]
          },
          {
            "name": "cf_framework_generic_or_offtissue_substitution",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "Tsoi_undifferentiated",
              "Tsoi_neural_crest_like",
              "Tsoi_transitory",
              "Tsoi_melanocytic",
              "Rambow_NCSC",
              "Wouters_mesenchymal",
              "Pozniak_TCF4_dependent",
              "tumor_cell",
              "melanoma_cell",
              "malignant_cell",
              "tumor_subpopulation",
              "keratinocyte",
              "hepatocyte",
              "lung_adenocarcinoma_cell",
              "colon_adenocarcinoma_cell",
              "Tirosh_cell_cycle_program",
              "Jerby_Arnon_program",
              "glioblastoma_cell",
              "breast_carcinoma_cell"
            ]
          },
          {
            "name": "cf_immune_response_direction_committed",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_immune_robustness_recall",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_ambiguous_hedge",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].causation_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_stress_generic_stress_response_evidence",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_conjugate_confounder_contamination_denied",
            "role": "hard_fail",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected"
            ]
          },
          {
            "name": "cf_ccl5_recall_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CCL5",
            "kind": "avoid",
            "expected": [
              "immune_response_tumor_state"
            ]
          },
          {
            "name": "cf_cxcl9_cxcl10_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CXCL9_CXCL10",
            "kind": "avoid",
            "expected": [
              "stress_hypoxia_tumor_state"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-20-beta-reasoning-pi",
      "modelName": "grok-4.20-beta-0309-reasoning",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_tumor_state_causation_v5",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 573,
      "cost": 0.09899104999999998,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "enriched_tumor_states",
            "kind": "list",
            "matchKey": "tumor_state",
            "k": 3,
            "expected": [
              {
                "key": "immune_response",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "direction_unresolved_cross_sectional"
                    ],
                    "canonical": "direction_unresolved_cross_sectional"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "IFN_response_axis",
                      "broad_antigen_presentation_MHC"
                    ],
                    "canonical": "IFN_response_axis"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "enrichment_diminished_by_decontamination",
                      "threshold_sensitive"
                    ],
                    "canonical": "enrichment_diminished_by_decontamination"
                  }
                }
              },
              {
                "key": "stress_hypoxia",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "data_supports_cell_intrinsic_cause"
                    ],
                    "canonical": "data_supports_cell_intrinsic_cause"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "HIF_hypoxia_program_specific"
                    ],
                    "canonical": "HIF_hypoxia_program_specific"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "robust_across_methods",
                      "robust_after_decontamination"
                    ],
                    "canonical": "robust_across_methods"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CXCL9_CXCL10": {
                "op": "in",
                "value": [
                  "immune_response_tumor_state"
                ]
              },
              "CCL5": {
                "op": "in",
                "value": [
                  "shared_across_tumor_states",
                  "partner_lymphocyte_derived",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "gate_conjugate_gate_confounder",
            "role": "gate",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "partner_lymphocyte_capture"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "antigen_presentation_decomposition",
            "kind": "dict",
            "expected": {
              "MHC_class_I": {
                "op": "in",
                "value": [
                  "co_regulated_with_dominant_program",
                  "tumor_intrinsic_enriched_independent"
                ]
              },
              "MHC_class_II": {
                "op": "in",
                "value": [
                  "partner_cell_attributable",
                  "flat_not_enriched",
                  "method_sensitive_no_robust_call",
                  "co_regulated_with_dominant_program"
                ]
              }
            }
          },
          {
            "name": "cf_non_robust_state_claimed_enriched",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "transitory_melanocytic",
              "intermediate_melanocytic",
              "melanocytic",
              "MITF_high",
              "neural_crest_like",
              "mitotic",
              "stress_p53",
              "undifferentiated",
              "dedifferentiated",
              "mesenchymal_tumor",
              "AXL_high_tumor",
              "invasive_tumor",
              "senescence_like",
              "EMT_transition",
              "cancer_stem_cell_like",
              "oxidative_stress_tumor"
            ]
          },
          {
            "name": "cf_framework_generic_or_offtissue_substitution",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "Tsoi_undifferentiated",
              "Tsoi_neural_crest_like",
              "Tsoi_transitory",
              "Tsoi_melanocytic",
              "Rambow_NCSC",
              "Wouters_mesenchymal",
              "Pozniak_TCF4_dependent",
              "tumor_cell",
              "melanoma_cell",
              "malignant_cell",
              "tumor_subpopulation",
              "keratinocyte",
              "hepatocyte",
              "lung_adenocarcinoma_cell",
              "colon_adenocarcinoma_cell",
              "Tirosh_cell_cycle_program",
              "Jerby_Arnon_program",
              "glioblastoma_cell",
              "breast_carcinoma_cell"
            ]
          },
          {
            "name": "cf_immune_response_direction_committed",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_immune_robustness_recall",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_ambiguous_hedge",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].causation_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_stress_generic_stress_response_evidence",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_conjugate_confounder_contamination_denied",
            "role": "hard_fail",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected"
            ]
          },
          {
            "name": "cf_ccl5_recall_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CCL5",
            "kind": "avoid",
            "expected": [
              "immune_response_tumor_state"
            ]
          },
          {
            "name": "cf_cxcl9_cxcl10_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CXCL9_CXCL10",
            "kind": "avoid",
            "expected": [
              "stress_hypoxia_tumor_state"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-20-beta-reasoning-pi",
      "modelName": "grok-4.20-beta-0309-reasoning",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_tumor_state_causation_v5",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1421,
      "cost": 0.10188015000000002,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "enriched_tumor_states",
            "kind": "list",
            "matchKey": "tumor_state",
            "k": 3,
            "expected": [
              {
                "key": "immune_response",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "direction_unresolved_cross_sectional"
                    ],
                    "canonical": "direction_unresolved_cross_sectional"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "IFN_response_axis",
                      "broad_antigen_presentation_MHC"
                    ],
                    "canonical": "IFN_response_axis"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "enrichment_diminished_by_decontamination",
                      "threshold_sensitive"
                    ],
                    "canonical": "enrichment_diminished_by_decontamination"
                  }
                }
              },
              {
                "key": "stress_hypoxia",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "data_supports_cell_intrinsic_cause"
                    ],
                    "canonical": "data_supports_cell_intrinsic_cause"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "HIF_hypoxia_program_specific"
                    ],
                    "canonical": "HIF_hypoxia_program_specific"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "robust_across_methods",
                      "robust_after_decontamination"
                    ],
                    "canonical": "robust_across_methods"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CXCL9_CXCL10": {
                "op": "in",
                "value": [
                  "immune_response_tumor_state"
                ]
              },
              "CCL5": {
                "op": "in",
                "value": [
                  "shared_across_tumor_states",
                  "partner_lymphocyte_derived",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "gate_conjugate_gate_confounder",
            "role": "gate",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "partner_lymphocyte_capture"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "antigen_presentation_decomposition",
            "kind": "dict",
            "expected": {
              "MHC_class_I": {
                "op": "in",
                "value": [
                  "co_regulated_with_dominant_program",
                  "tumor_intrinsic_enriched_independent"
                ]
              },
              "MHC_class_II": {
                "op": "in",
                "value": [
                  "partner_cell_attributable",
                  "flat_not_enriched",
                  "method_sensitive_no_robust_call",
                  "co_regulated_with_dominant_program"
                ]
              }
            }
          },
          {
            "name": "cf_non_robust_state_claimed_enriched",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "transitory_melanocytic",
              "intermediate_melanocytic",
              "melanocytic",
              "MITF_high",
              "neural_crest_like",
              "mitotic",
              "stress_p53",
              "undifferentiated",
              "dedifferentiated",
              "mesenchymal_tumor",
              "AXL_high_tumor",
              "invasive_tumor",
              "senescence_like",
              "EMT_transition",
              "cancer_stem_cell_like",
              "oxidative_stress_tumor"
            ]
          },
          {
            "name": "cf_framework_generic_or_offtissue_substitution",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "Tsoi_undifferentiated",
              "Tsoi_neural_crest_like",
              "Tsoi_transitory",
              "Tsoi_melanocytic",
              "Rambow_NCSC",
              "Wouters_mesenchymal",
              "Pozniak_TCF4_dependent",
              "tumor_cell",
              "melanoma_cell",
              "malignant_cell",
              "tumor_subpopulation",
              "keratinocyte",
              "hepatocyte",
              "lung_adenocarcinoma_cell",
              "colon_adenocarcinoma_cell",
              "Tirosh_cell_cycle_program",
              "Jerby_Arnon_program",
              "glioblastoma_cell",
              "breast_carcinoma_cell"
            ]
          },
          {
            "name": "cf_immune_response_direction_committed",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_immune_robustness_recall",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_ambiguous_hedge",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].causation_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_stress_generic_stress_response_evidence",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_conjugate_confounder_contamination_denied",
            "role": "hard_fail",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected"
            ]
          },
          {
            "name": "cf_ccl5_recall_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CCL5",
            "kind": "avoid",
            "expected": [
              "immune_response_tumor_state"
            ]
          },
          {
            "name": "cf_cxcl9_cxcl10_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CXCL9_CXCL10",
            "kind": "avoid",
            "expected": [
              "stress_hypoxia_tumor_state"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-3-pi",
      "modelName": "grok-4.3",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_tumor_state_causation_v5",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 326,
      "cost": 0.06549470000000002,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "enriched_tumor_states",
            "kind": "list",
            "matchKey": "tumor_state",
            "k": 3,
            "expected": [
              {
                "key": "immune_response",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "direction_unresolved_cross_sectional"
                    ],
                    "canonical": "direction_unresolved_cross_sectional"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "IFN_response_axis",
                      "broad_antigen_presentation_MHC"
                    ],
                    "canonical": "IFN_response_axis"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "enrichment_diminished_by_decontamination",
                      "threshold_sensitive"
                    ],
                    "canonical": "enrichment_diminished_by_decontamination"
                  }
                }
              },
              {
                "key": "stress_hypoxia",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "data_supports_cell_intrinsic_cause"
                    ],
                    "canonical": "data_supports_cell_intrinsic_cause"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "HIF_hypoxia_program_specific"
                    ],
                    "canonical": "HIF_hypoxia_program_specific"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "robust_across_methods",
                      "robust_after_decontamination"
                    ],
                    "canonical": "robust_across_methods"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CXCL9_CXCL10": {
                "op": "in",
                "value": [
                  "immune_response_tumor_state"
                ]
              },
              "CCL5": {
                "op": "in",
                "value": [
                  "shared_across_tumor_states",
                  "partner_lymphocyte_derived",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "gate_conjugate_gate_confounder",
            "role": "gate",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "partner_lymphocyte_capture"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "antigen_presentation_decomposition",
            "kind": "dict",
            "expected": {
              "MHC_class_I": {
                "op": "in",
                "value": [
                  "co_regulated_with_dominant_program",
                  "tumor_intrinsic_enriched_independent"
                ]
              },
              "MHC_class_II": {
                "op": "in",
                "value": [
                  "partner_cell_attributable",
                  "flat_not_enriched",
                  "method_sensitive_no_robust_call",
                  "co_regulated_with_dominant_program"
                ]
              }
            }
          },
          {
            "name": "cf_non_robust_state_claimed_enriched",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "transitory_melanocytic",
              "intermediate_melanocytic",
              "melanocytic",
              "MITF_high",
              "neural_crest_like",
              "mitotic",
              "stress_p53",
              "undifferentiated",
              "dedifferentiated",
              "mesenchymal_tumor",
              "AXL_high_tumor",
              "invasive_tumor",
              "senescence_like",
              "EMT_transition",
              "cancer_stem_cell_like",
              "oxidative_stress_tumor"
            ]
          },
          {
            "name": "cf_framework_generic_or_offtissue_substitution",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "Tsoi_undifferentiated",
              "Tsoi_neural_crest_like",
              "Tsoi_transitory",
              "Tsoi_melanocytic",
              "Rambow_NCSC",
              "Wouters_mesenchymal",
              "Pozniak_TCF4_dependent",
              "tumor_cell",
              "melanoma_cell",
              "malignant_cell",
              "tumor_subpopulation",
              "keratinocyte",
              "hepatocyte",
              "lung_adenocarcinoma_cell",
              "colon_adenocarcinoma_cell",
              "Tirosh_cell_cycle_program",
              "Jerby_Arnon_program",
              "glioblastoma_cell",
              "breast_carcinoma_cell"
            ]
          },
          {
            "name": "cf_immune_response_direction_committed",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_immune_robustness_recall",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_ambiguous_hedge",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].causation_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_stress_generic_stress_response_evidence",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_conjugate_confounder_contamination_denied",
            "role": "hard_fail",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected"
            ]
          },
          {
            "name": "cf_ccl5_recall_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CCL5",
            "kind": "avoid",
            "expected": [
              "immune_response_tumor_state"
            ]
          },
          {
            "name": "cf_cxcl9_cxcl10_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CXCL9_CXCL10",
            "kind": "avoid",
            "expected": [
              "stress_hypoxia_tumor_state"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-3-pi",
      "modelName": "grok-4.3",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_tumor_state_causation_v5",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 691,
      "cost": 0.04783525,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "enriched_tumor_states",
            "kind": "list",
            "matchKey": "tumor_state",
            "k": 3,
            "expected": [
              {
                "key": "immune_response",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "direction_unresolved_cross_sectional"
                    ],
                    "canonical": "direction_unresolved_cross_sectional"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "IFN_response_axis",
                      "broad_antigen_presentation_MHC"
                    ],
                    "canonical": "IFN_response_axis"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "enrichment_diminished_by_decontamination",
                      "threshold_sensitive"
                    ],
                    "canonical": "enrichment_diminished_by_decontamination"
                  }
                }
              },
              {
                "key": "stress_hypoxia",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "data_supports_cell_intrinsic_cause"
                    ],
                    "canonical": "data_supports_cell_intrinsic_cause"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "HIF_hypoxia_program_specific"
                    ],
                    "canonical": "HIF_hypoxia_program_specific"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "robust_across_methods",
                      "robust_after_decontamination"
                    ],
                    "canonical": "robust_across_methods"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CXCL9_CXCL10": {
                "op": "in",
                "value": [
                  "immune_response_tumor_state"
                ]
              },
              "CCL5": {
                "op": "in",
                "value": [
                  "shared_across_tumor_states",
                  "partner_lymphocyte_derived",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "gate_conjugate_gate_confounder",
            "role": "gate",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "partner_lymphocyte_capture"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "antigen_presentation_decomposition",
            "kind": "dict",
            "expected": {
              "MHC_class_I": {
                "op": "in",
                "value": [
                  "co_regulated_with_dominant_program",
                  "tumor_intrinsic_enriched_independent"
                ]
              },
              "MHC_class_II": {
                "op": "in",
                "value": [
                  "partner_cell_attributable",
                  "flat_not_enriched",
                  "method_sensitive_no_robust_call",
                  "co_regulated_with_dominant_program"
                ]
              }
            }
          },
          {
            "name": "cf_non_robust_state_claimed_enriched",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "transitory_melanocytic",
              "intermediate_melanocytic",
              "melanocytic",
              "MITF_high",
              "neural_crest_like",
              "mitotic",
              "stress_p53",
              "undifferentiated",
              "dedifferentiated",
              "mesenchymal_tumor",
              "AXL_high_tumor",
              "invasive_tumor",
              "senescence_like",
              "EMT_transition",
              "cancer_stem_cell_like",
              "oxidative_stress_tumor"
            ]
          },
          {
            "name": "cf_framework_generic_or_offtissue_substitution",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "Tsoi_undifferentiated",
              "Tsoi_neural_crest_like",
              "Tsoi_transitory",
              "Tsoi_melanocytic",
              "Rambow_NCSC",
              "Wouters_mesenchymal",
              "Pozniak_TCF4_dependent",
              "tumor_cell",
              "melanoma_cell",
              "malignant_cell",
              "tumor_subpopulation",
              "keratinocyte",
              "hepatocyte",
              "lung_adenocarcinoma_cell",
              "colon_adenocarcinoma_cell",
              "Tirosh_cell_cycle_program",
              "Jerby_Arnon_program",
              "glioblastoma_cell",
              "breast_carcinoma_cell"
            ]
          },
          {
            "name": "cf_immune_response_direction_committed",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_immune_robustness_recall",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_ambiguous_hedge",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].causation_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_stress_generic_stress_response_evidence",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_conjugate_confounder_contamination_denied",
            "role": "hard_fail",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected"
            ]
          },
          {
            "name": "cf_ccl5_recall_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CCL5",
            "kind": "avoid",
            "expected": [
              "immune_response_tumor_state"
            ]
          },
          {
            "name": "cf_cxcl9_cxcl10_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CXCL9_CXCL10",
            "kind": "avoid",
            "expected": [
              "stress_hypoxia_tumor_state"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-3-pi",
      "modelName": "grok-4.3",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_tumor_state_causation_v5",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 581,
      "cost": 0.05366945000000001,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "enriched_tumor_states",
            "kind": "list",
            "matchKey": "tumor_state",
            "k": 3,
            "expected": [
              {
                "key": "immune_response",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "direction_unresolved_cross_sectional"
                    ],
                    "canonical": "direction_unresolved_cross_sectional"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "IFN_response_axis",
                      "broad_antigen_presentation_MHC"
                    ],
                    "canonical": "IFN_response_axis"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "enrichment_diminished_by_decontamination",
                      "threshold_sensitive"
                    ],
                    "canonical": "enrichment_diminished_by_decontamination"
                  }
                }
              },
              {
                "key": "stress_hypoxia",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "data_supports_cell_intrinsic_cause"
                    ],
                    "canonical": "data_supports_cell_intrinsic_cause"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "HIF_hypoxia_program_specific"
                    ],
                    "canonical": "HIF_hypoxia_program_specific"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "robust_across_methods",
                      "robust_after_decontamination"
                    ],
                    "canonical": "robust_across_methods"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CXCL9_CXCL10": {
                "op": "in",
                "value": [
                  "immune_response_tumor_state"
                ]
              },
              "CCL5": {
                "op": "in",
                "value": [
                  "shared_across_tumor_states",
                  "partner_lymphocyte_derived",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "gate_conjugate_gate_confounder",
            "role": "gate",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "partner_lymphocyte_capture"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "antigen_presentation_decomposition",
            "kind": "dict",
            "expected": {
              "MHC_class_I": {
                "op": "in",
                "value": [
                  "co_regulated_with_dominant_program",
                  "tumor_intrinsic_enriched_independent"
                ]
              },
              "MHC_class_II": {
                "op": "in",
                "value": [
                  "partner_cell_attributable",
                  "flat_not_enriched",
                  "method_sensitive_no_robust_call",
                  "co_regulated_with_dominant_program"
                ]
              }
            }
          },
          {
            "name": "cf_non_robust_state_claimed_enriched",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "transitory_melanocytic",
              "intermediate_melanocytic",
              "melanocytic",
              "MITF_high",
              "neural_crest_like",
              "mitotic",
              "stress_p53",
              "undifferentiated",
              "dedifferentiated",
              "mesenchymal_tumor",
              "AXL_high_tumor",
              "invasive_tumor",
              "senescence_like",
              "EMT_transition",
              "cancer_stem_cell_like",
              "oxidative_stress_tumor"
            ]
          },
          {
            "name": "cf_framework_generic_or_offtissue_substitution",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "Tsoi_undifferentiated",
              "Tsoi_neural_crest_like",
              "Tsoi_transitory",
              "Tsoi_melanocytic",
              "Rambow_NCSC",
              "Wouters_mesenchymal",
              "Pozniak_TCF4_dependent",
              "tumor_cell",
              "melanoma_cell",
              "malignant_cell",
              "tumor_subpopulation",
              "keratinocyte",
              "hepatocyte",
              "lung_adenocarcinoma_cell",
              "colon_adenocarcinoma_cell",
              "Tirosh_cell_cycle_program",
              "Jerby_Arnon_program",
              "glioblastoma_cell",
              "breast_carcinoma_cell"
            ]
          },
          {
            "name": "cf_immune_response_direction_committed",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_immune_robustness_recall",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_ambiguous_hedge",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].causation_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_stress_generic_stress_response_evidence",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_conjugate_confounder_contamination_denied",
            "role": "hard_fail",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected"
            ]
          },
          {
            "name": "cf_ccl5_recall_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CCL5",
            "kind": "avoid",
            "expected": [
              "immune_response_tumor_state"
            ]
          },
          {
            "name": "cf_cxcl9_cxcl10_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CXCL9_CXCL10",
            "kind": "avoid",
            "expected": [
              "stress_hypoxia_tumor_state"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-kimi-k2p6-pi",
      "modelName": "kimi-k2p6",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_tumor_state_causation_v5",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 1698,
      "cost": 0.89369658,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "enriched_tumor_states",
            "kind": "list",
            "matchKey": "tumor_state",
            "k": 3,
            "expected": [
              {
                "key": "immune_response",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "direction_unresolved_cross_sectional"
                    ],
                    "canonical": "direction_unresolved_cross_sectional"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "IFN_response_axis",
                      "broad_antigen_presentation_MHC"
                    ],
                    "canonical": "IFN_response_axis"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "enrichment_diminished_by_decontamination",
                      "threshold_sensitive"
                    ],
                    "canonical": "enrichment_diminished_by_decontamination"
                  }
                }
              },
              {
                "key": "stress_hypoxia",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "data_supports_cell_intrinsic_cause"
                    ],
                    "canonical": "data_supports_cell_intrinsic_cause"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "HIF_hypoxia_program_specific"
                    ],
                    "canonical": "HIF_hypoxia_program_specific"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "robust_across_methods",
                      "robust_after_decontamination"
                    ],
                    "canonical": "robust_across_methods"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CXCL9_CXCL10": {
                "op": "in",
                "value": [
                  "immune_response_tumor_state"
                ]
              },
              "CCL5": {
                "op": "in",
                "value": [
                  "shared_across_tumor_states",
                  "partner_lymphocyte_derived",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "gate_conjugate_gate_confounder",
            "role": "gate",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "partner_lymphocyte_capture"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "antigen_presentation_decomposition",
            "kind": "dict",
            "expected": {
              "MHC_class_I": {
                "op": "in",
                "value": [
                  "co_regulated_with_dominant_program",
                  "tumor_intrinsic_enriched_independent"
                ]
              },
              "MHC_class_II": {
                "op": "in",
                "value": [
                  "partner_cell_attributable",
                  "flat_not_enriched",
                  "method_sensitive_no_robust_call",
                  "co_regulated_with_dominant_program"
                ]
              }
            }
          },
          {
            "name": "cf_non_robust_state_claimed_enriched",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "transitory_melanocytic",
              "intermediate_melanocytic",
              "melanocytic",
              "MITF_high",
              "neural_crest_like",
              "mitotic",
              "stress_p53",
              "undifferentiated",
              "dedifferentiated",
              "mesenchymal_tumor",
              "AXL_high_tumor",
              "invasive_tumor",
              "senescence_like",
              "EMT_transition",
              "cancer_stem_cell_like",
              "oxidative_stress_tumor"
            ]
          },
          {
            "name": "cf_framework_generic_or_offtissue_substitution",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "Tsoi_undifferentiated",
              "Tsoi_neural_crest_like",
              "Tsoi_transitory",
              "Tsoi_melanocytic",
              "Rambow_NCSC",
              "Wouters_mesenchymal",
              "Pozniak_TCF4_dependent",
              "tumor_cell",
              "melanoma_cell",
              "malignant_cell",
              "tumor_subpopulation",
              "keratinocyte",
              "hepatocyte",
              "lung_adenocarcinoma_cell",
              "colon_adenocarcinoma_cell",
              "Tirosh_cell_cycle_program",
              "Jerby_Arnon_program",
              "glioblastoma_cell",
              "breast_carcinoma_cell"
            ]
          },
          {
            "name": "cf_immune_response_direction_committed",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_immune_robustness_recall",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_ambiguous_hedge",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].causation_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_stress_generic_stress_response_evidence",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_conjugate_confounder_contamination_denied",
            "role": "hard_fail",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected"
            ]
          },
          {
            "name": "cf_ccl5_recall_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CCL5",
            "kind": "avoid",
            "expected": [
              "immune_response_tumor_state"
            ]
          },
          {
            "name": "cf_cxcl9_cxcl10_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CXCL9_CXCL10",
            "kind": "avoid",
            "expected": [
              "stress_hypoxia_tumor_state"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-kimi-k2p6-pi",
      "modelName": "kimi-k2p6",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_tumor_state_causation_v5",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 3373,
      "cost": 1.2361949399999999,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "enriched_tumor_states",
            "kind": "list",
            "matchKey": "tumor_state",
            "k": 3,
            "expected": [
              {
                "key": "immune_response",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "direction_unresolved_cross_sectional"
                    ],
                    "canonical": "direction_unresolved_cross_sectional"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "IFN_response_axis",
                      "broad_antigen_presentation_MHC"
                    ],
                    "canonical": "IFN_response_axis"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "enrichment_diminished_by_decontamination",
                      "threshold_sensitive"
                    ],
                    "canonical": "enrichment_diminished_by_decontamination"
                  }
                }
              },
              {
                "key": "stress_hypoxia",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "data_supports_cell_intrinsic_cause"
                    ],
                    "canonical": "data_supports_cell_intrinsic_cause"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "HIF_hypoxia_program_specific"
                    ],
                    "canonical": "HIF_hypoxia_program_specific"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "robust_across_methods",
                      "robust_after_decontamination"
                    ],
                    "canonical": "robust_across_methods"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CXCL9_CXCL10": {
                "op": "in",
                "value": [
                  "immune_response_tumor_state"
                ]
              },
              "CCL5": {
                "op": "in",
                "value": [
                  "shared_across_tumor_states",
                  "partner_lymphocyte_derived",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "gate_conjugate_gate_confounder",
            "role": "gate",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "partner_lymphocyte_capture"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "antigen_presentation_decomposition",
            "kind": "dict",
            "expected": {
              "MHC_class_I": {
                "op": "in",
                "value": [
                  "co_regulated_with_dominant_program",
                  "tumor_intrinsic_enriched_independent"
                ]
              },
              "MHC_class_II": {
                "op": "in",
                "value": [
                  "partner_cell_attributable",
                  "flat_not_enriched",
                  "method_sensitive_no_robust_call",
                  "co_regulated_with_dominant_program"
                ]
              }
            }
          },
          {
            "name": "cf_non_robust_state_claimed_enriched",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "transitory_melanocytic",
              "intermediate_melanocytic",
              "melanocytic",
              "MITF_high",
              "neural_crest_like",
              "mitotic",
              "stress_p53",
              "undifferentiated",
              "dedifferentiated",
              "mesenchymal_tumor",
              "AXL_high_tumor",
              "invasive_tumor",
              "senescence_like",
              "EMT_transition",
              "cancer_stem_cell_like",
              "oxidative_stress_tumor"
            ]
          },
          {
            "name": "cf_framework_generic_or_offtissue_substitution",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "Tsoi_undifferentiated",
              "Tsoi_neural_crest_like",
              "Tsoi_transitory",
              "Tsoi_melanocytic",
              "Rambow_NCSC",
              "Wouters_mesenchymal",
              "Pozniak_TCF4_dependent",
              "tumor_cell",
              "melanoma_cell",
              "malignant_cell",
              "tumor_subpopulation",
              "keratinocyte",
              "hepatocyte",
              "lung_adenocarcinoma_cell",
              "colon_adenocarcinoma_cell",
              "Tirosh_cell_cycle_program",
              "Jerby_Arnon_program",
              "glioblastoma_cell",
              "breast_carcinoma_cell"
            ]
          },
          {
            "name": "cf_immune_response_direction_committed",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_immune_robustness_recall",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_ambiguous_hedge",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].causation_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_stress_generic_stress_response_evidence",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_conjugate_confounder_contamination_denied",
            "role": "hard_fail",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected"
            ]
          },
          {
            "name": "cf_ccl5_recall_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CCL5",
            "kind": "avoid",
            "expected": [
              "immune_response_tumor_state"
            ]
          },
          {
            "name": "cf_cxcl9_cxcl10_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CXCL9_CXCL10",
            "kind": "avoid",
            "expected": [
              "stress_hypoxia_tumor_state"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-kimi-k2p6-pi",
      "modelName": "kimi-k2p6",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_tumor_state_causation_v5",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 2848,
      "cost": 0.9062564000000002,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "enriched_tumor_states",
            "kind": "list",
            "matchKey": "tumor_state",
            "k": 3,
            "expected": [
              {
                "key": "immune_response",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "direction_unresolved_cross_sectional"
                    ],
                    "canonical": "direction_unresolved_cross_sectional"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "IFN_response_axis",
                      "broad_antigen_presentation_MHC"
                    ],
                    "canonical": "IFN_response_axis"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "enrichment_diminished_by_decontamination",
                      "threshold_sensitive"
                    ],
                    "canonical": "enrichment_diminished_by_decontamination"
                  }
                }
              },
              {
                "key": "stress_hypoxia",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "data_supports_cell_intrinsic_cause"
                    ],
                    "canonical": "data_supports_cell_intrinsic_cause"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "HIF_hypoxia_program_specific"
                    ],
                    "canonical": "HIF_hypoxia_program_specific"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "robust_across_methods",
                      "robust_after_decontamination"
                    ],
                    "canonical": "robust_across_methods"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CXCL9_CXCL10": {
                "op": "in",
                "value": [
                  "immune_response_tumor_state"
                ]
              },
              "CCL5": {
                "op": "in",
                "value": [
                  "shared_across_tumor_states",
                  "partner_lymphocyte_derived",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "gate_conjugate_gate_confounder",
            "role": "gate",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "partner_lymphocyte_capture"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "antigen_presentation_decomposition",
            "kind": "dict",
            "expected": {
              "MHC_class_I": {
                "op": "in",
                "value": [
                  "co_regulated_with_dominant_program",
                  "tumor_intrinsic_enriched_independent"
                ]
              },
              "MHC_class_II": {
                "op": "in",
                "value": [
                  "partner_cell_attributable",
                  "flat_not_enriched",
                  "method_sensitive_no_robust_call",
                  "co_regulated_with_dominant_program"
                ]
              }
            }
          },
          {
            "name": "cf_non_robust_state_claimed_enriched",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "transitory_melanocytic",
              "intermediate_melanocytic",
              "melanocytic",
              "MITF_high",
              "neural_crest_like",
              "mitotic",
              "stress_p53",
              "undifferentiated",
              "dedifferentiated",
              "mesenchymal_tumor",
              "AXL_high_tumor",
              "invasive_tumor",
              "senescence_like",
              "EMT_transition",
              "cancer_stem_cell_like",
              "oxidative_stress_tumor"
            ]
          },
          {
            "name": "cf_framework_generic_or_offtissue_substitution",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "Tsoi_undifferentiated",
              "Tsoi_neural_crest_like",
              "Tsoi_transitory",
              "Tsoi_melanocytic",
              "Rambow_NCSC",
              "Wouters_mesenchymal",
              "Pozniak_TCF4_dependent",
              "tumor_cell",
              "melanoma_cell",
              "malignant_cell",
              "tumor_subpopulation",
              "keratinocyte",
              "hepatocyte",
              "lung_adenocarcinoma_cell",
              "colon_adenocarcinoma_cell",
              "Tirosh_cell_cycle_program",
              "Jerby_Arnon_program",
              "glioblastoma_cell",
              "breast_carcinoma_cell"
            ]
          },
          {
            "name": "cf_immune_response_direction_committed",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_immune_robustness_recall",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_ambiguous_hedge",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].causation_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_stress_generic_stress_response_evidence",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_conjugate_confounder_contamination_denied",
            "role": "hard_fail",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected"
            ]
          },
          {
            "name": "cf_ccl5_recall_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CCL5",
            "kind": "avoid",
            "expected": [
              "immune_response_tumor_state"
            ]
          },
          {
            "name": "cf_cxcl9_cxcl10_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CXCL9_CXCL10",
            "kind": "avoid",
            "expected": [
              "stress_hypoxia_tumor_state"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-6-claude-code",
      "modelName": "claude-opus-4-6",
      "harness": "claude-code",
      "task": "10x_5p_melanoma_cluster_vs_cd39_tumor_reactivity_dominance",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 1418,
      "cost": 2.338545,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "population_findings",
            "kind": "list",
            "matchKey": "capture_gate",
            "k": 4,
            "expected": [
              {
                "key": "tumor_cell_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "apc_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "combined_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "unselected_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "naive",
                      "central_memory",
                      "effector_memory"
                    ],
                    "canonical": "naive"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "low"
                    ],
                    "canonical": "low"
                  },
                  "purity_caveat": {
                    "op": "equals",
                    "values": [
                      "pure_T_cell"
                    ],
                    "canonical": "pure_T_cell"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "no_reactivity_evidence"
                    ],
                    "canonical": "no_reactivity_evidence"
                  }
                }
              },
              {
                "key": "marker_sorted_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "equals",
                    "values": [
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "terminal_exhausted_LAG3_high"
                  },
                  "clonal_expansion_level": {
                    "op": "in",
                    "values": [
                      "high",
                      "intermediate"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "marker_on_non_T_lineages",
                      "single_cell_gating_loses_target"
                    ],
                    "canonical": "marker_on_non_T_lineages"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "marker_or_exhaustion_defined_only"
                    ],
                    "canonical": "marker_or_exhaustion_defined_only"
                  }
                }
              }
            ]
          },
          {
            "name": "best_capture_approach_gate",
            "role": "gate",
            "field": "best_capture_approach",
            "kind": "value",
            "expected": "heterotypic_cell_conjugate"
          },
          {
            "name": "best_selection_marker_gate",
            "role": "gate",
            "field": "best_selection_marker",
            "kind": "value",
            "expected": "none"
          },
          {
            "name": "reactivity_criterion_gate",
            "role": "gate",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "clonal_expansion_and_physical_engagement"
          },
          {
            "name": "cross_patient_robustness_gate",
            "role": "gate",
            "field": "cross_patient_robustness",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_across_patients",
              "marginal_in_some_patients"
            ]
          },
          {
            "name": "marker_selection_consequence_gate",
            "role": "gate",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "discards_clonally_expanded_reactive_cells"
          },
          {
            "name": "superiority_claim_basis_gate",
            "role": "gate",
            "field": "superiority_claim_basis",
            "kind": "value",
            "expected": "requires_functional_reactivity_assay"
          },
          {
            "name": "conjugate_enriched_state_gate",
            "role": "gate",
            "field": "conjugate_enriched_state",
            "kind": "value",
            "expected": "progenitor_exhausted_TCF7_pos_stem_like"
          },
          {
            "name": "cf_circular_exhaustion_criterion",
            "role": "hard_fail",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "exhaustion_signature_intensity"
          },
          {
            "name": "cf_marker_sort_prior",
            "role": "hard_fail",
            "field": "best_selection_marker",
            "kind": "value",
            "op": "in",
            "expected": [
              "CD39",
              "PD1",
              "PD1_high",
              "TIM3",
              "PD1_CD39",
              "CD39_CD103"
            ]
          },
          {
            "name": "cf_single_cell_capture_prior",
            "role": "hard_fail",
            "field": "best_capture_approach",
            "kind": "value",
            "op": "in",
            "expected": [
              "dissociated_single_cell",
              "dissociated_bulk_population"
            ]
          },
          {
            "name": "cf_off_data_capture_gate",
            "role": "hard_fail",
            "field": "$.population_findings[*].capture_gate",
            "kind": "avoid",
            "expected": [
              "dissociated_bulk",
              "matched_peripheral_blood",
              "matched_lymph_node",
              "post_hoc_expanded_clonotype_group"
            ]
          },
          {
            "name": "cf_conjugate_mischaracterized",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_marker_singlet_called_engaged",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "contains",
            "matchKey": "capture_gate",
            "entity": "marker_sorted_singlet",
            "valueField": "reactivity_evidence",
            "expected": [
              "engagement_and_clonal_expansion"
            ]
          },
          {
            "name": "cf_no_engaged_reactive_population",
            "role": "hard_fail",
            "field": "$.population_findings[*].reactivity_evidence",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_marker_captures_all_reactive",
            "role": "hard_fail",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "captures_full_reactive_repertoire"
          },
          {
            "name": "cf_superiority_overclaimed_from_phenotype",
            "role": "hard_fail",
            "field": "superiority_claim_basis",
            "kind": "value",
            "op": "in",
            "expected": [
              "established_by_scrna_phenotype",
              "established_by_reactivity_proxy_scores"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-6-claude-code",
      "modelName": "claude-opus-4-6",
      "harness": "claude-code",
      "task": "10x_5p_melanoma_cluster_vs_cd39_tumor_reactivity_dominance",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 1208,
      "cost": 1.4746467499999998,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "population_findings",
            "kind": "list",
            "matchKey": "capture_gate",
            "k": 4,
            "expected": [
              {
                "key": "tumor_cell_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "apc_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "combined_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "unselected_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "naive",
                      "central_memory",
                      "effector_memory"
                    ],
                    "canonical": "naive"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "low"
                    ],
                    "canonical": "low"
                  },
                  "purity_caveat": {
                    "op": "equals",
                    "values": [
                      "pure_T_cell"
                    ],
                    "canonical": "pure_T_cell"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "no_reactivity_evidence"
                    ],
                    "canonical": "no_reactivity_evidence"
                  }
                }
              },
              {
                "key": "marker_sorted_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "equals",
                    "values": [
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "terminal_exhausted_LAG3_high"
                  },
                  "clonal_expansion_level": {
                    "op": "in",
                    "values": [
                      "high",
                      "intermediate"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "marker_on_non_T_lineages",
                      "single_cell_gating_loses_target"
                    ],
                    "canonical": "marker_on_non_T_lineages"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "marker_or_exhaustion_defined_only"
                    ],
                    "canonical": "marker_or_exhaustion_defined_only"
                  }
                }
              }
            ]
          },
          {
            "name": "best_capture_approach_gate",
            "role": "gate",
            "field": "best_capture_approach",
            "kind": "value",
            "expected": "heterotypic_cell_conjugate"
          },
          {
            "name": "best_selection_marker_gate",
            "role": "gate",
            "field": "best_selection_marker",
            "kind": "value",
            "expected": "none"
          },
          {
            "name": "reactivity_criterion_gate",
            "role": "gate",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "clonal_expansion_and_physical_engagement"
          },
          {
            "name": "cross_patient_robustness_gate",
            "role": "gate",
            "field": "cross_patient_robustness",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_across_patients",
              "marginal_in_some_patients"
            ]
          },
          {
            "name": "marker_selection_consequence_gate",
            "role": "gate",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "discards_clonally_expanded_reactive_cells"
          },
          {
            "name": "superiority_claim_basis_gate",
            "role": "gate",
            "field": "superiority_claim_basis",
            "kind": "value",
            "expected": "requires_functional_reactivity_assay"
          },
          {
            "name": "conjugate_enriched_state_gate",
            "role": "gate",
            "field": "conjugate_enriched_state",
            "kind": "value",
            "expected": "progenitor_exhausted_TCF7_pos_stem_like"
          },
          {
            "name": "cf_circular_exhaustion_criterion",
            "role": "hard_fail",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "exhaustion_signature_intensity"
          },
          {
            "name": "cf_marker_sort_prior",
            "role": "hard_fail",
            "field": "best_selection_marker",
            "kind": "value",
            "op": "in",
            "expected": [
              "CD39",
              "PD1",
              "PD1_high",
              "TIM3",
              "PD1_CD39",
              "CD39_CD103"
            ]
          },
          {
            "name": "cf_single_cell_capture_prior",
            "role": "hard_fail",
            "field": "best_capture_approach",
            "kind": "value",
            "op": "in",
            "expected": [
              "dissociated_single_cell",
              "dissociated_bulk_population"
            ]
          },
          {
            "name": "cf_off_data_capture_gate",
            "role": "hard_fail",
            "field": "$.population_findings[*].capture_gate",
            "kind": "avoid",
            "expected": [
              "dissociated_bulk",
              "matched_peripheral_blood",
              "matched_lymph_node",
              "post_hoc_expanded_clonotype_group"
            ]
          },
          {
            "name": "cf_conjugate_mischaracterized",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_marker_singlet_called_engaged",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "contains",
            "matchKey": "capture_gate",
            "entity": "marker_sorted_singlet",
            "valueField": "reactivity_evidence",
            "expected": [
              "engagement_and_clonal_expansion"
            ]
          },
          {
            "name": "cf_no_engaged_reactive_population",
            "role": "hard_fail",
            "field": "$.population_findings[*].reactivity_evidence",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_marker_captures_all_reactive",
            "role": "hard_fail",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "captures_full_reactive_repertoire"
          },
          {
            "name": "cf_superiority_overclaimed_from_phenotype",
            "role": "hard_fail",
            "field": "superiority_claim_basis",
            "kind": "value",
            "op": "in",
            "expected": [
              "established_by_scrna_phenotype",
              "established_by_reactivity_proxy_scores"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-6-claude-code",
      "modelName": "claude-opus-4-6",
      "harness": "claude-code",
      "task": "10x_5p_melanoma_cluster_vs_cd39_tumor_reactivity_dominance",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1145,
      "cost": 2.50553575,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "population_findings",
            "kind": "list",
            "matchKey": "capture_gate",
            "k": 4,
            "expected": [
              {
                "key": "tumor_cell_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "apc_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "combined_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "unselected_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "naive",
                      "central_memory",
                      "effector_memory"
                    ],
                    "canonical": "naive"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "low"
                    ],
                    "canonical": "low"
                  },
                  "purity_caveat": {
                    "op": "equals",
                    "values": [
                      "pure_T_cell"
                    ],
                    "canonical": "pure_T_cell"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "no_reactivity_evidence"
                    ],
                    "canonical": "no_reactivity_evidence"
                  }
                }
              },
              {
                "key": "marker_sorted_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "equals",
                    "values": [
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "terminal_exhausted_LAG3_high"
                  },
                  "clonal_expansion_level": {
                    "op": "in",
                    "values": [
                      "high",
                      "intermediate"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "marker_on_non_T_lineages",
                      "single_cell_gating_loses_target"
                    ],
                    "canonical": "marker_on_non_T_lineages"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "marker_or_exhaustion_defined_only"
                    ],
                    "canonical": "marker_or_exhaustion_defined_only"
                  }
                }
              }
            ]
          },
          {
            "name": "best_capture_approach_gate",
            "role": "gate",
            "field": "best_capture_approach",
            "kind": "value",
            "expected": "heterotypic_cell_conjugate"
          },
          {
            "name": "best_selection_marker_gate",
            "role": "gate",
            "field": "best_selection_marker",
            "kind": "value",
            "expected": "none"
          },
          {
            "name": "reactivity_criterion_gate",
            "role": "gate",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "clonal_expansion_and_physical_engagement"
          },
          {
            "name": "cross_patient_robustness_gate",
            "role": "gate",
            "field": "cross_patient_robustness",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_across_patients",
              "marginal_in_some_patients"
            ]
          },
          {
            "name": "marker_selection_consequence_gate",
            "role": "gate",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "discards_clonally_expanded_reactive_cells"
          },
          {
            "name": "superiority_claim_basis_gate",
            "role": "gate",
            "field": "superiority_claim_basis",
            "kind": "value",
            "expected": "requires_functional_reactivity_assay"
          },
          {
            "name": "conjugate_enriched_state_gate",
            "role": "gate",
            "field": "conjugate_enriched_state",
            "kind": "value",
            "expected": "progenitor_exhausted_TCF7_pos_stem_like"
          },
          {
            "name": "cf_circular_exhaustion_criterion",
            "role": "hard_fail",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "exhaustion_signature_intensity"
          },
          {
            "name": "cf_marker_sort_prior",
            "role": "hard_fail",
            "field": "best_selection_marker",
            "kind": "value",
            "op": "in",
            "expected": [
              "CD39",
              "PD1",
              "PD1_high",
              "TIM3",
              "PD1_CD39",
              "CD39_CD103"
            ]
          },
          {
            "name": "cf_single_cell_capture_prior",
            "role": "hard_fail",
            "field": "best_capture_approach",
            "kind": "value",
            "op": "in",
            "expected": [
              "dissociated_single_cell",
              "dissociated_bulk_population"
            ]
          },
          {
            "name": "cf_off_data_capture_gate",
            "role": "hard_fail",
            "field": "$.population_findings[*].capture_gate",
            "kind": "avoid",
            "expected": [
              "dissociated_bulk",
              "matched_peripheral_blood",
              "matched_lymph_node",
              "post_hoc_expanded_clonotype_group"
            ]
          },
          {
            "name": "cf_conjugate_mischaracterized",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_marker_singlet_called_engaged",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "contains",
            "matchKey": "capture_gate",
            "entity": "marker_sorted_singlet",
            "valueField": "reactivity_evidence",
            "expected": [
              "engagement_and_clonal_expansion"
            ]
          },
          {
            "name": "cf_no_engaged_reactive_population",
            "role": "hard_fail",
            "field": "$.population_findings[*].reactivity_evidence",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_marker_captures_all_reactive",
            "role": "hard_fail",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "captures_full_reactive_repertoire"
          },
          {
            "name": "cf_superiority_overclaimed_from_phenotype",
            "role": "hard_fail",
            "field": "superiority_claim_basis",
            "kind": "value",
            "op": "in",
            "expected": [
              "established_by_scrna_phenotype",
              "established_by_reactivity_proxy_scores"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-6-pi",
      "modelName": "claude-opus-4-6",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_vs_cd39_tumor_reactivity_dominance",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 611,
      "cost": 1.118825,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "population_findings",
            "kind": "list",
            "matchKey": "capture_gate",
            "k": 4,
            "expected": [
              {
                "key": "tumor_cell_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "apc_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "combined_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "unselected_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "naive",
                      "central_memory",
                      "effector_memory"
                    ],
                    "canonical": "naive"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "low"
                    ],
                    "canonical": "low"
                  },
                  "purity_caveat": {
                    "op": "equals",
                    "values": [
                      "pure_T_cell"
                    ],
                    "canonical": "pure_T_cell"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "no_reactivity_evidence"
                    ],
                    "canonical": "no_reactivity_evidence"
                  }
                }
              },
              {
                "key": "marker_sorted_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "equals",
                    "values": [
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "terminal_exhausted_LAG3_high"
                  },
                  "clonal_expansion_level": {
                    "op": "in",
                    "values": [
                      "high",
                      "intermediate"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "marker_on_non_T_lineages",
                      "single_cell_gating_loses_target"
                    ],
                    "canonical": "marker_on_non_T_lineages"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "marker_or_exhaustion_defined_only"
                    ],
                    "canonical": "marker_or_exhaustion_defined_only"
                  }
                }
              }
            ]
          },
          {
            "name": "best_capture_approach_gate",
            "role": "gate",
            "field": "best_capture_approach",
            "kind": "value",
            "expected": "heterotypic_cell_conjugate"
          },
          {
            "name": "best_selection_marker_gate",
            "role": "gate",
            "field": "best_selection_marker",
            "kind": "value",
            "expected": "none"
          },
          {
            "name": "reactivity_criterion_gate",
            "role": "gate",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "clonal_expansion_and_physical_engagement"
          },
          {
            "name": "cross_patient_robustness_gate",
            "role": "gate",
            "field": "cross_patient_robustness",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_across_patients",
              "marginal_in_some_patients"
            ]
          },
          {
            "name": "marker_selection_consequence_gate",
            "role": "gate",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "discards_clonally_expanded_reactive_cells"
          },
          {
            "name": "superiority_claim_basis_gate",
            "role": "gate",
            "field": "superiority_claim_basis",
            "kind": "value",
            "expected": "requires_functional_reactivity_assay"
          },
          {
            "name": "conjugate_enriched_state_gate",
            "role": "gate",
            "field": "conjugate_enriched_state",
            "kind": "value",
            "expected": "progenitor_exhausted_TCF7_pos_stem_like"
          },
          {
            "name": "cf_circular_exhaustion_criterion",
            "role": "hard_fail",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "exhaustion_signature_intensity"
          },
          {
            "name": "cf_marker_sort_prior",
            "role": "hard_fail",
            "field": "best_selection_marker",
            "kind": "value",
            "op": "in",
            "expected": [
              "CD39",
              "PD1",
              "PD1_high",
              "TIM3",
              "PD1_CD39",
              "CD39_CD103"
            ]
          },
          {
            "name": "cf_single_cell_capture_prior",
            "role": "hard_fail",
            "field": "best_capture_approach",
            "kind": "value",
            "op": "in",
            "expected": [
              "dissociated_single_cell",
              "dissociated_bulk_population"
            ]
          },
          {
            "name": "cf_off_data_capture_gate",
            "role": "hard_fail",
            "field": "$.population_findings[*].capture_gate",
            "kind": "avoid",
            "expected": [
              "dissociated_bulk",
              "matched_peripheral_blood",
              "matched_lymph_node",
              "post_hoc_expanded_clonotype_group"
            ]
          },
          {
            "name": "cf_conjugate_mischaracterized",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_marker_singlet_called_engaged",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "contains",
            "matchKey": "capture_gate",
            "entity": "marker_sorted_singlet",
            "valueField": "reactivity_evidence",
            "expected": [
              "engagement_and_clonal_expansion"
            ]
          },
          {
            "name": "cf_no_engaged_reactive_population",
            "role": "hard_fail",
            "field": "$.population_findings[*].reactivity_evidence",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_marker_captures_all_reactive",
            "role": "hard_fail",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "captures_full_reactive_repertoire"
          },
          {
            "name": "cf_superiority_overclaimed_from_phenotype",
            "role": "hard_fail",
            "field": "superiority_claim_basis",
            "kind": "value",
            "op": "in",
            "expected": [
              "established_by_scrna_phenotype",
              "established_by_reactivity_proxy_scores"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-6-pi",
      "modelName": "claude-opus-4-6",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_vs_cd39_tumor_reactivity_dominance",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 583,
      "cost": 1.256377,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "population_findings",
            "kind": "list",
            "matchKey": "capture_gate",
            "k": 4,
            "expected": [
              {
                "key": "tumor_cell_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "apc_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "combined_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "unselected_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "naive",
                      "central_memory",
                      "effector_memory"
                    ],
                    "canonical": "naive"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "low"
                    ],
                    "canonical": "low"
                  },
                  "purity_caveat": {
                    "op": "equals",
                    "values": [
                      "pure_T_cell"
                    ],
                    "canonical": "pure_T_cell"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "no_reactivity_evidence"
                    ],
                    "canonical": "no_reactivity_evidence"
                  }
                }
              },
              {
                "key": "marker_sorted_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "equals",
                    "values": [
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "terminal_exhausted_LAG3_high"
                  },
                  "clonal_expansion_level": {
                    "op": "in",
                    "values": [
                      "high",
                      "intermediate"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "marker_on_non_T_lineages",
                      "single_cell_gating_loses_target"
                    ],
                    "canonical": "marker_on_non_T_lineages"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "marker_or_exhaustion_defined_only"
                    ],
                    "canonical": "marker_or_exhaustion_defined_only"
                  }
                }
              }
            ]
          },
          {
            "name": "best_capture_approach_gate",
            "role": "gate",
            "field": "best_capture_approach",
            "kind": "value",
            "expected": "heterotypic_cell_conjugate"
          },
          {
            "name": "best_selection_marker_gate",
            "role": "gate",
            "field": "best_selection_marker",
            "kind": "value",
            "expected": "none"
          },
          {
            "name": "reactivity_criterion_gate",
            "role": "gate",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "clonal_expansion_and_physical_engagement"
          },
          {
            "name": "cross_patient_robustness_gate",
            "role": "gate",
            "field": "cross_patient_robustness",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_across_patients",
              "marginal_in_some_patients"
            ]
          },
          {
            "name": "marker_selection_consequence_gate",
            "role": "gate",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "discards_clonally_expanded_reactive_cells"
          },
          {
            "name": "superiority_claim_basis_gate",
            "role": "gate",
            "field": "superiority_claim_basis",
            "kind": "value",
            "expected": "requires_functional_reactivity_assay"
          },
          {
            "name": "conjugate_enriched_state_gate",
            "role": "gate",
            "field": "conjugate_enriched_state",
            "kind": "value",
            "expected": "progenitor_exhausted_TCF7_pos_stem_like"
          },
          {
            "name": "cf_circular_exhaustion_criterion",
            "role": "hard_fail",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "exhaustion_signature_intensity"
          },
          {
            "name": "cf_marker_sort_prior",
            "role": "hard_fail",
            "field": "best_selection_marker",
            "kind": "value",
            "op": "in",
            "expected": [
              "CD39",
              "PD1",
              "PD1_high",
              "TIM3",
              "PD1_CD39",
              "CD39_CD103"
            ]
          },
          {
            "name": "cf_single_cell_capture_prior",
            "role": "hard_fail",
            "field": "best_capture_approach",
            "kind": "value",
            "op": "in",
            "expected": [
              "dissociated_single_cell",
              "dissociated_bulk_population"
            ]
          },
          {
            "name": "cf_off_data_capture_gate",
            "role": "hard_fail",
            "field": "$.population_findings[*].capture_gate",
            "kind": "avoid",
            "expected": [
              "dissociated_bulk",
              "matched_peripheral_blood",
              "matched_lymph_node",
              "post_hoc_expanded_clonotype_group"
            ]
          },
          {
            "name": "cf_conjugate_mischaracterized",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_marker_singlet_called_engaged",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "contains",
            "matchKey": "capture_gate",
            "entity": "marker_sorted_singlet",
            "valueField": "reactivity_evidence",
            "expected": [
              "engagement_and_clonal_expansion"
            ]
          },
          {
            "name": "cf_no_engaged_reactive_population",
            "role": "hard_fail",
            "field": "$.population_findings[*].reactivity_evidence",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_marker_captures_all_reactive",
            "role": "hard_fail",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "captures_full_reactive_repertoire"
          },
          {
            "name": "cf_superiority_overclaimed_from_phenotype",
            "role": "hard_fail",
            "field": "superiority_claim_basis",
            "kind": "value",
            "op": "in",
            "expected": [
              "established_by_scrna_phenotype",
              "established_by_reactivity_proxy_scores"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-6-pi",
      "modelName": "claude-opus-4-6",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_vs_cd39_tumor_reactivity_dominance",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 713,
      "cost": 1.3302350000000003,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "population_findings",
            "kind": "list",
            "matchKey": "capture_gate",
            "k": 4,
            "expected": [
              {
                "key": "tumor_cell_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "apc_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "combined_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "unselected_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "naive",
                      "central_memory",
                      "effector_memory"
                    ],
                    "canonical": "naive"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "low"
                    ],
                    "canonical": "low"
                  },
                  "purity_caveat": {
                    "op": "equals",
                    "values": [
                      "pure_T_cell"
                    ],
                    "canonical": "pure_T_cell"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "no_reactivity_evidence"
                    ],
                    "canonical": "no_reactivity_evidence"
                  }
                }
              },
              {
                "key": "marker_sorted_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "equals",
                    "values": [
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "terminal_exhausted_LAG3_high"
                  },
                  "clonal_expansion_level": {
                    "op": "in",
                    "values": [
                      "high",
                      "intermediate"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "marker_on_non_T_lineages",
                      "single_cell_gating_loses_target"
                    ],
                    "canonical": "marker_on_non_T_lineages"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "marker_or_exhaustion_defined_only"
                    ],
                    "canonical": "marker_or_exhaustion_defined_only"
                  }
                }
              }
            ]
          },
          {
            "name": "best_capture_approach_gate",
            "role": "gate",
            "field": "best_capture_approach",
            "kind": "value",
            "expected": "heterotypic_cell_conjugate"
          },
          {
            "name": "best_selection_marker_gate",
            "role": "gate",
            "field": "best_selection_marker",
            "kind": "value",
            "expected": "none"
          },
          {
            "name": "reactivity_criterion_gate",
            "role": "gate",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "clonal_expansion_and_physical_engagement"
          },
          {
            "name": "cross_patient_robustness_gate",
            "role": "gate",
            "field": "cross_patient_robustness",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_across_patients",
              "marginal_in_some_patients"
            ]
          },
          {
            "name": "marker_selection_consequence_gate",
            "role": "gate",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "discards_clonally_expanded_reactive_cells"
          },
          {
            "name": "superiority_claim_basis_gate",
            "role": "gate",
            "field": "superiority_claim_basis",
            "kind": "value",
            "expected": "requires_functional_reactivity_assay"
          },
          {
            "name": "conjugate_enriched_state_gate",
            "role": "gate",
            "field": "conjugate_enriched_state",
            "kind": "value",
            "expected": "progenitor_exhausted_TCF7_pos_stem_like"
          },
          {
            "name": "cf_circular_exhaustion_criterion",
            "role": "hard_fail",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "exhaustion_signature_intensity"
          },
          {
            "name": "cf_marker_sort_prior",
            "role": "hard_fail",
            "field": "best_selection_marker",
            "kind": "value",
            "op": "in",
            "expected": [
              "CD39",
              "PD1",
              "PD1_high",
              "TIM3",
              "PD1_CD39",
              "CD39_CD103"
            ]
          },
          {
            "name": "cf_single_cell_capture_prior",
            "role": "hard_fail",
            "field": "best_capture_approach",
            "kind": "value",
            "op": "in",
            "expected": [
              "dissociated_single_cell",
              "dissociated_bulk_population"
            ]
          },
          {
            "name": "cf_off_data_capture_gate",
            "role": "hard_fail",
            "field": "$.population_findings[*].capture_gate",
            "kind": "avoid",
            "expected": [
              "dissociated_bulk",
              "matched_peripheral_blood",
              "matched_lymph_node",
              "post_hoc_expanded_clonotype_group"
            ]
          },
          {
            "name": "cf_conjugate_mischaracterized",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_marker_singlet_called_engaged",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "contains",
            "matchKey": "capture_gate",
            "entity": "marker_sorted_singlet",
            "valueField": "reactivity_evidence",
            "expected": [
              "engagement_and_clonal_expansion"
            ]
          },
          {
            "name": "cf_no_engaged_reactive_population",
            "role": "hard_fail",
            "field": "$.population_findings[*].reactivity_evidence",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_marker_captures_all_reactive",
            "role": "hard_fail",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "captures_full_reactive_repertoire"
          },
          {
            "name": "cf_superiority_overclaimed_from_phenotype",
            "role": "hard_fail",
            "field": "superiority_claim_basis",
            "kind": "value",
            "op": "in",
            "expected": [
              "established_by_scrna_phenotype",
              "established_by_reactivity_proxy_scores"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-7-claude-code",
      "modelName": "claude-opus-4-7",
      "harness": "claude-code",
      "task": "10x_5p_melanoma_cluster_vs_cd39_tumor_reactivity_dominance",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 609,
      "cost": 1.72370575,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "population_findings",
            "kind": "list",
            "matchKey": "capture_gate",
            "k": 4,
            "expected": [
              {
                "key": "tumor_cell_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "apc_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "combined_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "unselected_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "naive",
                      "central_memory",
                      "effector_memory"
                    ],
                    "canonical": "naive"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "low"
                    ],
                    "canonical": "low"
                  },
                  "purity_caveat": {
                    "op": "equals",
                    "values": [
                      "pure_T_cell"
                    ],
                    "canonical": "pure_T_cell"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "no_reactivity_evidence"
                    ],
                    "canonical": "no_reactivity_evidence"
                  }
                }
              },
              {
                "key": "marker_sorted_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "equals",
                    "values": [
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "terminal_exhausted_LAG3_high"
                  },
                  "clonal_expansion_level": {
                    "op": "in",
                    "values": [
                      "high",
                      "intermediate"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "marker_on_non_T_lineages",
                      "single_cell_gating_loses_target"
                    ],
                    "canonical": "marker_on_non_T_lineages"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "marker_or_exhaustion_defined_only"
                    ],
                    "canonical": "marker_or_exhaustion_defined_only"
                  }
                }
              }
            ]
          },
          {
            "name": "best_capture_approach_gate",
            "role": "gate",
            "field": "best_capture_approach",
            "kind": "value",
            "expected": "heterotypic_cell_conjugate"
          },
          {
            "name": "best_selection_marker_gate",
            "role": "gate",
            "field": "best_selection_marker",
            "kind": "value",
            "expected": "none"
          },
          {
            "name": "reactivity_criterion_gate",
            "role": "gate",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "clonal_expansion_and_physical_engagement"
          },
          {
            "name": "cross_patient_robustness_gate",
            "role": "gate",
            "field": "cross_patient_robustness",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_across_patients",
              "marginal_in_some_patients"
            ]
          },
          {
            "name": "marker_selection_consequence_gate",
            "role": "gate",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "discards_clonally_expanded_reactive_cells"
          },
          {
            "name": "superiority_claim_basis_gate",
            "role": "gate",
            "field": "superiority_claim_basis",
            "kind": "value",
            "expected": "requires_functional_reactivity_assay"
          },
          {
            "name": "conjugate_enriched_state_gate",
            "role": "gate",
            "field": "conjugate_enriched_state",
            "kind": "value",
            "expected": "progenitor_exhausted_TCF7_pos_stem_like"
          },
          {
            "name": "cf_circular_exhaustion_criterion",
            "role": "hard_fail",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "exhaustion_signature_intensity"
          },
          {
            "name": "cf_marker_sort_prior",
            "role": "hard_fail",
            "field": "best_selection_marker",
            "kind": "value",
            "op": "in",
            "expected": [
              "CD39",
              "PD1",
              "PD1_high",
              "TIM3",
              "PD1_CD39",
              "CD39_CD103"
            ]
          },
          {
            "name": "cf_single_cell_capture_prior",
            "role": "hard_fail",
            "field": "best_capture_approach",
            "kind": "value",
            "op": "in",
            "expected": [
              "dissociated_single_cell",
              "dissociated_bulk_population"
            ]
          },
          {
            "name": "cf_off_data_capture_gate",
            "role": "hard_fail",
            "field": "$.population_findings[*].capture_gate",
            "kind": "avoid",
            "expected": [
              "dissociated_bulk",
              "matched_peripheral_blood",
              "matched_lymph_node",
              "post_hoc_expanded_clonotype_group"
            ]
          },
          {
            "name": "cf_conjugate_mischaracterized",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_marker_singlet_called_engaged",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "contains",
            "matchKey": "capture_gate",
            "entity": "marker_sorted_singlet",
            "valueField": "reactivity_evidence",
            "expected": [
              "engagement_and_clonal_expansion"
            ]
          },
          {
            "name": "cf_no_engaged_reactive_population",
            "role": "hard_fail",
            "field": "$.population_findings[*].reactivity_evidence",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_marker_captures_all_reactive",
            "role": "hard_fail",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "captures_full_reactive_repertoire"
          },
          {
            "name": "cf_superiority_overclaimed_from_phenotype",
            "role": "hard_fail",
            "field": "superiority_claim_basis",
            "kind": "value",
            "op": "in",
            "expected": [
              "established_by_scrna_phenotype",
              "established_by_reactivity_proxy_scores"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-7-claude-code",
      "modelName": "claude-opus-4-7",
      "harness": "claude-code",
      "task": "10x_5p_melanoma_cluster_vs_cd39_tumor_reactivity_dominance",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 865,
      "cost": 3.2675265,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "population_findings",
            "kind": "list",
            "matchKey": "capture_gate",
            "k": 4,
            "expected": [
              {
                "key": "tumor_cell_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "apc_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "combined_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "unselected_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "naive",
                      "central_memory",
                      "effector_memory"
                    ],
                    "canonical": "naive"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "low"
                    ],
                    "canonical": "low"
                  },
                  "purity_caveat": {
                    "op": "equals",
                    "values": [
                      "pure_T_cell"
                    ],
                    "canonical": "pure_T_cell"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "no_reactivity_evidence"
                    ],
                    "canonical": "no_reactivity_evidence"
                  }
                }
              },
              {
                "key": "marker_sorted_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "equals",
                    "values": [
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "terminal_exhausted_LAG3_high"
                  },
                  "clonal_expansion_level": {
                    "op": "in",
                    "values": [
                      "high",
                      "intermediate"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "marker_on_non_T_lineages",
                      "single_cell_gating_loses_target"
                    ],
                    "canonical": "marker_on_non_T_lineages"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "marker_or_exhaustion_defined_only"
                    ],
                    "canonical": "marker_or_exhaustion_defined_only"
                  }
                }
              }
            ]
          },
          {
            "name": "best_capture_approach_gate",
            "role": "gate",
            "field": "best_capture_approach",
            "kind": "value",
            "expected": "heterotypic_cell_conjugate"
          },
          {
            "name": "best_selection_marker_gate",
            "role": "gate",
            "field": "best_selection_marker",
            "kind": "value",
            "expected": "none"
          },
          {
            "name": "reactivity_criterion_gate",
            "role": "gate",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "clonal_expansion_and_physical_engagement"
          },
          {
            "name": "cross_patient_robustness_gate",
            "role": "gate",
            "field": "cross_patient_robustness",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_across_patients",
              "marginal_in_some_patients"
            ]
          },
          {
            "name": "marker_selection_consequence_gate",
            "role": "gate",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "discards_clonally_expanded_reactive_cells"
          },
          {
            "name": "superiority_claim_basis_gate",
            "role": "gate",
            "field": "superiority_claim_basis",
            "kind": "value",
            "expected": "requires_functional_reactivity_assay"
          },
          {
            "name": "conjugate_enriched_state_gate",
            "role": "gate",
            "field": "conjugate_enriched_state",
            "kind": "value",
            "expected": "progenitor_exhausted_TCF7_pos_stem_like"
          },
          {
            "name": "cf_circular_exhaustion_criterion",
            "role": "hard_fail",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "exhaustion_signature_intensity"
          },
          {
            "name": "cf_marker_sort_prior",
            "role": "hard_fail",
            "field": "best_selection_marker",
            "kind": "value",
            "op": "in",
            "expected": [
              "CD39",
              "PD1",
              "PD1_high",
              "TIM3",
              "PD1_CD39",
              "CD39_CD103"
            ]
          },
          {
            "name": "cf_single_cell_capture_prior",
            "role": "hard_fail",
            "field": "best_capture_approach",
            "kind": "value",
            "op": "in",
            "expected": [
              "dissociated_single_cell",
              "dissociated_bulk_population"
            ]
          },
          {
            "name": "cf_off_data_capture_gate",
            "role": "hard_fail",
            "field": "$.population_findings[*].capture_gate",
            "kind": "avoid",
            "expected": [
              "dissociated_bulk",
              "matched_peripheral_blood",
              "matched_lymph_node",
              "post_hoc_expanded_clonotype_group"
            ]
          },
          {
            "name": "cf_conjugate_mischaracterized",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_marker_singlet_called_engaged",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "contains",
            "matchKey": "capture_gate",
            "entity": "marker_sorted_singlet",
            "valueField": "reactivity_evidence",
            "expected": [
              "engagement_and_clonal_expansion"
            ]
          },
          {
            "name": "cf_no_engaged_reactive_population",
            "role": "hard_fail",
            "field": "$.population_findings[*].reactivity_evidence",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_marker_captures_all_reactive",
            "role": "hard_fail",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "captures_full_reactive_repertoire"
          },
          {
            "name": "cf_superiority_overclaimed_from_phenotype",
            "role": "hard_fail",
            "field": "superiority_claim_basis",
            "kind": "value",
            "op": "in",
            "expected": [
              "established_by_scrna_phenotype",
              "established_by_reactivity_proxy_scores"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-7-claude-code",
      "modelName": "claude-opus-4-7",
      "harness": "claude-code",
      "task": "10x_5p_melanoma_cluster_vs_cd39_tumor_reactivity_dominance",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 910,
      "cost": 3.499384749999999,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "population_findings",
            "kind": "list",
            "matchKey": "capture_gate",
            "k": 4,
            "expected": [
              {
                "key": "tumor_cell_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "apc_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "combined_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "unselected_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "naive",
                      "central_memory",
                      "effector_memory"
                    ],
                    "canonical": "naive"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "low"
                    ],
                    "canonical": "low"
                  },
                  "purity_caveat": {
                    "op": "equals",
                    "values": [
                      "pure_T_cell"
                    ],
                    "canonical": "pure_T_cell"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "no_reactivity_evidence"
                    ],
                    "canonical": "no_reactivity_evidence"
                  }
                }
              },
              {
                "key": "marker_sorted_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "equals",
                    "values": [
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "terminal_exhausted_LAG3_high"
                  },
                  "clonal_expansion_level": {
                    "op": "in",
                    "values": [
                      "high",
                      "intermediate"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "marker_on_non_T_lineages",
                      "single_cell_gating_loses_target"
                    ],
                    "canonical": "marker_on_non_T_lineages"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "marker_or_exhaustion_defined_only"
                    ],
                    "canonical": "marker_or_exhaustion_defined_only"
                  }
                }
              }
            ]
          },
          {
            "name": "best_capture_approach_gate",
            "role": "gate",
            "field": "best_capture_approach",
            "kind": "value",
            "expected": "heterotypic_cell_conjugate"
          },
          {
            "name": "best_selection_marker_gate",
            "role": "gate",
            "field": "best_selection_marker",
            "kind": "value",
            "expected": "none"
          },
          {
            "name": "reactivity_criterion_gate",
            "role": "gate",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "clonal_expansion_and_physical_engagement"
          },
          {
            "name": "cross_patient_robustness_gate",
            "role": "gate",
            "field": "cross_patient_robustness",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_across_patients",
              "marginal_in_some_patients"
            ]
          },
          {
            "name": "marker_selection_consequence_gate",
            "role": "gate",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "discards_clonally_expanded_reactive_cells"
          },
          {
            "name": "superiority_claim_basis_gate",
            "role": "gate",
            "field": "superiority_claim_basis",
            "kind": "value",
            "expected": "requires_functional_reactivity_assay"
          },
          {
            "name": "conjugate_enriched_state_gate",
            "role": "gate",
            "field": "conjugate_enriched_state",
            "kind": "value",
            "expected": "progenitor_exhausted_TCF7_pos_stem_like"
          },
          {
            "name": "cf_circular_exhaustion_criterion",
            "role": "hard_fail",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "exhaustion_signature_intensity"
          },
          {
            "name": "cf_marker_sort_prior",
            "role": "hard_fail",
            "field": "best_selection_marker",
            "kind": "value",
            "op": "in",
            "expected": [
              "CD39",
              "PD1",
              "PD1_high",
              "TIM3",
              "PD1_CD39",
              "CD39_CD103"
            ]
          },
          {
            "name": "cf_single_cell_capture_prior",
            "role": "hard_fail",
            "field": "best_capture_approach",
            "kind": "value",
            "op": "in",
            "expected": [
              "dissociated_single_cell",
              "dissociated_bulk_population"
            ]
          },
          {
            "name": "cf_off_data_capture_gate",
            "role": "hard_fail",
            "field": "$.population_findings[*].capture_gate",
            "kind": "avoid",
            "expected": [
              "dissociated_bulk",
              "matched_peripheral_blood",
              "matched_lymph_node",
              "post_hoc_expanded_clonotype_group"
            ]
          },
          {
            "name": "cf_conjugate_mischaracterized",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_marker_singlet_called_engaged",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "contains",
            "matchKey": "capture_gate",
            "entity": "marker_sorted_singlet",
            "valueField": "reactivity_evidence",
            "expected": [
              "engagement_and_clonal_expansion"
            ]
          },
          {
            "name": "cf_no_engaged_reactive_population",
            "role": "hard_fail",
            "field": "$.population_findings[*].reactivity_evidence",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_marker_captures_all_reactive",
            "role": "hard_fail",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "captures_full_reactive_repertoire"
          },
          {
            "name": "cf_superiority_overclaimed_from_phenotype",
            "role": "hard_fail",
            "field": "superiority_claim_basis",
            "kind": "value",
            "op": "in",
            "expected": [
              "established_by_scrna_phenotype",
              "established_by_reactivity_proxy_scores"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-7-pi",
      "modelName": "claude-opus-4-7",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_vs_cd39_tumor_reactivity_dominance",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 914,
      "cost": 2.2130185,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "population_findings",
            "kind": "list",
            "matchKey": "capture_gate",
            "k": 4,
            "expected": [
              {
                "key": "tumor_cell_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "apc_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "combined_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "unselected_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "naive",
                      "central_memory",
                      "effector_memory"
                    ],
                    "canonical": "naive"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "low"
                    ],
                    "canonical": "low"
                  },
                  "purity_caveat": {
                    "op": "equals",
                    "values": [
                      "pure_T_cell"
                    ],
                    "canonical": "pure_T_cell"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "no_reactivity_evidence"
                    ],
                    "canonical": "no_reactivity_evidence"
                  }
                }
              },
              {
                "key": "marker_sorted_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "equals",
                    "values": [
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "terminal_exhausted_LAG3_high"
                  },
                  "clonal_expansion_level": {
                    "op": "in",
                    "values": [
                      "high",
                      "intermediate"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "marker_on_non_T_lineages",
                      "single_cell_gating_loses_target"
                    ],
                    "canonical": "marker_on_non_T_lineages"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "marker_or_exhaustion_defined_only"
                    ],
                    "canonical": "marker_or_exhaustion_defined_only"
                  }
                }
              }
            ]
          },
          {
            "name": "best_capture_approach_gate",
            "role": "gate",
            "field": "best_capture_approach",
            "kind": "value",
            "expected": "heterotypic_cell_conjugate"
          },
          {
            "name": "best_selection_marker_gate",
            "role": "gate",
            "field": "best_selection_marker",
            "kind": "value",
            "expected": "none"
          },
          {
            "name": "reactivity_criterion_gate",
            "role": "gate",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "clonal_expansion_and_physical_engagement"
          },
          {
            "name": "cross_patient_robustness_gate",
            "role": "gate",
            "field": "cross_patient_robustness",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_across_patients",
              "marginal_in_some_patients"
            ]
          },
          {
            "name": "marker_selection_consequence_gate",
            "role": "gate",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "discards_clonally_expanded_reactive_cells"
          },
          {
            "name": "superiority_claim_basis_gate",
            "role": "gate",
            "field": "superiority_claim_basis",
            "kind": "value",
            "expected": "requires_functional_reactivity_assay"
          },
          {
            "name": "conjugate_enriched_state_gate",
            "role": "gate",
            "field": "conjugate_enriched_state",
            "kind": "value",
            "expected": "progenitor_exhausted_TCF7_pos_stem_like"
          },
          {
            "name": "cf_circular_exhaustion_criterion",
            "role": "hard_fail",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "exhaustion_signature_intensity"
          },
          {
            "name": "cf_marker_sort_prior",
            "role": "hard_fail",
            "field": "best_selection_marker",
            "kind": "value",
            "op": "in",
            "expected": [
              "CD39",
              "PD1",
              "PD1_high",
              "TIM3",
              "PD1_CD39",
              "CD39_CD103"
            ]
          },
          {
            "name": "cf_single_cell_capture_prior",
            "role": "hard_fail",
            "field": "best_capture_approach",
            "kind": "value",
            "op": "in",
            "expected": [
              "dissociated_single_cell",
              "dissociated_bulk_population"
            ]
          },
          {
            "name": "cf_off_data_capture_gate",
            "role": "hard_fail",
            "field": "$.population_findings[*].capture_gate",
            "kind": "avoid",
            "expected": [
              "dissociated_bulk",
              "matched_peripheral_blood",
              "matched_lymph_node",
              "post_hoc_expanded_clonotype_group"
            ]
          },
          {
            "name": "cf_conjugate_mischaracterized",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_marker_singlet_called_engaged",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "contains",
            "matchKey": "capture_gate",
            "entity": "marker_sorted_singlet",
            "valueField": "reactivity_evidence",
            "expected": [
              "engagement_and_clonal_expansion"
            ]
          },
          {
            "name": "cf_no_engaged_reactive_population",
            "role": "hard_fail",
            "field": "$.population_findings[*].reactivity_evidence",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_marker_captures_all_reactive",
            "role": "hard_fail",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "captures_full_reactive_repertoire"
          },
          {
            "name": "cf_superiority_overclaimed_from_phenotype",
            "role": "hard_fail",
            "field": "superiority_claim_basis",
            "kind": "value",
            "op": "in",
            "expected": [
              "established_by_scrna_phenotype",
              "established_by_reactivity_proxy_scores"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-7-pi",
      "modelName": "claude-opus-4-7",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_vs_cd39_tumor_reactivity_dominance",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 1712,
      "cost": 3.6577669999999998,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "population_findings",
            "kind": "list",
            "matchKey": "capture_gate",
            "k": 4,
            "expected": [
              {
                "key": "tumor_cell_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "apc_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "combined_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "unselected_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "naive",
                      "central_memory",
                      "effector_memory"
                    ],
                    "canonical": "naive"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "low"
                    ],
                    "canonical": "low"
                  },
                  "purity_caveat": {
                    "op": "equals",
                    "values": [
                      "pure_T_cell"
                    ],
                    "canonical": "pure_T_cell"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "no_reactivity_evidence"
                    ],
                    "canonical": "no_reactivity_evidence"
                  }
                }
              },
              {
                "key": "marker_sorted_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "equals",
                    "values": [
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "terminal_exhausted_LAG3_high"
                  },
                  "clonal_expansion_level": {
                    "op": "in",
                    "values": [
                      "high",
                      "intermediate"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "marker_on_non_T_lineages",
                      "single_cell_gating_loses_target"
                    ],
                    "canonical": "marker_on_non_T_lineages"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "marker_or_exhaustion_defined_only"
                    ],
                    "canonical": "marker_or_exhaustion_defined_only"
                  }
                }
              }
            ]
          },
          {
            "name": "best_capture_approach_gate",
            "role": "gate",
            "field": "best_capture_approach",
            "kind": "value",
            "expected": "heterotypic_cell_conjugate"
          },
          {
            "name": "best_selection_marker_gate",
            "role": "gate",
            "field": "best_selection_marker",
            "kind": "value",
            "expected": "none"
          },
          {
            "name": "reactivity_criterion_gate",
            "role": "gate",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "clonal_expansion_and_physical_engagement"
          },
          {
            "name": "cross_patient_robustness_gate",
            "role": "gate",
            "field": "cross_patient_robustness",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_across_patients",
              "marginal_in_some_patients"
            ]
          },
          {
            "name": "marker_selection_consequence_gate",
            "role": "gate",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "discards_clonally_expanded_reactive_cells"
          },
          {
            "name": "superiority_claim_basis_gate",
            "role": "gate",
            "field": "superiority_claim_basis",
            "kind": "value",
            "expected": "requires_functional_reactivity_assay"
          },
          {
            "name": "conjugate_enriched_state_gate",
            "role": "gate",
            "field": "conjugate_enriched_state",
            "kind": "value",
            "expected": "progenitor_exhausted_TCF7_pos_stem_like"
          },
          {
            "name": "cf_circular_exhaustion_criterion",
            "role": "hard_fail",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "exhaustion_signature_intensity"
          },
          {
            "name": "cf_marker_sort_prior",
            "role": "hard_fail",
            "field": "best_selection_marker",
            "kind": "value",
            "op": "in",
            "expected": [
              "CD39",
              "PD1",
              "PD1_high",
              "TIM3",
              "PD1_CD39",
              "CD39_CD103"
            ]
          },
          {
            "name": "cf_single_cell_capture_prior",
            "role": "hard_fail",
            "field": "best_capture_approach",
            "kind": "value",
            "op": "in",
            "expected": [
              "dissociated_single_cell",
              "dissociated_bulk_population"
            ]
          },
          {
            "name": "cf_off_data_capture_gate",
            "role": "hard_fail",
            "field": "$.population_findings[*].capture_gate",
            "kind": "avoid",
            "expected": [
              "dissociated_bulk",
              "matched_peripheral_blood",
              "matched_lymph_node",
              "post_hoc_expanded_clonotype_group"
            ]
          },
          {
            "name": "cf_conjugate_mischaracterized",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_marker_singlet_called_engaged",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "contains",
            "matchKey": "capture_gate",
            "entity": "marker_sorted_singlet",
            "valueField": "reactivity_evidence",
            "expected": [
              "engagement_and_clonal_expansion"
            ]
          },
          {
            "name": "cf_no_engaged_reactive_population",
            "role": "hard_fail",
            "field": "$.population_findings[*].reactivity_evidence",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_marker_captures_all_reactive",
            "role": "hard_fail",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "captures_full_reactive_repertoire"
          },
          {
            "name": "cf_superiority_overclaimed_from_phenotype",
            "role": "hard_fail",
            "field": "superiority_claim_basis",
            "kind": "value",
            "op": "in",
            "expected": [
              "established_by_scrna_phenotype",
              "established_by_reactivity_proxy_scores"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-7-pi",
      "modelName": "claude-opus-4-7",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_vs_cd39_tumor_reactivity_dominance",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1035,
      "cost": 2.35209875,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "population_findings",
            "kind": "list",
            "matchKey": "capture_gate",
            "k": 4,
            "expected": [
              {
                "key": "tumor_cell_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "apc_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "combined_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "unselected_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "naive",
                      "central_memory",
                      "effector_memory"
                    ],
                    "canonical": "naive"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "low"
                    ],
                    "canonical": "low"
                  },
                  "purity_caveat": {
                    "op": "equals",
                    "values": [
                      "pure_T_cell"
                    ],
                    "canonical": "pure_T_cell"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "no_reactivity_evidence"
                    ],
                    "canonical": "no_reactivity_evidence"
                  }
                }
              },
              {
                "key": "marker_sorted_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "equals",
                    "values": [
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "terminal_exhausted_LAG3_high"
                  },
                  "clonal_expansion_level": {
                    "op": "in",
                    "values": [
                      "high",
                      "intermediate"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "marker_on_non_T_lineages",
                      "single_cell_gating_loses_target"
                    ],
                    "canonical": "marker_on_non_T_lineages"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "marker_or_exhaustion_defined_only"
                    ],
                    "canonical": "marker_or_exhaustion_defined_only"
                  }
                }
              }
            ]
          },
          {
            "name": "best_capture_approach_gate",
            "role": "gate",
            "field": "best_capture_approach",
            "kind": "value",
            "expected": "heterotypic_cell_conjugate"
          },
          {
            "name": "best_selection_marker_gate",
            "role": "gate",
            "field": "best_selection_marker",
            "kind": "value",
            "expected": "none"
          },
          {
            "name": "reactivity_criterion_gate",
            "role": "gate",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "clonal_expansion_and_physical_engagement"
          },
          {
            "name": "cross_patient_robustness_gate",
            "role": "gate",
            "field": "cross_patient_robustness",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_across_patients",
              "marginal_in_some_patients"
            ]
          },
          {
            "name": "marker_selection_consequence_gate",
            "role": "gate",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "discards_clonally_expanded_reactive_cells"
          },
          {
            "name": "superiority_claim_basis_gate",
            "role": "gate",
            "field": "superiority_claim_basis",
            "kind": "value",
            "expected": "requires_functional_reactivity_assay"
          },
          {
            "name": "conjugate_enriched_state_gate",
            "role": "gate",
            "field": "conjugate_enriched_state",
            "kind": "value",
            "expected": "progenitor_exhausted_TCF7_pos_stem_like"
          },
          {
            "name": "cf_circular_exhaustion_criterion",
            "role": "hard_fail",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "exhaustion_signature_intensity"
          },
          {
            "name": "cf_marker_sort_prior",
            "role": "hard_fail",
            "field": "best_selection_marker",
            "kind": "value",
            "op": "in",
            "expected": [
              "CD39",
              "PD1",
              "PD1_high",
              "TIM3",
              "PD1_CD39",
              "CD39_CD103"
            ]
          },
          {
            "name": "cf_single_cell_capture_prior",
            "role": "hard_fail",
            "field": "best_capture_approach",
            "kind": "value",
            "op": "in",
            "expected": [
              "dissociated_single_cell",
              "dissociated_bulk_population"
            ]
          },
          {
            "name": "cf_off_data_capture_gate",
            "role": "hard_fail",
            "field": "$.population_findings[*].capture_gate",
            "kind": "avoid",
            "expected": [
              "dissociated_bulk",
              "matched_peripheral_blood",
              "matched_lymph_node",
              "post_hoc_expanded_clonotype_group"
            ]
          },
          {
            "name": "cf_conjugate_mischaracterized",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_marker_singlet_called_engaged",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "contains",
            "matchKey": "capture_gate",
            "entity": "marker_sorted_singlet",
            "valueField": "reactivity_evidence",
            "expected": [
              "engagement_and_clonal_expansion"
            ]
          },
          {
            "name": "cf_no_engaged_reactive_population",
            "role": "hard_fail",
            "field": "$.population_findings[*].reactivity_evidence",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_marker_captures_all_reactive",
            "role": "hard_fail",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "captures_full_reactive_repertoire"
          },
          {
            "name": "cf_superiority_overclaimed_from_phenotype",
            "role": "hard_fail",
            "field": "superiority_claim_basis",
            "kind": "value",
            "op": "in",
            "expected": [
              "established_by_scrna_phenotype",
              "established_by_reactivity_proxy_scores"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-8-claude-code",
      "modelName": "claude-opus-4-8",
      "harness": "claude-code",
      "task": "10x_5p_melanoma_cluster_vs_cd39_tumor_reactivity_dominance",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 1659,
      "cost": 5.3636795,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "population_findings",
            "kind": "list",
            "matchKey": "capture_gate",
            "k": 4,
            "expected": [
              {
                "key": "tumor_cell_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "apc_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "combined_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "unselected_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "naive",
                      "central_memory",
                      "effector_memory"
                    ],
                    "canonical": "naive"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "low"
                    ],
                    "canonical": "low"
                  },
                  "purity_caveat": {
                    "op": "equals",
                    "values": [
                      "pure_T_cell"
                    ],
                    "canonical": "pure_T_cell"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "no_reactivity_evidence"
                    ],
                    "canonical": "no_reactivity_evidence"
                  }
                }
              },
              {
                "key": "marker_sorted_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "equals",
                    "values": [
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "terminal_exhausted_LAG3_high"
                  },
                  "clonal_expansion_level": {
                    "op": "in",
                    "values": [
                      "high",
                      "intermediate"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "marker_on_non_T_lineages",
                      "single_cell_gating_loses_target"
                    ],
                    "canonical": "marker_on_non_T_lineages"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "marker_or_exhaustion_defined_only"
                    ],
                    "canonical": "marker_or_exhaustion_defined_only"
                  }
                }
              }
            ]
          },
          {
            "name": "best_capture_approach_gate",
            "role": "gate",
            "field": "best_capture_approach",
            "kind": "value",
            "expected": "heterotypic_cell_conjugate"
          },
          {
            "name": "best_selection_marker_gate",
            "role": "gate",
            "field": "best_selection_marker",
            "kind": "value",
            "expected": "none"
          },
          {
            "name": "reactivity_criterion_gate",
            "role": "gate",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "clonal_expansion_and_physical_engagement"
          },
          {
            "name": "cross_patient_robustness_gate",
            "role": "gate",
            "field": "cross_patient_robustness",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_across_patients",
              "marginal_in_some_patients"
            ]
          },
          {
            "name": "marker_selection_consequence_gate",
            "role": "gate",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "discards_clonally_expanded_reactive_cells"
          },
          {
            "name": "superiority_claim_basis_gate",
            "role": "gate",
            "field": "superiority_claim_basis",
            "kind": "value",
            "expected": "requires_functional_reactivity_assay"
          },
          {
            "name": "conjugate_enriched_state_gate",
            "role": "gate",
            "field": "conjugate_enriched_state",
            "kind": "value",
            "expected": "progenitor_exhausted_TCF7_pos_stem_like"
          },
          {
            "name": "cf_circular_exhaustion_criterion",
            "role": "hard_fail",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "exhaustion_signature_intensity"
          },
          {
            "name": "cf_marker_sort_prior",
            "role": "hard_fail",
            "field": "best_selection_marker",
            "kind": "value",
            "op": "in",
            "expected": [
              "CD39",
              "PD1",
              "PD1_high",
              "TIM3",
              "PD1_CD39",
              "CD39_CD103"
            ]
          },
          {
            "name": "cf_single_cell_capture_prior",
            "role": "hard_fail",
            "field": "best_capture_approach",
            "kind": "value",
            "op": "in",
            "expected": [
              "dissociated_single_cell",
              "dissociated_bulk_population"
            ]
          },
          {
            "name": "cf_off_data_capture_gate",
            "role": "hard_fail",
            "field": "$.population_findings[*].capture_gate",
            "kind": "avoid",
            "expected": [
              "dissociated_bulk",
              "matched_peripheral_blood",
              "matched_lymph_node",
              "post_hoc_expanded_clonotype_group"
            ]
          },
          {
            "name": "cf_conjugate_mischaracterized",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_marker_singlet_called_engaged",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "contains",
            "matchKey": "capture_gate",
            "entity": "marker_sorted_singlet",
            "valueField": "reactivity_evidence",
            "expected": [
              "engagement_and_clonal_expansion"
            ]
          },
          {
            "name": "cf_no_engaged_reactive_population",
            "role": "hard_fail",
            "field": "$.population_findings[*].reactivity_evidence",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_marker_captures_all_reactive",
            "role": "hard_fail",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "captures_full_reactive_repertoire"
          },
          {
            "name": "cf_superiority_overclaimed_from_phenotype",
            "role": "hard_fail",
            "field": "superiority_claim_basis",
            "kind": "value",
            "op": "in",
            "expected": [
              "established_by_scrna_phenotype",
              "established_by_reactivity_proxy_scores"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-8-claude-code",
      "modelName": "claude-opus-4-8",
      "harness": "claude-code",
      "task": "10x_5p_melanoma_cluster_vs_cd39_tumor_reactivity_dominance",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 1704,
      "cost": 5.217047500000001,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "population_findings",
            "kind": "list",
            "matchKey": "capture_gate",
            "k": 4,
            "expected": [
              {
                "key": "tumor_cell_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "apc_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "combined_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "unselected_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "naive",
                      "central_memory",
                      "effector_memory"
                    ],
                    "canonical": "naive"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "low"
                    ],
                    "canonical": "low"
                  },
                  "purity_caveat": {
                    "op": "equals",
                    "values": [
                      "pure_T_cell"
                    ],
                    "canonical": "pure_T_cell"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "no_reactivity_evidence"
                    ],
                    "canonical": "no_reactivity_evidence"
                  }
                }
              },
              {
                "key": "marker_sorted_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "equals",
                    "values": [
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "terminal_exhausted_LAG3_high"
                  },
                  "clonal_expansion_level": {
                    "op": "in",
                    "values": [
                      "high",
                      "intermediate"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "marker_on_non_T_lineages",
                      "single_cell_gating_loses_target"
                    ],
                    "canonical": "marker_on_non_T_lineages"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "marker_or_exhaustion_defined_only"
                    ],
                    "canonical": "marker_or_exhaustion_defined_only"
                  }
                }
              }
            ]
          },
          {
            "name": "best_capture_approach_gate",
            "role": "gate",
            "field": "best_capture_approach",
            "kind": "value",
            "expected": "heterotypic_cell_conjugate"
          },
          {
            "name": "best_selection_marker_gate",
            "role": "gate",
            "field": "best_selection_marker",
            "kind": "value",
            "expected": "none"
          },
          {
            "name": "reactivity_criterion_gate",
            "role": "gate",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "clonal_expansion_and_physical_engagement"
          },
          {
            "name": "cross_patient_robustness_gate",
            "role": "gate",
            "field": "cross_patient_robustness",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_across_patients",
              "marginal_in_some_patients"
            ]
          },
          {
            "name": "marker_selection_consequence_gate",
            "role": "gate",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "discards_clonally_expanded_reactive_cells"
          },
          {
            "name": "superiority_claim_basis_gate",
            "role": "gate",
            "field": "superiority_claim_basis",
            "kind": "value",
            "expected": "requires_functional_reactivity_assay"
          },
          {
            "name": "conjugate_enriched_state_gate",
            "role": "gate",
            "field": "conjugate_enriched_state",
            "kind": "value",
            "expected": "progenitor_exhausted_TCF7_pos_stem_like"
          },
          {
            "name": "cf_circular_exhaustion_criterion",
            "role": "hard_fail",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "exhaustion_signature_intensity"
          },
          {
            "name": "cf_marker_sort_prior",
            "role": "hard_fail",
            "field": "best_selection_marker",
            "kind": "value",
            "op": "in",
            "expected": [
              "CD39",
              "PD1",
              "PD1_high",
              "TIM3",
              "PD1_CD39",
              "CD39_CD103"
            ]
          },
          {
            "name": "cf_single_cell_capture_prior",
            "role": "hard_fail",
            "field": "best_capture_approach",
            "kind": "value",
            "op": "in",
            "expected": [
              "dissociated_single_cell",
              "dissociated_bulk_population"
            ]
          },
          {
            "name": "cf_off_data_capture_gate",
            "role": "hard_fail",
            "field": "$.population_findings[*].capture_gate",
            "kind": "avoid",
            "expected": [
              "dissociated_bulk",
              "matched_peripheral_blood",
              "matched_lymph_node",
              "post_hoc_expanded_clonotype_group"
            ]
          },
          {
            "name": "cf_conjugate_mischaracterized",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_marker_singlet_called_engaged",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "contains",
            "matchKey": "capture_gate",
            "entity": "marker_sorted_singlet",
            "valueField": "reactivity_evidence",
            "expected": [
              "engagement_and_clonal_expansion"
            ]
          },
          {
            "name": "cf_no_engaged_reactive_population",
            "role": "hard_fail",
            "field": "$.population_findings[*].reactivity_evidence",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_marker_captures_all_reactive",
            "role": "hard_fail",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "captures_full_reactive_repertoire"
          },
          {
            "name": "cf_superiority_overclaimed_from_phenotype",
            "role": "hard_fail",
            "field": "superiority_claim_basis",
            "kind": "value",
            "op": "in",
            "expected": [
              "established_by_scrna_phenotype",
              "established_by_reactivity_proxy_scores"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-8-claude-code",
      "modelName": "claude-opus-4-8",
      "harness": "claude-code",
      "task": "10x_5p_melanoma_cluster_vs_cd39_tumor_reactivity_dominance",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": true,
      "score": null,
      "durationS": 2261,
      "cost": 9.191800750000002,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "population_findings",
            "kind": "list",
            "matchKey": "capture_gate",
            "k": 4,
            "expected": [
              {
                "key": "tumor_cell_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "apc_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "combined_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "unselected_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "naive",
                      "central_memory",
                      "effector_memory"
                    ],
                    "canonical": "naive"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "low"
                    ],
                    "canonical": "low"
                  },
                  "purity_caveat": {
                    "op": "equals",
                    "values": [
                      "pure_T_cell"
                    ],
                    "canonical": "pure_T_cell"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "no_reactivity_evidence"
                    ],
                    "canonical": "no_reactivity_evidence"
                  }
                }
              },
              {
                "key": "marker_sorted_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "equals",
                    "values": [
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "terminal_exhausted_LAG3_high"
                  },
                  "clonal_expansion_level": {
                    "op": "in",
                    "values": [
                      "high",
                      "intermediate"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "marker_on_non_T_lineages",
                      "single_cell_gating_loses_target"
                    ],
                    "canonical": "marker_on_non_T_lineages"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "marker_or_exhaustion_defined_only"
                    ],
                    "canonical": "marker_or_exhaustion_defined_only"
                  }
                }
              }
            ]
          },
          {
            "name": "best_capture_approach_gate",
            "role": "gate",
            "field": "best_capture_approach",
            "kind": "value",
            "expected": "heterotypic_cell_conjugate"
          },
          {
            "name": "best_selection_marker_gate",
            "role": "gate",
            "field": "best_selection_marker",
            "kind": "value",
            "expected": "none"
          },
          {
            "name": "reactivity_criterion_gate",
            "role": "gate",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "clonal_expansion_and_physical_engagement"
          },
          {
            "name": "cross_patient_robustness_gate",
            "role": "gate",
            "field": "cross_patient_robustness",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_across_patients",
              "marginal_in_some_patients"
            ]
          },
          {
            "name": "marker_selection_consequence_gate",
            "role": "gate",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "discards_clonally_expanded_reactive_cells"
          },
          {
            "name": "superiority_claim_basis_gate",
            "role": "gate",
            "field": "superiority_claim_basis",
            "kind": "value",
            "expected": "requires_functional_reactivity_assay"
          },
          {
            "name": "conjugate_enriched_state_gate",
            "role": "gate",
            "field": "conjugate_enriched_state",
            "kind": "value",
            "expected": "progenitor_exhausted_TCF7_pos_stem_like"
          },
          {
            "name": "cf_circular_exhaustion_criterion",
            "role": "hard_fail",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "exhaustion_signature_intensity"
          },
          {
            "name": "cf_marker_sort_prior",
            "role": "hard_fail",
            "field": "best_selection_marker",
            "kind": "value",
            "op": "in",
            "expected": [
              "CD39",
              "PD1",
              "PD1_high",
              "TIM3",
              "PD1_CD39",
              "CD39_CD103"
            ]
          },
          {
            "name": "cf_single_cell_capture_prior",
            "role": "hard_fail",
            "field": "best_capture_approach",
            "kind": "value",
            "op": "in",
            "expected": [
              "dissociated_single_cell",
              "dissociated_bulk_population"
            ]
          },
          {
            "name": "cf_off_data_capture_gate",
            "role": "hard_fail",
            "field": "$.population_findings[*].capture_gate",
            "kind": "avoid",
            "expected": [
              "dissociated_bulk",
              "matched_peripheral_blood",
              "matched_lymph_node",
              "post_hoc_expanded_clonotype_group"
            ]
          },
          {
            "name": "cf_conjugate_mischaracterized",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_marker_singlet_called_engaged",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "contains",
            "matchKey": "capture_gate",
            "entity": "marker_sorted_singlet",
            "valueField": "reactivity_evidence",
            "expected": [
              "engagement_and_clonal_expansion"
            ]
          },
          {
            "name": "cf_no_engaged_reactive_population",
            "role": "hard_fail",
            "field": "$.population_findings[*].reactivity_evidence",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_marker_captures_all_reactive",
            "role": "hard_fail",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "captures_full_reactive_repertoire"
          },
          {
            "name": "cf_superiority_overclaimed_from_phenotype",
            "role": "hard_fail",
            "field": "superiority_claim_basis",
            "kind": "value",
            "op": "in",
            "expected": [
              "established_by_scrna_phenotype",
              "established_by_reactivity_proxy_scores"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-8-pi",
      "modelName": "claude-opus-4-8",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_vs_cd39_tumor_reactivity_dominance",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 1,
      "passed": true,
      "score": null,
      "durationS": 1783,
      "cost": 4.201779500000001,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "population_findings",
            "kind": "list",
            "matchKey": "capture_gate",
            "k": 4,
            "expected": [
              {
                "key": "tumor_cell_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "apc_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "combined_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "unselected_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "naive",
                      "central_memory",
                      "effector_memory"
                    ],
                    "canonical": "naive"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "low"
                    ],
                    "canonical": "low"
                  },
                  "purity_caveat": {
                    "op": "equals",
                    "values": [
                      "pure_T_cell"
                    ],
                    "canonical": "pure_T_cell"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "no_reactivity_evidence"
                    ],
                    "canonical": "no_reactivity_evidence"
                  }
                }
              },
              {
                "key": "marker_sorted_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "equals",
                    "values": [
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "terminal_exhausted_LAG3_high"
                  },
                  "clonal_expansion_level": {
                    "op": "in",
                    "values": [
                      "high",
                      "intermediate"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "marker_on_non_T_lineages",
                      "single_cell_gating_loses_target"
                    ],
                    "canonical": "marker_on_non_T_lineages"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "marker_or_exhaustion_defined_only"
                    ],
                    "canonical": "marker_or_exhaustion_defined_only"
                  }
                }
              }
            ]
          },
          {
            "name": "best_capture_approach_gate",
            "role": "gate",
            "field": "best_capture_approach",
            "kind": "value",
            "expected": "heterotypic_cell_conjugate"
          },
          {
            "name": "best_selection_marker_gate",
            "role": "gate",
            "field": "best_selection_marker",
            "kind": "value",
            "expected": "none"
          },
          {
            "name": "reactivity_criterion_gate",
            "role": "gate",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "clonal_expansion_and_physical_engagement"
          },
          {
            "name": "cross_patient_robustness_gate",
            "role": "gate",
            "field": "cross_patient_robustness",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_across_patients",
              "marginal_in_some_patients"
            ]
          },
          {
            "name": "marker_selection_consequence_gate",
            "role": "gate",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "discards_clonally_expanded_reactive_cells"
          },
          {
            "name": "superiority_claim_basis_gate",
            "role": "gate",
            "field": "superiority_claim_basis",
            "kind": "value",
            "expected": "requires_functional_reactivity_assay"
          },
          {
            "name": "conjugate_enriched_state_gate",
            "role": "gate",
            "field": "conjugate_enriched_state",
            "kind": "value",
            "expected": "progenitor_exhausted_TCF7_pos_stem_like"
          },
          {
            "name": "cf_circular_exhaustion_criterion",
            "role": "hard_fail",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "exhaustion_signature_intensity"
          },
          {
            "name": "cf_marker_sort_prior",
            "role": "hard_fail",
            "field": "best_selection_marker",
            "kind": "value",
            "op": "in",
            "expected": [
              "CD39",
              "PD1",
              "PD1_high",
              "TIM3",
              "PD1_CD39",
              "CD39_CD103"
            ]
          },
          {
            "name": "cf_single_cell_capture_prior",
            "role": "hard_fail",
            "field": "best_capture_approach",
            "kind": "value",
            "op": "in",
            "expected": [
              "dissociated_single_cell",
              "dissociated_bulk_population"
            ]
          },
          {
            "name": "cf_off_data_capture_gate",
            "role": "hard_fail",
            "field": "$.population_findings[*].capture_gate",
            "kind": "avoid",
            "expected": [
              "dissociated_bulk",
              "matched_peripheral_blood",
              "matched_lymph_node",
              "post_hoc_expanded_clonotype_group"
            ]
          },
          {
            "name": "cf_conjugate_mischaracterized",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_marker_singlet_called_engaged",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "contains",
            "matchKey": "capture_gate",
            "entity": "marker_sorted_singlet",
            "valueField": "reactivity_evidence",
            "expected": [
              "engagement_and_clonal_expansion"
            ]
          },
          {
            "name": "cf_no_engaged_reactive_population",
            "role": "hard_fail",
            "field": "$.population_findings[*].reactivity_evidence",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_marker_captures_all_reactive",
            "role": "hard_fail",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "captures_full_reactive_repertoire"
          },
          {
            "name": "cf_superiority_overclaimed_from_phenotype",
            "role": "hard_fail",
            "field": "superiority_claim_basis",
            "kind": "value",
            "op": "in",
            "expected": [
              "established_by_scrna_phenotype",
              "established_by_reactivity_proxy_scores"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-8-pi",
      "modelName": "claude-opus-4-8",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_vs_cd39_tumor_reactivity_dominance",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 1783,
      "cost": 4.201779500000001,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "population_findings",
            "kind": "list",
            "matchKey": "capture_gate",
            "k": 4,
            "expected": [
              {
                "key": "tumor_cell_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "apc_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "combined_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "unselected_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "naive",
                      "central_memory",
                      "effector_memory"
                    ],
                    "canonical": "naive"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "low"
                    ],
                    "canonical": "low"
                  },
                  "purity_caveat": {
                    "op": "equals",
                    "values": [
                      "pure_T_cell"
                    ],
                    "canonical": "pure_T_cell"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "no_reactivity_evidence"
                    ],
                    "canonical": "no_reactivity_evidence"
                  }
                }
              },
              {
                "key": "marker_sorted_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "equals",
                    "values": [
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "terminal_exhausted_LAG3_high"
                  },
                  "clonal_expansion_level": {
                    "op": "in",
                    "values": [
                      "high",
                      "intermediate"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "marker_on_non_T_lineages",
                      "single_cell_gating_loses_target"
                    ],
                    "canonical": "marker_on_non_T_lineages"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "marker_or_exhaustion_defined_only"
                    ],
                    "canonical": "marker_or_exhaustion_defined_only"
                  }
                }
              }
            ]
          },
          {
            "name": "best_capture_approach_gate",
            "role": "gate",
            "field": "best_capture_approach",
            "kind": "value",
            "expected": "heterotypic_cell_conjugate"
          },
          {
            "name": "best_selection_marker_gate",
            "role": "gate",
            "field": "best_selection_marker",
            "kind": "value",
            "expected": "none"
          },
          {
            "name": "reactivity_criterion_gate",
            "role": "gate",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "clonal_expansion_and_physical_engagement"
          },
          {
            "name": "cross_patient_robustness_gate",
            "role": "gate",
            "field": "cross_patient_robustness",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_across_patients",
              "marginal_in_some_patients"
            ]
          },
          {
            "name": "marker_selection_consequence_gate",
            "role": "gate",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "discards_clonally_expanded_reactive_cells"
          },
          {
            "name": "superiority_claim_basis_gate",
            "role": "gate",
            "field": "superiority_claim_basis",
            "kind": "value",
            "expected": "requires_functional_reactivity_assay"
          },
          {
            "name": "conjugate_enriched_state_gate",
            "role": "gate",
            "field": "conjugate_enriched_state",
            "kind": "value",
            "expected": "progenitor_exhausted_TCF7_pos_stem_like"
          },
          {
            "name": "cf_circular_exhaustion_criterion",
            "role": "hard_fail",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "exhaustion_signature_intensity"
          },
          {
            "name": "cf_marker_sort_prior",
            "role": "hard_fail",
            "field": "best_selection_marker",
            "kind": "value",
            "op": "in",
            "expected": [
              "CD39",
              "PD1",
              "PD1_high",
              "TIM3",
              "PD1_CD39",
              "CD39_CD103"
            ]
          },
          {
            "name": "cf_single_cell_capture_prior",
            "role": "hard_fail",
            "field": "best_capture_approach",
            "kind": "value",
            "op": "in",
            "expected": [
              "dissociated_single_cell",
              "dissociated_bulk_population"
            ]
          },
          {
            "name": "cf_off_data_capture_gate",
            "role": "hard_fail",
            "field": "$.population_findings[*].capture_gate",
            "kind": "avoid",
            "expected": [
              "dissociated_bulk",
              "matched_peripheral_blood",
              "matched_lymph_node",
              "post_hoc_expanded_clonotype_group"
            ]
          },
          {
            "name": "cf_conjugate_mischaracterized",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_marker_singlet_called_engaged",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "contains",
            "matchKey": "capture_gate",
            "entity": "marker_sorted_singlet",
            "valueField": "reactivity_evidence",
            "expected": [
              "engagement_and_clonal_expansion"
            ]
          },
          {
            "name": "cf_no_engaged_reactive_population",
            "role": "hard_fail",
            "field": "$.population_findings[*].reactivity_evidence",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_marker_captures_all_reactive",
            "role": "hard_fail",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "captures_full_reactive_repertoire"
          },
          {
            "name": "cf_superiority_overclaimed_from_phenotype",
            "role": "hard_fail",
            "field": "superiority_claim_basis",
            "kind": "value",
            "op": "in",
            "expected": [
              "established_by_scrna_phenotype",
              "established_by_reactivity_proxy_scores"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-8-pi",
      "modelName": "claude-opus-4-8",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_vs_cd39_tumor_reactivity_dominance",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1680,
      "cost": 4.07992975,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "population_findings",
            "kind": "list",
            "matchKey": "capture_gate",
            "k": 4,
            "expected": [
              {
                "key": "tumor_cell_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "apc_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "combined_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "unselected_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "naive",
                      "central_memory",
                      "effector_memory"
                    ],
                    "canonical": "naive"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "low"
                    ],
                    "canonical": "low"
                  },
                  "purity_caveat": {
                    "op": "equals",
                    "values": [
                      "pure_T_cell"
                    ],
                    "canonical": "pure_T_cell"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "no_reactivity_evidence"
                    ],
                    "canonical": "no_reactivity_evidence"
                  }
                }
              },
              {
                "key": "marker_sorted_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "equals",
                    "values": [
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "terminal_exhausted_LAG3_high"
                  },
                  "clonal_expansion_level": {
                    "op": "in",
                    "values": [
                      "high",
                      "intermediate"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "marker_on_non_T_lineages",
                      "single_cell_gating_loses_target"
                    ],
                    "canonical": "marker_on_non_T_lineages"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "marker_or_exhaustion_defined_only"
                    ],
                    "canonical": "marker_or_exhaustion_defined_only"
                  }
                }
              }
            ]
          },
          {
            "name": "best_capture_approach_gate",
            "role": "gate",
            "field": "best_capture_approach",
            "kind": "value",
            "expected": "heterotypic_cell_conjugate"
          },
          {
            "name": "best_selection_marker_gate",
            "role": "gate",
            "field": "best_selection_marker",
            "kind": "value",
            "expected": "none"
          },
          {
            "name": "reactivity_criterion_gate",
            "role": "gate",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "clonal_expansion_and_physical_engagement"
          },
          {
            "name": "cross_patient_robustness_gate",
            "role": "gate",
            "field": "cross_patient_robustness",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_across_patients",
              "marginal_in_some_patients"
            ]
          },
          {
            "name": "marker_selection_consequence_gate",
            "role": "gate",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "discards_clonally_expanded_reactive_cells"
          },
          {
            "name": "superiority_claim_basis_gate",
            "role": "gate",
            "field": "superiority_claim_basis",
            "kind": "value",
            "expected": "requires_functional_reactivity_assay"
          },
          {
            "name": "conjugate_enriched_state_gate",
            "role": "gate",
            "field": "conjugate_enriched_state",
            "kind": "value",
            "expected": "progenitor_exhausted_TCF7_pos_stem_like"
          },
          {
            "name": "cf_circular_exhaustion_criterion",
            "role": "hard_fail",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "exhaustion_signature_intensity"
          },
          {
            "name": "cf_marker_sort_prior",
            "role": "hard_fail",
            "field": "best_selection_marker",
            "kind": "value",
            "op": "in",
            "expected": [
              "CD39",
              "PD1",
              "PD1_high",
              "TIM3",
              "PD1_CD39",
              "CD39_CD103"
            ]
          },
          {
            "name": "cf_single_cell_capture_prior",
            "role": "hard_fail",
            "field": "best_capture_approach",
            "kind": "value",
            "op": "in",
            "expected": [
              "dissociated_single_cell",
              "dissociated_bulk_population"
            ]
          },
          {
            "name": "cf_off_data_capture_gate",
            "role": "hard_fail",
            "field": "$.population_findings[*].capture_gate",
            "kind": "avoid",
            "expected": [
              "dissociated_bulk",
              "matched_peripheral_blood",
              "matched_lymph_node",
              "post_hoc_expanded_clonotype_group"
            ]
          },
          {
            "name": "cf_conjugate_mischaracterized",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_marker_singlet_called_engaged",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "contains",
            "matchKey": "capture_gate",
            "entity": "marker_sorted_singlet",
            "valueField": "reactivity_evidence",
            "expected": [
              "engagement_and_clonal_expansion"
            ]
          },
          {
            "name": "cf_no_engaged_reactive_population",
            "role": "hard_fail",
            "field": "$.population_findings[*].reactivity_evidence",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_marker_captures_all_reactive",
            "role": "hard_fail",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "captures_full_reactive_repertoire"
          },
          {
            "name": "cf_superiority_overclaimed_from_phenotype",
            "role": "hard_fail",
            "field": "superiority_claim_basis",
            "kind": "value",
            "op": "in",
            "expected": [
              "established_by_scrna_phenotype",
              "established_by_reactivity_proxy_scores"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-4-6-pi",
      "modelName": "claude-sonnet-4-6",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_vs_cd39_tumor_reactivity_dominance",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 1360,
      "cost": 1.0553691,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "population_findings",
            "kind": "list",
            "matchKey": "capture_gate",
            "k": 4,
            "expected": [
              {
                "key": "tumor_cell_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "apc_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "combined_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "unselected_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "naive",
                      "central_memory",
                      "effector_memory"
                    ],
                    "canonical": "naive"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "low"
                    ],
                    "canonical": "low"
                  },
                  "purity_caveat": {
                    "op": "equals",
                    "values": [
                      "pure_T_cell"
                    ],
                    "canonical": "pure_T_cell"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "no_reactivity_evidence"
                    ],
                    "canonical": "no_reactivity_evidence"
                  }
                }
              },
              {
                "key": "marker_sorted_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "equals",
                    "values": [
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "terminal_exhausted_LAG3_high"
                  },
                  "clonal_expansion_level": {
                    "op": "in",
                    "values": [
                      "high",
                      "intermediate"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "marker_on_non_T_lineages",
                      "single_cell_gating_loses_target"
                    ],
                    "canonical": "marker_on_non_T_lineages"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "marker_or_exhaustion_defined_only"
                    ],
                    "canonical": "marker_or_exhaustion_defined_only"
                  }
                }
              }
            ]
          },
          {
            "name": "best_capture_approach_gate",
            "role": "gate",
            "field": "best_capture_approach",
            "kind": "value",
            "expected": "heterotypic_cell_conjugate"
          },
          {
            "name": "best_selection_marker_gate",
            "role": "gate",
            "field": "best_selection_marker",
            "kind": "value",
            "expected": "none"
          },
          {
            "name": "reactivity_criterion_gate",
            "role": "gate",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "clonal_expansion_and_physical_engagement"
          },
          {
            "name": "cross_patient_robustness_gate",
            "role": "gate",
            "field": "cross_patient_robustness",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_across_patients",
              "marginal_in_some_patients"
            ]
          },
          {
            "name": "marker_selection_consequence_gate",
            "role": "gate",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "discards_clonally_expanded_reactive_cells"
          },
          {
            "name": "superiority_claim_basis_gate",
            "role": "gate",
            "field": "superiority_claim_basis",
            "kind": "value",
            "expected": "requires_functional_reactivity_assay"
          },
          {
            "name": "conjugate_enriched_state_gate",
            "role": "gate",
            "field": "conjugate_enriched_state",
            "kind": "value",
            "expected": "progenitor_exhausted_TCF7_pos_stem_like"
          },
          {
            "name": "cf_circular_exhaustion_criterion",
            "role": "hard_fail",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "exhaustion_signature_intensity"
          },
          {
            "name": "cf_marker_sort_prior",
            "role": "hard_fail",
            "field": "best_selection_marker",
            "kind": "value",
            "op": "in",
            "expected": [
              "CD39",
              "PD1",
              "PD1_high",
              "TIM3",
              "PD1_CD39",
              "CD39_CD103"
            ]
          },
          {
            "name": "cf_single_cell_capture_prior",
            "role": "hard_fail",
            "field": "best_capture_approach",
            "kind": "value",
            "op": "in",
            "expected": [
              "dissociated_single_cell",
              "dissociated_bulk_population"
            ]
          },
          {
            "name": "cf_off_data_capture_gate",
            "role": "hard_fail",
            "field": "$.population_findings[*].capture_gate",
            "kind": "avoid",
            "expected": [
              "dissociated_bulk",
              "matched_peripheral_blood",
              "matched_lymph_node",
              "post_hoc_expanded_clonotype_group"
            ]
          },
          {
            "name": "cf_conjugate_mischaracterized",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_marker_singlet_called_engaged",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "contains",
            "matchKey": "capture_gate",
            "entity": "marker_sorted_singlet",
            "valueField": "reactivity_evidence",
            "expected": [
              "engagement_and_clonal_expansion"
            ]
          },
          {
            "name": "cf_no_engaged_reactive_population",
            "role": "hard_fail",
            "field": "$.population_findings[*].reactivity_evidence",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_marker_captures_all_reactive",
            "role": "hard_fail",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "captures_full_reactive_repertoire"
          },
          {
            "name": "cf_superiority_overclaimed_from_phenotype",
            "role": "hard_fail",
            "field": "superiority_claim_basis",
            "kind": "value",
            "op": "in",
            "expected": [
              "established_by_scrna_phenotype",
              "established_by_reactivity_proxy_scores"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-4-6-pi",
      "modelName": "claude-sonnet-4-6",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_vs_cd39_tumor_reactivity_dominance",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 1306,
      "cost": 0.8866483499999999,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "population_findings",
            "kind": "list",
            "matchKey": "capture_gate",
            "k": 4,
            "expected": [
              {
                "key": "tumor_cell_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "apc_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "combined_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "unselected_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "naive",
                      "central_memory",
                      "effector_memory"
                    ],
                    "canonical": "naive"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "low"
                    ],
                    "canonical": "low"
                  },
                  "purity_caveat": {
                    "op": "equals",
                    "values": [
                      "pure_T_cell"
                    ],
                    "canonical": "pure_T_cell"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "no_reactivity_evidence"
                    ],
                    "canonical": "no_reactivity_evidence"
                  }
                }
              },
              {
                "key": "marker_sorted_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "equals",
                    "values": [
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "terminal_exhausted_LAG3_high"
                  },
                  "clonal_expansion_level": {
                    "op": "in",
                    "values": [
                      "high",
                      "intermediate"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "marker_on_non_T_lineages",
                      "single_cell_gating_loses_target"
                    ],
                    "canonical": "marker_on_non_T_lineages"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "marker_or_exhaustion_defined_only"
                    ],
                    "canonical": "marker_or_exhaustion_defined_only"
                  }
                }
              }
            ]
          },
          {
            "name": "best_capture_approach_gate",
            "role": "gate",
            "field": "best_capture_approach",
            "kind": "value",
            "expected": "heterotypic_cell_conjugate"
          },
          {
            "name": "best_selection_marker_gate",
            "role": "gate",
            "field": "best_selection_marker",
            "kind": "value",
            "expected": "none"
          },
          {
            "name": "reactivity_criterion_gate",
            "role": "gate",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "clonal_expansion_and_physical_engagement"
          },
          {
            "name": "cross_patient_robustness_gate",
            "role": "gate",
            "field": "cross_patient_robustness",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_across_patients",
              "marginal_in_some_patients"
            ]
          },
          {
            "name": "marker_selection_consequence_gate",
            "role": "gate",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "discards_clonally_expanded_reactive_cells"
          },
          {
            "name": "superiority_claim_basis_gate",
            "role": "gate",
            "field": "superiority_claim_basis",
            "kind": "value",
            "expected": "requires_functional_reactivity_assay"
          },
          {
            "name": "conjugate_enriched_state_gate",
            "role": "gate",
            "field": "conjugate_enriched_state",
            "kind": "value",
            "expected": "progenitor_exhausted_TCF7_pos_stem_like"
          },
          {
            "name": "cf_circular_exhaustion_criterion",
            "role": "hard_fail",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "exhaustion_signature_intensity"
          },
          {
            "name": "cf_marker_sort_prior",
            "role": "hard_fail",
            "field": "best_selection_marker",
            "kind": "value",
            "op": "in",
            "expected": [
              "CD39",
              "PD1",
              "PD1_high",
              "TIM3",
              "PD1_CD39",
              "CD39_CD103"
            ]
          },
          {
            "name": "cf_single_cell_capture_prior",
            "role": "hard_fail",
            "field": "best_capture_approach",
            "kind": "value",
            "op": "in",
            "expected": [
              "dissociated_single_cell",
              "dissociated_bulk_population"
            ]
          },
          {
            "name": "cf_off_data_capture_gate",
            "role": "hard_fail",
            "field": "$.population_findings[*].capture_gate",
            "kind": "avoid",
            "expected": [
              "dissociated_bulk",
              "matched_peripheral_blood",
              "matched_lymph_node",
              "post_hoc_expanded_clonotype_group"
            ]
          },
          {
            "name": "cf_conjugate_mischaracterized",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_marker_singlet_called_engaged",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "contains",
            "matchKey": "capture_gate",
            "entity": "marker_sorted_singlet",
            "valueField": "reactivity_evidence",
            "expected": [
              "engagement_and_clonal_expansion"
            ]
          },
          {
            "name": "cf_no_engaged_reactive_population",
            "role": "hard_fail",
            "field": "$.population_findings[*].reactivity_evidence",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_marker_captures_all_reactive",
            "role": "hard_fail",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "captures_full_reactive_repertoire"
          },
          {
            "name": "cf_superiority_overclaimed_from_phenotype",
            "role": "hard_fail",
            "field": "superiority_claim_basis",
            "kind": "value",
            "op": "in",
            "expected": [
              "established_by_scrna_phenotype",
              "established_by_reactivity_proxy_scores"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-4-6-pi",
      "modelName": "claude-sonnet-4-6",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_vs_cd39_tumor_reactivity_dominance",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1330,
      "cost": 1.0905048,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "population_findings",
            "kind": "list",
            "matchKey": "capture_gate",
            "k": 4,
            "expected": [
              {
                "key": "tumor_cell_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "apc_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "combined_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "unselected_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "naive",
                      "central_memory",
                      "effector_memory"
                    ],
                    "canonical": "naive"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "low"
                    ],
                    "canonical": "low"
                  },
                  "purity_caveat": {
                    "op": "equals",
                    "values": [
                      "pure_T_cell"
                    ],
                    "canonical": "pure_T_cell"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "no_reactivity_evidence"
                    ],
                    "canonical": "no_reactivity_evidence"
                  }
                }
              },
              {
                "key": "marker_sorted_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "equals",
                    "values": [
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "terminal_exhausted_LAG3_high"
                  },
                  "clonal_expansion_level": {
                    "op": "in",
                    "values": [
                      "high",
                      "intermediate"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "marker_on_non_T_lineages",
                      "single_cell_gating_loses_target"
                    ],
                    "canonical": "marker_on_non_T_lineages"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "marker_or_exhaustion_defined_only"
                    ],
                    "canonical": "marker_or_exhaustion_defined_only"
                  }
                }
              }
            ]
          },
          {
            "name": "best_capture_approach_gate",
            "role": "gate",
            "field": "best_capture_approach",
            "kind": "value",
            "expected": "heterotypic_cell_conjugate"
          },
          {
            "name": "best_selection_marker_gate",
            "role": "gate",
            "field": "best_selection_marker",
            "kind": "value",
            "expected": "none"
          },
          {
            "name": "reactivity_criterion_gate",
            "role": "gate",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "clonal_expansion_and_physical_engagement"
          },
          {
            "name": "cross_patient_robustness_gate",
            "role": "gate",
            "field": "cross_patient_robustness",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_across_patients",
              "marginal_in_some_patients"
            ]
          },
          {
            "name": "marker_selection_consequence_gate",
            "role": "gate",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "discards_clonally_expanded_reactive_cells"
          },
          {
            "name": "superiority_claim_basis_gate",
            "role": "gate",
            "field": "superiority_claim_basis",
            "kind": "value",
            "expected": "requires_functional_reactivity_assay"
          },
          {
            "name": "conjugate_enriched_state_gate",
            "role": "gate",
            "field": "conjugate_enriched_state",
            "kind": "value",
            "expected": "progenitor_exhausted_TCF7_pos_stem_like"
          },
          {
            "name": "cf_circular_exhaustion_criterion",
            "role": "hard_fail",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "exhaustion_signature_intensity"
          },
          {
            "name": "cf_marker_sort_prior",
            "role": "hard_fail",
            "field": "best_selection_marker",
            "kind": "value",
            "op": "in",
            "expected": [
              "CD39",
              "PD1",
              "PD1_high",
              "TIM3",
              "PD1_CD39",
              "CD39_CD103"
            ]
          },
          {
            "name": "cf_single_cell_capture_prior",
            "role": "hard_fail",
            "field": "best_capture_approach",
            "kind": "value",
            "op": "in",
            "expected": [
              "dissociated_single_cell",
              "dissociated_bulk_population"
            ]
          },
          {
            "name": "cf_off_data_capture_gate",
            "role": "hard_fail",
            "field": "$.population_findings[*].capture_gate",
            "kind": "avoid",
            "expected": [
              "dissociated_bulk",
              "matched_peripheral_blood",
              "matched_lymph_node",
              "post_hoc_expanded_clonotype_group"
            ]
          },
          {
            "name": "cf_conjugate_mischaracterized",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_marker_singlet_called_engaged",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "contains",
            "matchKey": "capture_gate",
            "entity": "marker_sorted_singlet",
            "valueField": "reactivity_evidence",
            "expected": [
              "engagement_and_clonal_expansion"
            ]
          },
          {
            "name": "cf_no_engaged_reactive_population",
            "role": "hard_fail",
            "field": "$.population_findings[*].reactivity_evidence",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_marker_captures_all_reactive",
            "role": "hard_fail",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "captures_full_reactive_repertoire"
          },
          {
            "name": "cf_superiority_overclaimed_from_phenotype",
            "role": "hard_fail",
            "field": "superiority_claim_basis",
            "kind": "value",
            "op": "in",
            "expected": [
              "established_by_scrna_phenotype",
              "established_by_reactivity_proxy_scores"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gemini-3-1-pro-preview-pi",
      "modelName": "gemini-3.1-pro-preview",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_vs_cd39_tumor_reactivity_dominance",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 647,
      "cost": 0.6965042,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "population_findings",
            "kind": "list",
            "matchKey": "capture_gate",
            "k": 4,
            "expected": [
              {
                "key": "tumor_cell_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "apc_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "combined_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "unselected_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "naive",
                      "central_memory",
                      "effector_memory"
                    ],
                    "canonical": "naive"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "low"
                    ],
                    "canonical": "low"
                  },
                  "purity_caveat": {
                    "op": "equals",
                    "values": [
                      "pure_T_cell"
                    ],
                    "canonical": "pure_T_cell"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "no_reactivity_evidence"
                    ],
                    "canonical": "no_reactivity_evidence"
                  }
                }
              },
              {
                "key": "marker_sorted_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "equals",
                    "values": [
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "terminal_exhausted_LAG3_high"
                  },
                  "clonal_expansion_level": {
                    "op": "in",
                    "values": [
                      "high",
                      "intermediate"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "marker_on_non_T_lineages",
                      "single_cell_gating_loses_target"
                    ],
                    "canonical": "marker_on_non_T_lineages"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "marker_or_exhaustion_defined_only"
                    ],
                    "canonical": "marker_or_exhaustion_defined_only"
                  }
                }
              }
            ]
          },
          {
            "name": "best_capture_approach_gate",
            "role": "gate",
            "field": "best_capture_approach",
            "kind": "value",
            "expected": "heterotypic_cell_conjugate"
          },
          {
            "name": "best_selection_marker_gate",
            "role": "gate",
            "field": "best_selection_marker",
            "kind": "value",
            "expected": "none"
          },
          {
            "name": "reactivity_criterion_gate",
            "role": "gate",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "clonal_expansion_and_physical_engagement"
          },
          {
            "name": "cross_patient_robustness_gate",
            "role": "gate",
            "field": "cross_patient_robustness",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_across_patients",
              "marginal_in_some_patients"
            ]
          },
          {
            "name": "marker_selection_consequence_gate",
            "role": "gate",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "discards_clonally_expanded_reactive_cells"
          },
          {
            "name": "superiority_claim_basis_gate",
            "role": "gate",
            "field": "superiority_claim_basis",
            "kind": "value",
            "expected": "requires_functional_reactivity_assay"
          },
          {
            "name": "conjugate_enriched_state_gate",
            "role": "gate",
            "field": "conjugate_enriched_state",
            "kind": "value",
            "expected": "progenitor_exhausted_TCF7_pos_stem_like"
          },
          {
            "name": "cf_circular_exhaustion_criterion",
            "role": "hard_fail",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "exhaustion_signature_intensity"
          },
          {
            "name": "cf_marker_sort_prior",
            "role": "hard_fail",
            "field": "best_selection_marker",
            "kind": "value",
            "op": "in",
            "expected": [
              "CD39",
              "PD1",
              "PD1_high",
              "TIM3",
              "PD1_CD39",
              "CD39_CD103"
            ]
          },
          {
            "name": "cf_single_cell_capture_prior",
            "role": "hard_fail",
            "field": "best_capture_approach",
            "kind": "value",
            "op": "in",
            "expected": [
              "dissociated_single_cell",
              "dissociated_bulk_population"
            ]
          },
          {
            "name": "cf_off_data_capture_gate",
            "role": "hard_fail",
            "field": "$.population_findings[*].capture_gate",
            "kind": "avoid",
            "expected": [
              "dissociated_bulk",
              "matched_peripheral_blood",
              "matched_lymph_node",
              "post_hoc_expanded_clonotype_group"
            ]
          },
          {
            "name": "cf_conjugate_mischaracterized",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_marker_singlet_called_engaged",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "contains",
            "matchKey": "capture_gate",
            "entity": "marker_sorted_singlet",
            "valueField": "reactivity_evidence",
            "expected": [
              "engagement_and_clonal_expansion"
            ]
          },
          {
            "name": "cf_no_engaged_reactive_population",
            "role": "hard_fail",
            "field": "$.population_findings[*].reactivity_evidence",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_marker_captures_all_reactive",
            "role": "hard_fail",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "captures_full_reactive_repertoire"
          },
          {
            "name": "cf_superiority_overclaimed_from_phenotype",
            "role": "hard_fail",
            "field": "superiority_claim_basis",
            "kind": "value",
            "op": "in",
            "expected": [
              "established_by_scrna_phenotype",
              "established_by_reactivity_proxy_scores"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gemini-3-1-pro-preview-pi",
      "modelName": "gemini-3.1-pro-preview",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_vs_cd39_tumor_reactivity_dominance",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 1034,
      "cost": 0.7521934000000001,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "population_findings",
            "kind": "list",
            "matchKey": "capture_gate",
            "k": 4,
            "expected": [
              {
                "key": "tumor_cell_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "apc_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "combined_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "unselected_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "naive",
                      "central_memory",
                      "effector_memory"
                    ],
                    "canonical": "naive"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "low"
                    ],
                    "canonical": "low"
                  },
                  "purity_caveat": {
                    "op": "equals",
                    "values": [
                      "pure_T_cell"
                    ],
                    "canonical": "pure_T_cell"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "no_reactivity_evidence"
                    ],
                    "canonical": "no_reactivity_evidence"
                  }
                }
              },
              {
                "key": "marker_sorted_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "equals",
                    "values": [
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "terminal_exhausted_LAG3_high"
                  },
                  "clonal_expansion_level": {
                    "op": "in",
                    "values": [
                      "high",
                      "intermediate"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "marker_on_non_T_lineages",
                      "single_cell_gating_loses_target"
                    ],
                    "canonical": "marker_on_non_T_lineages"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "marker_or_exhaustion_defined_only"
                    ],
                    "canonical": "marker_or_exhaustion_defined_only"
                  }
                }
              }
            ]
          },
          {
            "name": "best_capture_approach_gate",
            "role": "gate",
            "field": "best_capture_approach",
            "kind": "value",
            "expected": "heterotypic_cell_conjugate"
          },
          {
            "name": "best_selection_marker_gate",
            "role": "gate",
            "field": "best_selection_marker",
            "kind": "value",
            "expected": "none"
          },
          {
            "name": "reactivity_criterion_gate",
            "role": "gate",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "clonal_expansion_and_physical_engagement"
          },
          {
            "name": "cross_patient_robustness_gate",
            "role": "gate",
            "field": "cross_patient_robustness",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_across_patients",
              "marginal_in_some_patients"
            ]
          },
          {
            "name": "marker_selection_consequence_gate",
            "role": "gate",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "discards_clonally_expanded_reactive_cells"
          },
          {
            "name": "superiority_claim_basis_gate",
            "role": "gate",
            "field": "superiority_claim_basis",
            "kind": "value",
            "expected": "requires_functional_reactivity_assay"
          },
          {
            "name": "conjugate_enriched_state_gate",
            "role": "gate",
            "field": "conjugate_enriched_state",
            "kind": "value",
            "expected": "progenitor_exhausted_TCF7_pos_stem_like"
          },
          {
            "name": "cf_circular_exhaustion_criterion",
            "role": "hard_fail",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "exhaustion_signature_intensity"
          },
          {
            "name": "cf_marker_sort_prior",
            "role": "hard_fail",
            "field": "best_selection_marker",
            "kind": "value",
            "op": "in",
            "expected": [
              "CD39",
              "PD1",
              "PD1_high",
              "TIM3",
              "PD1_CD39",
              "CD39_CD103"
            ]
          },
          {
            "name": "cf_single_cell_capture_prior",
            "role": "hard_fail",
            "field": "best_capture_approach",
            "kind": "value",
            "op": "in",
            "expected": [
              "dissociated_single_cell",
              "dissociated_bulk_population"
            ]
          },
          {
            "name": "cf_off_data_capture_gate",
            "role": "hard_fail",
            "field": "$.population_findings[*].capture_gate",
            "kind": "avoid",
            "expected": [
              "dissociated_bulk",
              "matched_peripheral_blood",
              "matched_lymph_node",
              "post_hoc_expanded_clonotype_group"
            ]
          },
          {
            "name": "cf_conjugate_mischaracterized",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_marker_singlet_called_engaged",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "contains",
            "matchKey": "capture_gate",
            "entity": "marker_sorted_singlet",
            "valueField": "reactivity_evidence",
            "expected": [
              "engagement_and_clonal_expansion"
            ]
          },
          {
            "name": "cf_no_engaged_reactive_population",
            "role": "hard_fail",
            "field": "$.population_findings[*].reactivity_evidence",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_marker_captures_all_reactive",
            "role": "hard_fail",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "captures_full_reactive_repertoire"
          },
          {
            "name": "cf_superiority_overclaimed_from_phenotype",
            "role": "hard_fail",
            "field": "superiority_claim_basis",
            "kind": "value",
            "op": "in",
            "expected": [
              "established_by_scrna_phenotype",
              "established_by_reactivity_proxy_scores"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gemini-3-1-pro-preview-pi",
      "modelName": "gemini-3.1-pro-preview",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_vs_cd39_tumor_reactivity_dominance",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 707,
      "cost": 0.5268728,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "population_findings",
            "kind": "list",
            "matchKey": "capture_gate",
            "k": 4,
            "expected": [
              {
                "key": "tumor_cell_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "apc_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "combined_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "unselected_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "naive",
                      "central_memory",
                      "effector_memory"
                    ],
                    "canonical": "naive"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "low"
                    ],
                    "canonical": "low"
                  },
                  "purity_caveat": {
                    "op": "equals",
                    "values": [
                      "pure_T_cell"
                    ],
                    "canonical": "pure_T_cell"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "no_reactivity_evidence"
                    ],
                    "canonical": "no_reactivity_evidence"
                  }
                }
              },
              {
                "key": "marker_sorted_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "equals",
                    "values": [
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "terminal_exhausted_LAG3_high"
                  },
                  "clonal_expansion_level": {
                    "op": "in",
                    "values": [
                      "high",
                      "intermediate"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "marker_on_non_T_lineages",
                      "single_cell_gating_loses_target"
                    ],
                    "canonical": "marker_on_non_T_lineages"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "marker_or_exhaustion_defined_only"
                    ],
                    "canonical": "marker_or_exhaustion_defined_only"
                  }
                }
              }
            ]
          },
          {
            "name": "best_capture_approach_gate",
            "role": "gate",
            "field": "best_capture_approach",
            "kind": "value",
            "expected": "heterotypic_cell_conjugate"
          },
          {
            "name": "best_selection_marker_gate",
            "role": "gate",
            "field": "best_selection_marker",
            "kind": "value",
            "expected": "none"
          },
          {
            "name": "reactivity_criterion_gate",
            "role": "gate",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "clonal_expansion_and_physical_engagement"
          },
          {
            "name": "cross_patient_robustness_gate",
            "role": "gate",
            "field": "cross_patient_robustness",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_across_patients",
              "marginal_in_some_patients"
            ]
          },
          {
            "name": "marker_selection_consequence_gate",
            "role": "gate",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "discards_clonally_expanded_reactive_cells"
          },
          {
            "name": "superiority_claim_basis_gate",
            "role": "gate",
            "field": "superiority_claim_basis",
            "kind": "value",
            "expected": "requires_functional_reactivity_assay"
          },
          {
            "name": "conjugate_enriched_state_gate",
            "role": "gate",
            "field": "conjugate_enriched_state",
            "kind": "value",
            "expected": "progenitor_exhausted_TCF7_pos_stem_like"
          },
          {
            "name": "cf_circular_exhaustion_criterion",
            "role": "hard_fail",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "exhaustion_signature_intensity"
          },
          {
            "name": "cf_marker_sort_prior",
            "role": "hard_fail",
            "field": "best_selection_marker",
            "kind": "value",
            "op": "in",
            "expected": [
              "CD39",
              "PD1",
              "PD1_high",
              "TIM3",
              "PD1_CD39",
              "CD39_CD103"
            ]
          },
          {
            "name": "cf_single_cell_capture_prior",
            "role": "hard_fail",
            "field": "best_capture_approach",
            "kind": "value",
            "op": "in",
            "expected": [
              "dissociated_single_cell",
              "dissociated_bulk_population"
            ]
          },
          {
            "name": "cf_off_data_capture_gate",
            "role": "hard_fail",
            "field": "$.population_findings[*].capture_gate",
            "kind": "avoid",
            "expected": [
              "dissociated_bulk",
              "matched_peripheral_blood",
              "matched_lymph_node",
              "post_hoc_expanded_clonotype_group"
            ]
          },
          {
            "name": "cf_conjugate_mischaracterized",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_marker_singlet_called_engaged",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "contains",
            "matchKey": "capture_gate",
            "entity": "marker_sorted_singlet",
            "valueField": "reactivity_evidence",
            "expected": [
              "engagement_and_clonal_expansion"
            ]
          },
          {
            "name": "cf_no_engaged_reactive_population",
            "role": "hard_fail",
            "field": "$.population_findings[*].reactivity_evidence",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_marker_captures_all_reactive",
            "role": "hard_fail",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "captures_full_reactive_repertoire"
          },
          {
            "name": "cf_superiority_overclaimed_from_phenotype",
            "role": "hard_fail",
            "field": "superiority_claim_basis",
            "kind": "value",
            "op": "in",
            "expected": [
              "established_by_scrna_phenotype",
              "established_by_reactivity_proxy_scores"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gemini-3-5-flash-pi",
      "modelName": "gemini-3.5-flash",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_vs_cd39_tumor_reactivity_dominance",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 827,
      "cost": 1.358328,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "population_findings",
            "kind": "list",
            "matchKey": "capture_gate",
            "k": 4,
            "expected": [
              {
                "key": "tumor_cell_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "apc_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "combined_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "unselected_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "naive",
                      "central_memory",
                      "effector_memory"
                    ],
                    "canonical": "naive"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "low"
                    ],
                    "canonical": "low"
                  },
                  "purity_caveat": {
                    "op": "equals",
                    "values": [
                      "pure_T_cell"
                    ],
                    "canonical": "pure_T_cell"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "no_reactivity_evidence"
                    ],
                    "canonical": "no_reactivity_evidence"
                  }
                }
              },
              {
                "key": "marker_sorted_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "equals",
                    "values": [
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "terminal_exhausted_LAG3_high"
                  },
                  "clonal_expansion_level": {
                    "op": "in",
                    "values": [
                      "high",
                      "intermediate"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "marker_on_non_T_lineages",
                      "single_cell_gating_loses_target"
                    ],
                    "canonical": "marker_on_non_T_lineages"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "marker_or_exhaustion_defined_only"
                    ],
                    "canonical": "marker_or_exhaustion_defined_only"
                  }
                }
              }
            ]
          },
          {
            "name": "best_capture_approach_gate",
            "role": "gate",
            "field": "best_capture_approach",
            "kind": "value",
            "expected": "heterotypic_cell_conjugate"
          },
          {
            "name": "best_selection_marker_gate",
            "role": "gate",
            "field": "best_selection_marker",
            "kind": "value",
            "expected": "none"
          },
          {
            "name": "reactivity_criterion_gate",
            "role": "gate",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "clonal_expansion_and_physical_engagement"
          },
          {
            "name": "cross_patient_robustness_gate",
            "role": "gate",
            "field": "cross_patient_robustness",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_across_patients",
              "marginal_in_some_patients"
            ]
          },
          {
            "name": "marker_selection_consequence_gate",
            "role": "gate",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "discards_clonally_expanded_reactive_cells"
          },
          {
            "name": "superiority_claim_basis_gate",
            "role": "gate",
            "field": "superiority_claim_basis",
            "kind": "value",
            "expected": "requires_functional_reactivity_assay"
          },
          {
            "name": "conjugate_enriched_state_gate",
            "role": "gate",
            "field": "conjugate_enriched_state",
            "kind": "value",
            "expected": "progenitor_exhausted_TCF7_pos_stem_like"
          },
          {
            "name": "cf_circular_exhaustion_criterion",
            "role": "hard_fail",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "exhaustion_signature_intensity"
          },
          {
            "name": "cf_marker_sort_prior",
            "role": "hard_fail",
            "field": "best_selection_marker",
            "kind": "value",
            "op": "in",
            "expected": [
              "CD39",
              "PD1",
              "PD1_high",
              "TIM3",
              "PD1_CD39",
              "CD39_CD103"
            ]
          },
          {
            "name": "cf_single_cell_capture_prior",
            "role": "hard_fail",
            "field": "best_capture_approach",
            "kind": "value",
            "op": "in",
            "expected": [
              "dissociated_single_cell",
              "dissociated_bulk_population"
            ]
          },
          {
            "name": "cf_off_data_capture_gate",
            "role": "hard_fail",
            "field": "$.population_findings[*].capture_gate",
            "kind": "avoid",
            "expected": [
              "dissociated_bulk",
              "matched_peripheral_blood",
              "matched_lymph_node",
              "post_hoc_expanded_clonotype_group"
            ]
          },
          {
            "name": "cf_conjugate_mischaracterized",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_marker_singlet_called_engaged",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "contains",
            "matchKey": "capture_gate",
            "entity": "marker_sorted_singlet",
            "valueField": "reactivity_evidence",
            "expected": [
              "engagement_and_clonal_expansion"
            ]
          },
          {
            "name": "cf_no_engaged_reactive_population",
            "role": "hard_fail",
            "field": "$.population_findings[*].reactivity_evidence",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_marker_captures_all_reactive",
            "role": "hard_fail",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "captures_full_reactive_repertoire"
          },
          {
            "name": "cf_superiority_overclaimed_from_phenotype",
            "role": "hard_fail",
            "field": "superiority_claim_basis",
            "kind": "value",
            "op": "in",
            "expected": [
              "established_by_scrna_phenotype",
              "established_by_reactivity_proxy_scores"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gemini-3-5-flash-pi",
      "modelName": "gemini-3.5-flash",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_vs_cd39_tumor_reactivity_dominance",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 1009,
      "cost": 1.3579702500000002,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "population_findings",
            "kind": "list",
            "matchKey": "capture_gate",
            "k": 4,
            "expected": [
              {
                "key": "tumor_cell_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "apc_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "combined_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "unselected_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "naive",
                      "central_memory",
                      "effector_memory"
                    ],
                    "canonical": "naive"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "low"
                    ],
                    "canonical": "low"
                  },
                  "purity_caveat": {
                    "op": "equals",
                    "values": [
                      "pure_T_cell"
                    ],
                    "canonical": "pure_T_cell"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "no_reactivity_evidence"
                    ],
                    "canonical": "no_reactivity_evidence"
                  }
                }
              },
              {
                "key": "marker_sorted_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "equals",
                    "values": [
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "terminal_exhausted_LAG3_high"
                  },
                  "clonal_expansion_level": {
                    "op": "in",
                    "values": [
                      "high",
                      "intermediate"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "marker_on_non_T_lineages",
                      "single_cell_gating_loses_target"
                    ],
                    "canonical": "marker_on_non_T_lineages"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "marker_or_exhaustion_defined_only"
                    ],
                    "canonical": "marker_or_exhaustion_defined_only"
                  }
                }
              }
            ]
          },
          {
            "name": "best_capture_approach_gate",
            "role": "gate",
            "field": "best_capture_approach",
            "kind": "value",
            "expected": "heterotypic_cell_conjugate"
          },
          {
            "name": "best_selection_marker_gate",
            "role": "gate",
            "field": "best_selection_marker",
            "kind": "value",
            "expected": "none"
          },
          {
            "name": "reactivity_criterion_gate",
            "role": "gate",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "clonal_expansion_and_physical_engagement"
          },
          {
            "name": "cross_patient_robustness_gate",
            "role": "gate",
            "field": "cross_patient_robustness",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_across_patients",
              "marginal_in_some_patients"
            ]
          },
          {
            "name": "marker_selection_consequence_gate",
            "role": "gate",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "discards_clonally_expanded_reactive_cells"
          },
          {
            "name": "superiority_claim_basis_gate",
            "role": "gate",
            "field": "superiority_claim_basis",
            "kind": "value",
            "expected": "requires_functional_reactivity_assay"
          },
          {
            "name": "conjugate_enriched_state_gate",
            "role": "gate",
            "field": "conjugate_enriched_state",
            "kind": "value",
            "expected": "progenitor_exhausted_TCF7_pos_stem_like"
          },
          {
            "name": "cf_circular_exhaustion_criterion",
            "role": "hard_fail",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "exhaustion_signature_intensity"
          },
          {
            "name": "cf_marker_sort_prior",
            "role": "hard_fail",
            "field": "best_selection_marker",
            "kind": "value",
            "op": "in",
            "expected": [
              "CD39",
              "PD1",
              "PD1_high",
              "TIM3",
              "PD1_CD39",
              "CD39_CD103"
            ]
          },
          {
            "name": "cf_single_cell_capture_prior",
            "role": "hard_fail",
            "field": "best_capture_approach",
            "kind": "value",
            "op": "in",
            "expected": [
              "dissociated_single_cell",
              "dissociated_bulk_population"
            ]
          },
          {
            "name": "cf_off_data_capture_gate",
            "role": "hard_fail",
            "field": "$.population_findings[*].capture_gate",
            "kind": "avoid",
            "expected": [
              "dissociated_bulk",
              "matched_peripheral_blood",
              "matched_lymph_node",
              "post_hoc_expanded_clonotype_group"
            ]
          },
          {
            "name": "cf_conjugate_mischaracterized",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_marker_singlet_called_engaged",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "contains",
            "matchKey": "capture_gate",
            "entity": "marker_sorted_singlet",
            "valueField": "reactivity_evidence",
            "expected": [
              "engagement_and_clonal_expansion"
            ]
          },
          {
            "name": "cf_no_engaged_reactive_population",
            "role": "hard_fail",
            "field": "$.population_findings[*].reactivity_evidence",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_marker_captures_all_reactive",
            "role": "hard_fail",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "captures_full_reactive_repertoire"
          },
          {
            "name": "cf_superiority_overclaimed_from_phenotype",
            "role": "hard_fail",
            "field": "superiority_claim_basis",
            "kind": "value",
            "op": "in",
            "expected": [
              "established_by_scrna_phenotype",
              "established_by_reactivity_proxy_scores"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gemini-3-5-flash-pi",
      "modelName": "gemini-3.5-flash",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_vs_cd39_tumor_reactivity_dominance",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 783,
      "cost": 1.0402579499999995,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "population_findings",
            "kind": "list",
            "matchKey": "capture_gate",
            "k": 4,
            "expected": [
              {
                "key": "tumor_cell_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "apc_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "combined_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "unselected_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "naive",
                      "central_memory",
                      "effector_memory"
                    ],
                    "canonical": "naive"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "low"
                    ],
                    "canonical": "low"
                  },
                  "purity_caveat": {
                    "op": "equals",
                    "values": [
                      "pure_T_cell"
                    ],
                    "canonical": "pure_T_cell"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "no_reactivity_evidence"
                    ],
                    "canonical": "no_reactivity_evidence"
                  }
                }
              },
              {
                "key": "marker_sorted_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "equals",
                    "values": [
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "terminal_exhausted_LAG3_high"
                  },
                  "clonal_expansion_level": {
                    "op": "in",
                    "values": [
                      "high",
                      "intermediate"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "marker_on_non_T_lineages",
                      "single_cell_gating_loses_target"
                    ],
                    "canonical": "marker_on_non_T_lineages"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "marker_or_exhaustion_defined_only"
                    ],
                    "canonical": "marker_or_exhaustion_defined_only"
                  }
                }
              }
            ]
          },
          {
            "name": "best_capture_approach_gate",
            "role": "gate",
            "field": "best_capture_approach",
            "kind": "value",
            "expected": "heterotypic_cell_conjugate"
          },
          {
            "name": "best_selection_marker_gate",
            "role": "gate",
            "field": "best_selection_marker",
            "kind": "value",
            "expected": "none"
          },
          {
            "name": "reactivity_criterion_gate",
            "role": "gate",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "clonal_expansion_and_physical_engagement"
          },
          {
            "name": "cross_patient_robustness_gate",
            "role": "gate",
            "field": "cross_patient_robustness",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_across_patients",
              "marginal_in_some_patients"
            ]
          },
          {
            "name": "marker_selection_consequence_gate",
            "role": "gate",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "discards_clonally_expanded_reactive_cells"
          },
          {
            "name": "superiority_claim_basis_gate",
            "role": "gate",
            "field": "superiority_claim_basis",
            "kind": "value",
            "expected": "requires_functional_reactivity_assay"
          },
          {
            "name": "conjugate_enriched_state_gate",
            "role": "gate",
            "field": "conjugate_enriched_state",
            "kind": "value",
            "expected": "progenitor_exhausted_TCF7_pos_stem_like"
          },
          {
            "name": "cf_circular_exhaustion_criterion",
            "role": "hard_fail",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "exhaustion_signature_intensity"
          },
          {
            "name": "cf_marker_sort_prior",
            "role": "hard_fail",
            "field": "best_selection_marker",
            "kind": "value",
            "op": "in",
            "expected": [
              "CD39",
              "PD1",
              "PD1_high",
              "TIM3",
              "PD1_CD39",
              "CD39_CD103"
            ]
          },
          {
            "name": "cf_single_cell_capture_prior",
            "role": "hard_fail",
            "field": "best_capture_approach",
            "kind": "value",
            "op": "in",
            "expected": [
              "dissociated_single_cell",
              "dissociated_bulk_population"
            ]
          },
          {
            "name": "cf_off_data_capture_gate",
            "role": "hard_fail",
            "field": "$.population_findings[*].capture_gate",
            "kind": "avoid",
            "expected": [
              "dissociated_bulk",
              "matched_peripheral_blood",
              "matched_lymph_node",
              "post_hoc_expanded_clonotype_group"
            ]
          },
          {
            "name": "cf_conjugate_mischaracterized",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_marker_singlet_called_engaged",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "contains",
            "matchKey": "capture_gate",
            "entity": "marker_sorted_singlet",
            "valueField": "reactivity_evidence",
            "expected": [
              "engagement_and_clonal_expansion"
            ]
          },
          {
            "name": "cf_no_engaged_reactive_population",
            "role": "hard_fail",
            "field": "$.population_findings[*].reactivity_evidence",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_marker_captures_all_reactive",
            "role": "hard_fail",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "captures_full_reactive_repertoire"
          },
          {
            "name": "cf_superiority_overclaimed_from_phenotype",
            "role": "hard_fail",
            "field": "superiority_claim_basis",
            "kind": "value",
            "op": "in",
            "expected": [
              "established_by_scrna_phenotype",
              "established_by_reactivity_proxy_scores"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-glm-5p2-pi",
      "modelName": "glm-5p2",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_vs_cd39_tumor_reactivity_dominance",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 2130,
      "cost": 2.1004082399999997,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "population_findings",
            "kind": "list",
            "matchKey": "capture_gate",
            "k": 4,
            "expected": [
              {
                "key": "tumor_cell_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "apc_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "combined_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "unselected_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "naive",
                      "central_memory",
                      "effector_memory"
                    ],
                    "canonical": "naive"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "low"
                    ],
                    "canonical": "low"
                  },
                  "purity_caveat": {
                    "op": "equals",
                    "values": [
                      "pure_T_cell"
                    ],
                    "canonical": "pure_T_cell"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "no_reactivity_evidence"
                    ],
                    "canonical": "no_reactivity_evidence"
                  }
                }
              },
              {
                "key": "marker_sorted_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "equals",
                    "values": [
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "terminal_exhausted_LAG3_high"
                  },
                  "clonal_expansion_level": {
                    "op": "in",
                    "values": [
                      "high",
                      "intermediate"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "marker_on_non_T_lineages",
                      "single_cell_gating_loses_target"
                    ],
                    "canonical": "marker_on_non_T_lineages"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "marker_or_exhaustion_defined_only"
                    ],
                    "canonical": "marker_or_exhaustion_defined_only"
                  }
                }
              }
            ]
          },
          {
            "name": "best_capture_approach_gate",
            "role": "gate",
            "field": "best_capture_approach",
            "kind": "value",
            "expected": "heterotypic_cell_conjugate"
          },
          {
            "name": "best_selection_marker_gate",
            "role": "gate",
            "field": "best_selection_marker",
            "kind": "value",
            "expected": "none"
          },
          {
            "name": "reactivity_criterion_gate",
            "role": "gate",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "clonal_expansion_and_physical_engagement"
          },
          {
            "name": "cross_patient_robustness_gate",
            "role": "gate",
            "field": "cross_patient_robustness",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_across_patients",
              "marginal_in_some_patients"
            ]
          },
          {
            "name": "marker_selection_consequence_gate",
            "role": "gate",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "discards_clonally_expanded_reactive_cells"
          },
          {
            "name": "superiority_claim_basis_gate",
            "role": "gate",
            "field": "superiority_claim_basis",
            "kind": "value",
            "expected": "requires_functional_reactivity_assay"
          },
          {
            "name": "conjugate_enriched_state_gate",
            "role": "gate",
            "field": "conjugate_enriched_state",
            "kind": "value",
            "expected": "progenitor_exhausted_TCF7_pos_stem_like"
          },
          {
            "name": "cf_circular_exhaustion_criterion",
            "role": "hard_fail",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "exhaustion_signature_intensity"
          },
          {
            "name": "cf_marker_sort_prior",
            "role": "hard_fail",
            "field": "best_selection_marker",
            "kind": "value",
            "op": "in",
            "expected": [
              "CD39",
              "PD1",
              "PD1_high",
              "TIM3",
              "PD1_CD39",
              "CD39_CD103"
            ]
          },
          {
            "name": "cf_single_cell_capture_prior",
            "role": "hard_fail",
            "field": "best_capture_approach",
            "kind": "value",
            "op": "in",
            "expected": [
              "dissociated_single_cell",
              "dissociated_bulk_population"
            ]
          },
          {
            "name": "cf_off_data_capture_gate",
            "role": "hard_fail",
            "field": "$.population_findings[*].capture_gate",
            "kind": "avoid",
            "expected": [
              "dissociated_bulk",
              "matched_peripheral_blood",
              "matched_lymph_node",
              "post_hoc_expanded_clonotype_group"
            ]
          },
          {
            "name": "cf_conjugate_mischaracterized",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_marker_singlet_called_engaged",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "contains",
            "matchKey": "capture_gate",
            "entity": "marker_sorted_singlet",
            "valueField": "reactivity_evidence",
            "expected": [
              "engagement_and_clonal_expansion"
            ]
          },
          {
            "name": "cf_no_engaged_reactive_population",
            "role": "hard_fail",
            "field": "$.population_findings[*].reactivity_evidence",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_marker_captures_all_reactive",
            "role": "hard_fail",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "captures_full_reactive_repertoire"
          },
          {
            "name": "cf_superiority_overclaimed_from_phenotype",
            "role": "hard_fail",
            "field": "superiority_claim_basis",
            "kind": "value",
            "op": "in",
            "expected": [
              "established_by_scrna_phenotype",
              "established_by_reactivity_proxy_scores"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-glm-5p2-pi",
      "modelName": "glm-5p2",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_vs_cd39_tumor_reactivity_dominance",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 1395,
      "cost": 1.0906125800000002,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "population_findings",
            "kind": "list",
            "matchKey": "capture_gate",
            "k": 4,
            "expected": [
              {
                "key": "tumor_cell_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "apc_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "combined_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "unselected_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "naive",
                      "central_memory",
                      "effector_memory"
                    ],
                    "canonical": "naive"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "low"
                    ],
                    "canonical": "low"
                  },
                  "purity_caveat": {
                    "op": "equals",
                    "values": [
                      "pure_T_cell"
                    ],
                    "canonical": "pure_T_cell"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "no_reactivity_evidence"
                    ],
                    "canonical": "no_reactivity_evidence"
                  }
                }
              },
              {
                "key": "marker_sorted_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "equals",
                    "values": [
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "terminal_exhausted_LAG3_high"
                  },
                  "clonal_expansion_level": {
                    "op": "in",
                    "values": [
                      "high",
                      "intermediate"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "marker_on_non_T_lineages",
                      "single_cell_gating_loses_target"
                    ],
                    "canonical": "marker_on_non_T_lineages"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "marker_or_exhaustion_defined_only"
                    ],
                    "canonical": "marker_or_exhaustion_defined_only"
                  }
                }
              }
            ]
          },
          {
            "name": "best_capture_approach_gate",
            "role": "gate",
            "field": "best_capture_approach",
            "kind": "value",
            "expected": "heterotypic_cell_conjugate"
          },
          {
            "name": "best_selection_marker_gate",
            "role": "gate",
            "field": "best_selection_marker",
            "kind": "value",
            "expected": "none"
          },
          {
            "name": "reactivity_criterion_gate",
            "role": "gate",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "clonal_expansion_and_physical_engagement"
          },
          {
            "name": "cross_patient_robustness_gate",
            "role": "gate",
            "field": "cross_patient_robustness",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_across_patients",
              "marginal_in_some_patients"
            ]
          },
          {
            "name": "marker_selection_consequence_gate",
            "role": "gate",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "discards_clonally_expanded_reactive_cells"
          },
          {
            "name": "superiority_claim_basis_gate",
            "role": "gate",
            "field": "superiority_claim_basis",
            "kind": "value",
            "expected": "requires_functional_reactivity_assay"
          },
          {
            "name": "conjugate_enriched_state_gate",
            "role": "gate",
            "field": "conjugate_enriched_state",
            "kind": "value",
            "expected": "progenitor_exhausted_TCF7_pos_stem_like"
          },
          {
            "name": "cf_circular_exhaustion_criterion",
            "role": "hard_fail",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "exhaustion_signature_intensity"
          },
          {
            "name": "cf_marker_sort_prior",
            "role": "hard_fail",
            "field": "best_selection_marker",
            "kind": "value",
            "op": "in",
            "expected": [
              "CD39",
              "PD1",
              "PD1_high",
              "TIM3",
              "PD1_CD39",
              "CD39_CD103"
            ]
          },
          {
            "name": "cf_single_cell_capture_prior",
            "role": "hard_fail",
            "field": "best_capture_approach",
            "kind": "value",
            "op": "in",
            "expected": [
              "dissociated_single_cell",
              "dissociated_bulk_population"
            ]
          },
          {
            "name": "cf_off_data_capture_gate",
            "role": "hard_fail",
            "field": "$.population_findings[*].capture_gate",
            "kind": "avoid",
            "expected": [
              "dissociated_bulk",
              "matched_peripheral_blood",
              "matched_lymph_node",
              "post_hoc_expanded_clonotype_group"
            ]
          },
          {
            "name": "cf_conjugate_mischaracterized",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_marker_singlet_called_engaged",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "contains",
            "matchKey": "capture_gate",
            "entity": "marker_sorted_singlet",
            "valueField": "reactivity_evidence",
            "expected": [
              "engagement_and_clonal_expansion"
            ]
          },
          {
            "name": "cf_no_engaged_reactive_population",
            "role": "hard_fail",
            "field": "$.population_findings[*].reactivity_evidence",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_marker_captures_all_reactive",
            "role": "hard_fail",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "captures_full_reactive_repertoire"
          },
          {
            "name": "cf_superiority_overclaimed_from_phenotype",
            "role": "hard_fail",
            "field": "superiority_claim_basis",
            "kind": "value",
            "op": "in",
            "expected": [
              "established_by_scrna_phenotype",
              "established_by_reactivity_proxy_scores"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-glm-5p2-pi",
      "modelName": "glm-5p2",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_vs_cd39_tumor_reactivity_dominance",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 2591,
      "cost": 2.2093003199999997,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "population_findings",
            "kind": "list",
            "matchKey": "capture_gate",
            "k": 4,
            "expected": [
              {
                "key": "tumor_cell_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "apc_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "combined_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "unselected_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "naive",
                      "central_memory",
                      "effector_memory"
                    ],
                    "canonical": "naive"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "low"
                    ],
                    "canonical": "low"
                  },
                  "purity_caveat": {
                    "op": "equals",
                    "values": [
                      "pure_T_cell"
                    ],
                    "canonical": "pure_T_cell"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "no_reactivity_evidence"
                    ],
                    "canonical": "no_reactivity_evidence"
                  }
                }
              },
              {
                "key": "marker_sorted_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "equals",
                    "values": [
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "terminal_exhausted_LAG3_high"
                  },
                  "clonal_expansion_level": {
                    "op": "in",
                    "values": [
                      "high",
                      "intermediate"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "marker_on_non_T_lineages",
                      "single_cell_gating_loses_target"
                    ],
                    "canonical": "marker_on_non_T_lineages"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "marker_or_exhaustion_defined_only"
                    ],
                    "canonical": "marker_or_exhaustion_defined_only"
                  }
                }
              }
            ]
          },
          {
            "name": "best_capture_approach_gate",
            "role": "gate",
            "field": "best_capture_approach",
            "kind": "value",
            "expected": "heterotypic_cell_conjugate"
          },
          {
            "name": "best_selection_marker_gate",
            "role": "gate",
            "field": "best_selection_marker",
            "kind": "value",
            "expected": "none"
          },
          {
            "name": "reactivity_criterion_gate",
            "role": "gate",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "clonal_expansion_and_physical_engagement"
          },
          {
            "name": "cross_patient_robustness_gate",
            "role": "gate",
            "field": "cross_patient_robustness",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_across_patients",
              "marginal_in_some_patients"
            ]
          },
          {
            "name": "marker_selection_consequence_gate",
            "role": "gate",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "discards_clonally_expanded_reactive_cells"
          },
          {
            "name": "superiority_claim_basis_gate",
            "role": "gate",
            "field": "superiority_claim_basis",
            "kind": "value",
            "expected": "requires_functional_reactivity_assay"
          },
          {
            "name": "conjugate_enriched_state_gate",
            "role": "gate",
            "field": "conjugate_enriched_state",
            "kind": "value",
            "expected": "progenitor_exhausted_TCF7_pos_stem_like"
          },
          {
            "name": "cf_circular_exhaustion_criterion",
            "role": "hard_fail",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "exhaustion_signature_intensity"
          },
          {
            "name": "cf_marker_sort_prior",
            "role": "hard_fail",
            "field": "best_selection_marker",
            "kind": "value",
            "op": "in",
            "expected": [
              "CD39",
              "PD1",
              "PD1_high",
              "TIM3",
              "PD1_CD39",
              "CD39_CD103"
            ]
          },
          {
            "name": "cf_single_cell_capture_prior",
            "role": "hard_fail",
            "field": "best_capture_approach",
            "kind": "value",
            "op": "in",
            "expected": [
              "dissociated_single_cell",
              "dissociated_bulk_population"
            ]
          },
          {
            "name": "cf_off_data_capture_gate",
            "role": "hard_fail",
            "field": "$.population_findings[*].capture_gate",
            "kind": "avoid",
            "expected": [
              "dissociated_bulk",
              "matched_peripheral_blood",
              "matched_lymph_node",
              "post_hoc_expanded_clonotype_group"
            ]
          },
          {
            "name": "cf_conjugate_mischaracterized",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_marker_singlet_called_engaged",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "contains",
            "matchKey": "capture_gate",
            "entity": "marker_sorted_singlet",
            "valueField": "reactivity_evidence",
            "expected": [
              "engagement_and_clonal_expansion"
            ]
          },
          {
            "name": "cf_no_engaged_reactive_population",
            "role": "hard_fail",
            "field": "$.population_findings[*].reactivity_evidence",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_marker_captures_all_reactive",
            "role": "hard_fail",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "captures_full_reactive_repertoire"
          },
          {
            "name": "cf_superiority_overclaimed_from_phenotype",
            "role": "hard_fail",
            "field": "superiority_claim_basis",
            "kind": "value",
            "op": "in",
            "expected": [
              "established_by_scrna_phenotype",
              "established_by_reactivity_proxy_scores"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-4-codex",
      "modelName": "gpt-5.4",
      "harness": "openai-codex",
      "task": "10x_5p_melanoma_cluster_vs_cd39_tumor_reactivity_dominance",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 690,
      "cost": 1.4831794999999999,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "population_findings",
            "kind": "list",
            "matchKey": "capture_gate",
            "k": 4,
            "expected": [
              {
                "key": "tumor_cell_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "apc_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "combined_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "unselected_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "naive",
                      "central_memory",
                      "effector_memory"
                    ],
                    "canonical": "naive"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "low"
                    ],
                    "canonical": "low"
                  },
                  "purity_caveat": {
                    "op": "equals",
                    "values": [
                      "pure_T_cell"
                    ],
                    "canonical": "pure_T_cell"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "no_reactivity_evidence"
                    ],
                    "canonical": "no_reactivity_evidence"
                  }
                }
              },
              {
                "key": "marker_sorted_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "equals",
                    "values": [
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "terminal_exhausted_LAG3_high"
                  },
                  "clonal_expansion_level": {
                    "op": "in",
                    "values": [
                      "high",
                      "intermediate"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "marker_on_non_T_lineages",
                      "single_cell_gating_loses_target"
                    ],
                    "canonical": "marker_on_non_T_lineages"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "marker_or_exhaustion_defined_only"
                    ],
                    "canonical": "marker_or_exhaustion_defined_only"
                  }
                }
              }
            ]
          },
          {
            "name": "best_capture_approach_gate",
            "role": "gate",
            "field": "best_capture_approach",
            "kind": "value",
            "expected": "heterotypic_cell_conjugate"
          },
          {
            "name": "best_selection_marker_gate",
            "role": "gate",
            "field": "best_selection_marker",
            "kind": "value",
            "expected": "none"
          },
          {
            "name": "reactivity_criterion_gate",
            "role": "gate",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "clonal_expansion_and_physical_engagement"
          },
          {
            "name": "cross_patient_robustness_gate",
            "role": "gate",
            "field": "cross_patient_robustness",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_across_patients",
              "marginal_in_some_patients"
            ]
          },
          {
            "name": "marker_selection_consequence_gate",
            "role": "gate",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "discards_clonally_expanded_reactive_cells"
          },
          {
            "name": "superiority_claim_basis_gate",
            "role": "gate",
            "field": "superiority_claim_basis",
            "kind": "value",
            "expected": "requires_functional_reactivity_assay"
          },
          {
            "name": "conjugate_enriched_state_gate",
            "role": "gate",
            "field": "conjugate_enriched_state",
            "kind": "value",
            "expected": "progenitor_exhausted_TCF7_pos_stem_like"
          },
          {
            "name": "cf_circular_exhaustion_criterion",
            "role": "hard_fail",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "exhaustion_signature_intensity"
          },
          {
            "name": "cf_marker_sort_prior",
            "role": "hard_fail",
            "field": "best_selection_marker",
            "kind": "value",
            "op": "in",
            "expected": [
              "CD39",
              "PD1",
              "PD1_high",
              "TIM3",
              "PD1_CD39",
              "CD39_CD103"
            ]
          },
          {
            "name": "cf_single_cell_capture_prior",
            "role": "hard_fail",
            "field": "best_capture_approach",
            "kind": "value",
            "op": "in",
            "expected": [
              "dissociated_single_cell",
              "dissociated_bulk_population"
            ]
          },
          {
            "name": "cf_off_data_capture_gate",
            "role": "hard_fail",
            "field": "$.population_findings[*].capture_gate",
            "kind": "avoid",
            "expected": [
              "dissociated_bulk",
              "matched_peripheral_blood",
              "matched_lymph_node",
              "post_hoc_expanded_clonotype_group"
            ]
          },
          {
            "name": "cf_conjugate_mischaracterized",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_marker_singlet_called_engaged",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "contains",
            "matchKey": "capture_gate",
            "entity": "marker_sorted_singlet",
            "valueField": "reactivity_evidence",
            "expected": [
              "engagement_and_clonal_expansion"
            ]
          },
          {
            "name": "cf_no_engaged_reactive_population",
            "role": "hard_fail",
            "field": "$.population_findings[*].reactivity_evidence",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_marker_captures_all_reactive",
            "role": "hard_fail",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "captures_full_reactive_repertoire"
          },
          {
            "name": "cf_superiority_overclaimed_from_phenotype",
            "role": "hard_fail",
            "field": "superiority_claim_basis",
            "kind": "value",
            "op": "in",
            "expected": [
              "established_by_scrna_phenotype",
              "established_by_reactivity_proxy_scores"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-4-codex",
      "modelName": "gpt-5.4",
      "harness": "openai-codex",
      "task": "10x_5p_melanoma_cluster_vs_cd39_tumor_reactivity_dominance",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 927,
      "cost": 1.7522915000000001,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "population_findings",
            "kind": "list",
            "matchKey": "capture_gate",
            "k": 4,
            "expected": [
              {
                "key": "tumor_cell_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "apc_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "combined_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "unselected_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "naive",
                      "central_memory",
                      "effector_memory"
                    ],
                    "canonical": "naive"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "low"
                    ],
                    "canonical": "low"
                  },
                  "purity_caveat": {
                    "op": "equals",
                    "values": [
                      "pure_T_cell"
                    ],
                    "canonical": "pure_T_cell"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "no_reactivity_evidence"
                    ],
                    "canonical": "no_reactivity_evidence"
                  }
                }
              },
              {
                "key": "marker_sorted_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "equals",
                    "values": [
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "terminal_exhausted_LAG3_high"
                  },
                  "clonal_expansion_level": {
                    "op": "in",
                    "values": [
                      "high",
                      "intermediate"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "marker_on_non_T_lineages",
                      "single_cell_gating_loses_target"
                    ],
                    "canonical": "marker_on_non_T_lineages"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "marker_or_exhaustion_defined_only"
                    ],
                    "canonical": "marker_or_exhaustion_defined_only"
                  }
                }
              }
            ]
          },
          {
            "name": "best_capture_approach_gate",
            "role": "gate",
            "field": "best_capture_approach",
            "kind": "value",
            "expected": "heterotypic_cell_conjugate"
          },
          {
            "name": "best_selection_marker_gate",
            "role": "gate",
            "field": "best_selection_marker",
            "kind": "value",
            "expected": "none"
          },
          {
            "name": "reactivity_criterion_gate",
            "role": "gate",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "clonal_expansion_and_physical_engagement"
          },
          {
            "name": "cross_patient_robustness_gate",
            "role": "gate",
            "field": "cross_patient_robustness",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_across_patients",
              "marginal_in_some_patients"
            ]
          },
          {
            "name": "marker_selection_consequence_gate",
            "role": "gate",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "discards_clonally_expanded_reactive_cells"
          },
          {
            "name": "superiority_claim_basis_gate",
            "role": "gate",
            "field": "superiority_claim_basis",
            "kind": "value",
            "expected": "requires_functional_reactivity_assay"
          },
          {
            "name": "conjugate_enriched_state_gate",
            "role": "gate",
            "field": "conjugate_enriched_state",
            "kind": "value",
            "expected": "progenitor_exhausted_TCF7_pos_stem_like"
          },
          {
            "name": "cf_circular_exhaustion_criterion",
            "role": "hard_fail",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "exhaustion_signature_intensity"
          },
          {
            "name": "cf_marker_sort_prior",
            "role": "hard_fail",
            "field": "best_selection_marker",
            "kind": "value",
            "op": "in",
            "expected": [
              "CD39",
              "PD1",
              "PD1_high",
              "TIM3",
              "PD1_CD39",
              "CD39_CD103"
            ]
          },
          {
            "name": "cf_single_cell_capture_prior",
            "role": "hard_fail",
            "field": "best_capture_approach",
            "kind": "value",
            "op": "in",
            "expected": [
              "dissociated_single_cell",
              "dissociated_bulk_population"
            ]
          },
          {
            "name": "cf_off_data_capture_gate",
            "role": "hard_fail",
            "field": "$.population_findings[*].capture_gate",
            "kind": "avoid",
            "expected": [
              "dissociated_bulk",
              "matched_peripheral_blood",
              "matched_lymph_node",
              "post_hoc_expanded_clonotype_group"
            ]
          },
          {
            "name": "cf_conjugate_mischaracterized",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_marker_singlet_called_engaged",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "contains",
            "matchKey": "capture_gate",
            "entity": "marker_sorted_singlet",
            "valueField": "reactivity_evidence",
            "expected": [
              "engagement_and_clonal_expansion"
            ]
          },
          {
            "name": "cf_no_engaged_reactive_population",
            "role": "hard_fail",
            "field": "$.population_findings[*].reactivity_evidence",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_marker_captures_all_reactive",
            "role": "hard_fail",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "captures_full_reactive_repertoire"
          },
          {
            "name": "cf_superiority_overclaimed_from_phenotype",
            "role": "hard_fail",
            "field": "superiority_claim_basis",
            "kind": "value",
            "op": "in",
            "expected": [
              "established_by_scrna_phenotype",
              "established_by_reactivity_proxy_scores"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-4-codex",
      "modelName": "gpt-5.4",
      "harness": "openai-codex",
      "task": "10x_5p_melanoma_cluster_vs_cd39_tumor_reactivity_dominance",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 836,
      "cost": 2.123499,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "population_findings",
            "kind": "list",
            "matchKey": "capture_gate",
            "k": 4,
            "expected": [
              {
                "key": "tumor_cell_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "apc_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "combined_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "unselected_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "naive",
                      "central_memory",
                      "effector_memory"
                    ],
                    "canonical": "naive"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "low"
                    ],
                    "canonical": "low"
                  },
                  "purity_caveat": {
                    "op": "equals",
                    "values": [
                      "pure_T_cell"
                    ],
                    "canonical": "pure_T_cell"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "no_reactivity_evidence"
                    ],
                    "canonical": "no_reactivity_evidence"
                  }
                }
              },
              {
                "key": "marker_sorted_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "equals",
                    "values": [
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "terminal_exhausted_LAG3_high"
                  },
                  "clonal_expansion_level": {
                    "op": "in",
                    "values": [
                      "high",
                      "intermediate"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "marker_on_non_T_lineages",
                      "single_cell_gating_loses_target"
                    ],
                    "canonical": "marker_on_non_T_lineages"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "marker_or_exhaustion_defined_only"
                    ],
                    "canonical": "marker_or_exhaustion_defined_only"
                  }
                }
              }
            ]
          },
          {
            "name": "best_capture_approach_gate",
            "role": "gate",
            "field": "best_capture_approach",
            "kind": "value",
            "expected": "heterotypic_cell_conjugate"
          },
          {
            "name": "best_selection_marker_gate",
            "role": "gate",
            "field": "best_selection_marker",
            "kind": "value",
            "expected": "none"
          },
          {
            "name": "reactivity_criterion_gate",
            "role": "gate",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "clonal_expansion_and_physical_engagement"
          },
          {
            "name": "cross_patient_robustness_gate",
            "role": "gate",
            "field": "cross_patient_robustness",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_across_patients",
              "marginal_in_some_patients"
            ]
          },
          {
            "name": "marker_selection_consequence_gate",
            "role": "gate",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "discards_clonally_expanded_reactive_cells"
          },
          {
            "name": "superiority_claim_basis_gate",
            "role": "gate",
            "field": "superiority_claim_basis",
            "kind": "value",
            "expected": "requires_functional_reactivity_assay"
          },
          {
            "name": "conjugate_enriched_state_gate",
            "role": "gate",
            "field": "conjugate_enriched_state",
            "kind": "value",
            "expected": "progenitor_exhausted_TCF7_pos_stem_like"
          },
          {
            "name": "cf_circular_exhaustion_criterion",
            "role": "hard_fail",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "exhaustion_signature_intensity"
          },
          {
            "name": "cf_marker_sort_prior",
            "role": "hard_fail",
            "field": "best_selection_marker",
            "kind": "value",
            "op": "in",
            "expected": [
              "CD39",
              "PD1",
              "PD1_high",
              "TIM3",
              "PD1_CD39",
              "CD39_CD103"
            ]
          },
          {
            "name": "cf_single_cell_capture_prior",
            "role": "hard_fail",
            "field": "best_capture_approach",
            "kind": "value",
            "op": "in",
            "expected": [
              "dissociated_single_cell",
              "dissociated_bulk_population"
            ]
          },
          {
            "name": "cf_off_data_capture_gate",
            "role": "hard_fail",
            "field": "$.population_findings[*].capture_gate",
            "kind": "avoid",
            "expected": [
              "dissociated_bulk",
              "matched_peripheral_blood",
              "matched_lymph_node",
              "post_hoc_expanded_clonotype_group"
            ]
          },
          {
            "name": "cf_conjugate_mischaracterized",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_marker_singlet_called_engaged",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "contains",
            "matchKey": "capture_gate",
            "entity": "marker_sorted_singlet",
            "valueField": "reactivity_evidence",
            "expected": [
              "engagement_and_clonal_expansion"
            ]
          },
          {
            "name": "cf_no_engaged_reactive_population",
            "role": "hard_fail",
            "field": "$.population_findings[*].reactivity_evidence",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_marker_captures_all_reactive",
            "role": "hard_fail",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "captures_full_reactive_repertoire"
          },
          {
            "name": "cf_superiority_overclaimed_from_phenotype",
            "role": "hard_fail",
            "field": "superiority_claim_basis",
            "kind": "value",
            "op": "in",
            "expected": [
              "established_by_scrna_phenotype",
              "established_by_reactivity_proxy_scores"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-4-pi",
      "modelName": "gpt-5.4",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_vs_cd39_tumor_reactivity_dominance",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 1941,
      "cost": 1.6288215000000001,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "population_findings",
            "kind": "list",
            "matchKey": "capture_gate",
            "k": 4,
            "expected": [
              {
                "key": "tumor_cell_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "apc_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "combined_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "unselected_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "naive",
                      "central_memory",
                      "effector_memory"
                    ],
                    "canonical": "naive"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "low"
                    ],
                    "canonical": "low"
                  },
                  "purity_caveat": {
                    "op": "equals",
                    "values": [
                      "pure_T_cell"
                    ],
                    "canonical": "pure_T_cell"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "no_reactivity_evidence"
                    ],
                    "canonical": "no_reactivity_evidence"
                  }
                }
              },
              {
                "key": "marker_sorted_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "equals",
                    "values": [
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "terminal_exhausted_LAG3_high"
                  },
                  "clonal_expansion_level": {
                    "op": "in",
                    "values": [
                      "high",
                      "intermediate"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "marker_on_non_T_lineages",
                      "single_cell_gating_loses_target"
                    ],
                    "canonical": "marker_on_non_T_lineages"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "marker_or_exhaustion_defined_only"
                    ],
                    "canonical": "marker_or_exhaustion_defined_only"
                  }
                }
              }
            ]
          },
          {
            "name": "best_capture_approach_gate",
            "role": "gate",
            "field": "best_capture_approach",
            "kind": "value",
            "expected": "heterotypic_cell_conjugate"
          },
          {
            "name": "best_selection_marker_gate",
            "role": "gate",
            "field": "best_selection_marker",
            "kind": "value",
            "expected": "none"
          },
          {
            "name": "reactivity_criterion_gate",
            "role": "gate",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "clonal_expansion_and_physical_engagement"
          },
          {
            "name": "cross_patient_robustness_gate",
            "role": "gate",
            "field": "cross_patient_robustness",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_across_patients",
              "marginal_in_some_patients"
            ]
          },
          {
            "name": "marker_selection_consequence_gate",
            "role": "gate",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "discards_clonally_expanded_reactive_cells"
          },
          {
            "name": "superiority_claim_basis_gate",
            "role": "gate",
            "field": "superiority_claim_basis",
            "kind": "value",
            "expected": "requires_functional_reactivity_assay"
          },
          {
            "name": "conjugate_enriched_state_gate",
            "role": "gate",
            "field": "conjugate_enriched_state",
            "kind": "value",
            "expected": "progenitor_exhausted_TCF7_pos_stem_like"
          },
          {
            "name": "cf_circular_exhaustion_criterion",
            "role": "hard_fail",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "exhaustion_signature_intensity"
          },
          {
            "name": "cf_marker_sort_prior",
            "role": "hard_fail",
            "field": "best_selection_marker",
            "kind": "value",
            "op": "in",
            "expected": [
              "CD39",
              "PD1",
              "PD1_high",
              "TIM3",
              "PD1_CD39",
              "CD39_CD103"
            ]
          },
          {
            "name": "cf_single_cell_capture_prior",
            "role": "hard_fail",
            "field": "best_capture_approach",
            "kind": "value",
            "op": "in",
            "expected": [
              "dissociated_single_cell",
              "dissociated_bulk_population"
            ]
          },
          {
            "name": "cf_off_data_capture_gate",
            "role": "hard_fail",
            "field": "$.population_findings[*].capture_gate",
            "kind": "avoid",
            "expected": [
              "dissociated_bulk",
              "matched_peripheral_blood",
              "matched_lymph_node",
              "post_hoc_expanded_clonotype_group"
            ]
          },
          {
            "name": "cf_conjugate_mischaracterized",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_marker_singlet_called_engaged",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "contains",
            "matchKey": "capture_gate",
            "entity": "marker_sorted_singlet",
            "valueField": "reactivity_evidence",
            "expected": [
              "engagement_and_clonal_expansion"
            ]
          },
          {
            "name": "cf_no_engaged_reactive_population",
            "role": "hard_fail",
            "field": "$.population_findings[*].reactivity_evidence",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_marker_captures_all_reactive",
            "role": "hard_fail",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "captures_full_reactive_repertoire"
          },
          {
            "name": "cf_superiority_overclaimed_from_phenotype",
            "role": "hard_fail",
            "field": "superiority_claim_basis",
            "kind": "value",
            "op": "in",
            "expected": [
              "established_by_scrna_phenotype",
              "established_by_reactivity_proxy_scores"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-4-pi",
      "modelName": "gpt-5.4",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_vs_cd39_tumor_reactivity_dominance",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 1119,
      "cost": 1.2926975,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "population_findings",
            "kind": "list",
            "matchKey": "capture_gate",
            "k": 4,
            "expected": [
              {
                "key": "tumor_cell_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "apc_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "combined_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "unselected_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "naive",
                      "central_memory",
                      "effector_memory"
                    ],
                    "canonical": "naive"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "low"
                    ],
                    "canonical": "low"
                  },
                  "purity_caveat": {
                    "op": "equals",
                    "values": [
                      "pure_T_cell"
                    ],
                    "canonical": "pure_T_cell"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "no_reactivity_evidence"
                    ],
                    "canonical": "no_reactivity_evidence"
                  }
                }
              },
              {
                "key": "marker_sorted_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "equals",
                    "values": [
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "terminal_exhausted_LAG3_high"
                  },
                  "clonal_expansion_level": {
                    "op": "in",
                    "values": [
                      "high",
                      "intermediate"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "marker_on_non_T_lineages",
                      "single_cell_gating_loses_target"
                    ],
                    "canonical": "marker_on_non_T_lineages"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "marker_or_exhaustion_defined_only"
                    ],
                    "canonical": "marker_or_exhaustion_defined_only"
                  }
                }
              }
            ]
          },
          {
            "name": "best_capture_approach_gate",
            "role": "gate",
            "field": "best_capture_approach",
            "kind": "value",
            "expected": "heterotypic_cell_conjugate"
          },
          {
            "name": "best_selection_marker_gate",
            "role": "gate",
            "field": "best_selection_marker",
            "kind": "value",
            "expected": "none"
          },
          {
            "name": "reactivity_criterion_gate",
            "role": "gate",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "clonal_expansion_and_physical_engagement"
          },
          {
            "name": "cross_patient_robustness_gate",
            "role": "gate",
            "field": "cross_patient_robustness",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_across_patients",
              "marginal_in_some_patients"
            ]
          },
          {
            "name": "marker_selection_consequence_gate",
            "role": "gate",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "discards_clonally_expanded_reactive_cells"
          },
          {
            "name": "superiority_claim_basis_gate",
            "role": "gate",
            "field": "superiority_claim_basis",
            "kind": "value",
            "expected": "requires_functional_reactivity_assay"
          },
          {
            "name": "conjugate_enriched_state_gate",
            "role": "gate",
            "field": "conjugate_enriched_state",
            "kind": "value",
            "expected": "progenitor_exhausted_TCF7_pos_stem_like"
          },
          {
            "name": "cf_circular_exhaustion_criterion",
            "role": "hard_fail",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "exhaustion_signature_intensity"
          },
          {
            "name": "cf_marker_sort_prior",
            "role": "hard_fail",
            "field": "best_selection_marker",
            "kind": "value",
            "op": "in",
            "expected": [
              "CD39",
              "PD1",
              "PD1_high",
              "TIM3",
              "PD1_CD39",
              "CD39_CD103"
            ]
          },
          {
            "name": "cf_single_cell_capture_prior",
            "role": "hard_fail",
            "field": "best_capture_approach",
            "kind": "value",
            "op": "in",
            "expected": [
              "dissociated_single_cell",
              "dissociated_bulk_population"
            ]
          },
          {
            "name": "cf_off_data_capture_gate",
            "role": "hard_fail",
            "field": "$.population_findings[*].capture_gate",
            "kind": "avoid",
            "expected": [
              "dissociated_bulk",
              "matched_peripheral_blood",
              "matched_lymph_node",
              "post_hoc_expanded_clonotype_group"
            ]
          },
          {
            "name": "cf_conjugate_mischaracterized",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_marker_singlet_called_engaged",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "contains",
            "matchKey": "capture_gate",
            "entity": "marker_sorted_singlet",
            "valueField": "reactivity_evidence",
            "expected": [
              "engagement_and_clonal_expansion"
            ]
          },
          {
            "name": "cf_no_engaged_reactive_population",
            "role": "hard_fail",
            "field": "$.population_findings[*].reactivity_evidence",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_marker_captures_all_reactive",
            "role": "hard_fail",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "captures_full_reactive_repertoire"
          },
          {
            "name": "cf_superiority_overclaimed_from_phenotype",
            "role": "hard_fail",
            "field": "superiority_claim_basis",
            "kind": "value",
            "op": "in",
            "expected": [
              "established_by_scrna_phenotype",
              "established_by_reactivity_proxy_scores"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-4-pi",
      "modelName": "gpt-5.4",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_vs_cd39_tumor_reactivity_dominance",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1862,
      "cost": 1.6747435,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "population_findings",
            "kind": "list",
            "matchKey": "capture_gate",
            "k": 4,
            "expected": [
              {
                "key": "tumor_cell_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "apc_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "combined_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "unselected_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "naive",
                      "central_memory",
                      "effector_memory"
                    ],
                    "canonical": "naive"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "low"
                    ],
                    "canonical": "low"
                  },
                  "purity_caveat": {
                    "op": "equals",
                    "values": [
                      "pure_T_cell"
                    ],
                    "canonical": "pure_T_cell"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "no_reactivity_evidence"
                    ],
                    "canonical": "no_reactivity_evidence"
                  }
                }
              },
              {
                "key": "marker_sorted_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "equals",
                    "values": [
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "terminal_exhausted_LAG3_high"
                  },
                  "clonal_expansion_level": {
                    "op": "in",
                    "values": [
                      "high",
                      "intermediate"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "marker_on_non_T_lineages",
                      "single_cell_gating_loses_target"
                    ],
                    "canonical": "marker_on_non_T_lineages"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "marker_or_exhaustion_defined_only"
                    ],
                    "canonical": "marker_or_exhaustion_defined_only"
                  }
                }
              }
            ]
          },
          {
            "name": "best_capture_approach_gate",
            "role": "gate",
            "field": "best_capture_approach",
            "kind": "value",
            "expected": "heterotypic_cell_conjugate"
          },
          {
            "name": "best_selection_marker_gate",
            "role": "gate",
            "field": "best_selection_marker",
            "kind": "value",
            "expected": "none"
          },
          {
            "name": "reactivity_criterion_gate",
            "role": "gate",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "clonal_expansion_and_physical_engagement"
          },
          {
            "name": "cross_patient_robustness_gate",
            "role": "gate",
            "field": "cross_patient_robustness",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_across_patients",
              "marginal_in_some_patients"
            ]
          },
          {
            "name": "marker_selection_consequence_gate",
            "role": "gate",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "discards_clonally_expanded_reactive_cells"
          },
          {
            "name": "superiority_claim_basis_gate",
            "role": "gate",
            "field": "superiority_claim_basis",
            "kind": "value",
            "expected": "requires_functional_reactivity_assay"
          },
          {
            "name": "conjugate_enriched_state_gate",
            "role": "gate",
            "field": "conjugate_enriched_state",
            "kind": "value",
            "expected": "progenitor_exhausted_TCF7_pos_stem_like"
          },
          {
            "name": "cf_circular_exhaustion_criterion",
            "role": "hard_fail",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "exhaustion_signature_intensity"
          },
          {
            "name": "cf_marker_sort_prior",
            "role": "hard_fail",
            "field": "best_selection_marker",
            "kind": "value",
            "op": "in",
            "expected": [
              "CD39",
              "PD1",
              "PD1_high",
              "TIM3",
              "PD1_CD39",
              "CD39_CD103"
            ]
          },
          {
            "name": "cf_single_cell_capture_prior",
            "role": "hard_fail",
            "field": "best_capture_approach",
            "kind": "value",
            "op": "in",
            "expected": [
              "dissociated_single_cell",
              "dissociated_bulk_population"
            ]
          },
          {
            "name": "cf_off_data_capture_gate",
            "role": "hard_fail",
            "field": "$.population_findings[*].capture_gate",
            "kind": "avoid",
            "expected": [
              "dissociated_bulk",
              "matched_peripheral_blood",
              "matched_lymph_node",
              "post_hoc_expanded_clonotype_group"
            ]
          },
          {
            "name": "cf_conjugate_mischaracterized",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_marker_singlet_called_engaged",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "contains",
            "matchKey": "capture_gate",
            "entity": "marker_sorted_singlet",
            "valueField": "reactivity_evidence",
            "expected": [
              "engagement_and_clonal_expansion"
            ]
          },
          {
            "name": "cf_no_engaged_reactive_population",
            "role": "hard_fail",
            "field": "$.population_findings[*].reactivity_evidence",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_marker_captures_all_reactive",
            "role": "hard_fail",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "captures_full_reactive_repertoire"
          },
          {
            "name": "cf_superiority_overclaimed_from_phenotype",
            "role": "hard_fail",
            "field": "superiority_claim_basis",
            "kind": "value",
            "op": "in",
            "expected": [
              "established_by_scrna_phenotype",
              "established_by_reactivity_proxy_scores"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-5-codex",
      "modelName": "gpt-5.5",
      "harness": "openai-codex",
      "task": "10x_5p_melanoma_cluster_vs_cd39_tumor_reactivity_dominance",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 694,
      "cost": 2.008783,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "population_findings",
            "kind": "list",
            "matchKey": "capture_gate",
            "k": 4,
            "expected": [
              {
                "key": "tumor_cell_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "apc_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "combined_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "unselected_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "naive",
                      "central_memory",
                      "effector_memory"
                    ],
                    "canonical": "naive"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "low"
                    ],
                    "canonical": "low"
                  },
                  "purity_caveat": {
                    "op": "equals",
                    "values": [
                      "pure_T_cell"
                    ],
                    "canonical": "pure_T_cell"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "no_reactivity_evidence"
                    ],
                    "canonical": "no_reactivity_evidence"
                  }
                }
              },
              {
                "key": "marker_sorted_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "equals",
                    "values": [
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "terminal_exhausted_LAG3_high"
                  },
                  "clonal_expansion_level": {
                    "op": "in",
                    "values": [
                      "high",
                      "intermediate"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "marker_on_non_T_lineages",
                      "single_cell_gating_loses_target"
                    ],
                    "canonical": "marker_on_non_T_lineages"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "marker_or_exhaustion_defined_only"
                    ],
                    "canonical": "marker_or_exhaustion_defined_only"
                  }
                }
              }
            ]
          },
          {
            "name": "best_capture_approach_gate",
            "role": "gate",
            "field": "best_capture_approach",
            "kind": "value",
            "expected": "heterotypic_cell_conjugate"
          },
          {
            "name": "best_selection_marker_gate",
            "role": "gate",
            "field": "best_selection_marker",
            "kind": "value",
            "expected": "none"
          },
          {
            "name": "reactivity_criterion_gate",
            "role": "gate",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "clonal_expansion_and_physical_engagement"
          },
          {
            "name": "cross_patient_robustness_gate",
            "role": "gate",
            "field": "cross_patient_robustness",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_across_patients",
              "marginal_in_some_patients"
            ]
          },
          {
            "name": "marker_selection_consequence_gate",
            "role": "gate",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "discards_clonally_expanded_reactive_cells"
          },
          {
            "name": "superiority_claim_basis_gate",
            "role": "gate",
            "field": "superiority_claim_basis",
            "kind": "value",
            "expected": "requires_functional_reactivity_assay"
          },
          {
            "name": "conjugate_enriched_state_gate",
            "role": "gate",
            "field": "conjugate_enriched_state",
            "kind": "value",
            "expected": "progenitor_exhausted_TCF7_pos_stem_like"
          },
          {
            "name": "cf_circular_exhaustion_criterion",
            "role": "hard_fail",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "exhaustion_signature_intensity"
          },
          {
            "name": "cf_marker_sort_prior",
            "role": "hard_fail",
            "field": "best_selection_marker",
            "kind": "value",
            "op": "in",
            "expected": [
              "CD39",
              "PD1",
              "PD1_high",
              "TIM3",
              "PD1_CD39",
              "CD39_CD103"
            ]
          },
          {
            "name": "cf_single_cell_capture_prior",
            "role": "hard_fail",
            "field": "best_capture_approach",
            "kind": "value",
            "op": "in",
            "expected": [
              "dissociated_single_cell",
              "dissociated_bulk_population"
            ]
          },
          {
            "name": "cf_off_data_capture_gate",
            "role": "hard_fail",
            "field": "$.population_findings[*].capture_gate",
            "kind": "avoid",
            "expected": [
              "dissociated_bulk",
              "matched_peripheral_blood",
              "matched_lymph_node",
              "post_hoc_expanded_clonotype_group"
            ]
          },
          {
            "name": "cf_conjugate_mischaracterized",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_marker_singlet_called_engaged",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "contains",
            "matchKey": "capture_gate",
            "entity": "marker_sorted_singlet",
            "valueField": "reactivity_evidence",
            "expected": [
              "engagement_and_clonal_expansion"
            ]
          },
          {
            "name": "cf_no_engaged_reactive_population",
            "role": "hard_fail",
            "field": "$.population_findings[*].reactivity_evidence",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_marker_captures_all_reactive",
            "role": "hard_fail",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "captures_full_reactive_repertoire"
          },
          {
            "name": "cf_superiority_overclaimed_from_phenotype",
            "role": "hard_fail",
            "field": "superiority_claim_basis",
            "kind": "value",
            "op": "in",
            "expected": [
              "established_by_scrna_phenotype",
              "established_by_reactivity_proxy_scores"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-5-codex",
      "modelName": "gpt-5.5",
      "harness": "openai-codex",
      "task": "10x_5p_melanoma_cluster_vs_cd39_tumor_reactivity_dominance",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 1034,
      "cost": 1.964522,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "population_findings",
            "kind": "list",
            "matchKey": "capture_gate",
            "k": 4,
            "expected": [
              {
                "key": "tumor_cell_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "apc_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "combined_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "unselected_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "naive",
                      "central_memory",
                      "effector_memory"
                    ],
                    "canonical": "naive"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "low"
                    ],
                    "canonical": "low"
                  },
                  "purity_caveat": {
                    "op": "equals",
                    "values": [
                      "pure_T_cell"
                    ],
                    "canonical": "pure_T_cell"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "no_reactivity_evidence"
                    ],
                    "canonical": "no_reactivity_evidence"
                  }
                }
              },
              {
                "key": "marker_sorted_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "equals",
                    "values": [
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "terminal_exhausted_LAG3_high"
                  },
                  "clonal_expansion_level": {
                    "op": "in",
                    "values": [
                      "high",
                      "intermediate"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "marker_on_non_T_lineages",
                      "single_cell_gating_loses_target"
                    ],
                    "canonical": "marker_on_non_T_lineages"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "marker_or_exhaustion_defined_only"
                    ],
                    "canonical": "marker_or_exhaustion_defined_only"
                  }
                }
              }
            ]
          },
          {
            "name": "best_capture_approach_gate",
            "role": "gate",
            "field": "best_capture_approach",
            "kind": "value",
            "expected": "heterotypic_cell_conjugate"
          },
          {
            "name": "best_selection_marker_gate",
            "role": "gate",
            "field": "best_selection_marker",
            "kind": "value",
            "expected": "none"
          },
          {
            "name": "reactivity_criterion_gate",
            "role": "gate",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "clonal_expansion_and_physical_engagement"
          },
          {
            "name": "cross_patient_robustness_gate",
            "role": "gate",
            "field": "cross_patient_robustness",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_across_patients",
              "marginal_in_some_patients"
            ]
          },
          {
            "name": "marker_selection_consequence_gate",
            "role": "gate",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "discards_clonally_expanded_reactive_cells"
          },
          {
            "name": "superiority_claim_basis_gate",
            "role": "gate",
            "field": "superiority_claim_basis",
            "kind": "value",
            "expected": "requires_functional_reactivity_assay"
          },
          {
            "name": "conjugate_enriched_state_gate",
            "role": "gate",
            "field": "conjugate_enriched_state",
            "kind": "value",
            "expected": "progenitor_exhausted_TCF7_pos_stem_like"
          },
          {
            "name": "cf_circular_exhaustion_criterion",
            "role": "hard_fail",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "exhaustion_signature_intensity"
          },
          {
            "name": "cf_marker_sort_prior",
            "role": "hard_fail",
            "field": "best_selection_marker",
            "kind": "value",
            "op": "in",
            "expected": [
              "CD39",
              "PD1",
              "PD1_high",
              "TIM3",
              "PD1_CD39",
              "CD39_CD103"
            ]
          },
          {
            "name": "cf_single_cell_capture_prior",
            "role": "hard_fail",
            "field": "best_capture_approach",
            "kind": "value",
            "op": "in",
            "expected": [
              "dissociated_single_cell",
              "dissociated_bulk_population"
            ]
          },
          {
            "name": "cf_off_data_capture_gate",
            "role": "hard_fail",
            "field": "$.population_findings[*].capture_gate",
            "kind": "avoid",
            "expected": [
              "dissociated_bulk",
              "matched_peripheral_blood",
              "matched_lymph_node",
              "post_hoc_expanded_clonotype_group"
            ]
          },
          {
            "name": "cf_conjugate_mischaracterized",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_marker_singlet_called_engaged",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "contains",
            "matchKey": "capture_gate",
            "entity": "marker_sorted_singlet",
            "valueField": "reactivity_evidence",
            "expected": [
              "engagement_and_clonal_expansion"
            ]
          },
          {
            "name": "cf_no_engaged_reactive_population",
            "role": "hard_fail",
            "field": "$.population_findings[*].reactivity_evidence",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_marker_captures_all_reactive",
            "role": "hard_fail",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "captures_full_reactive_repertoire"
          },
          {
            "name": "cf_superiority_overclaimed_from_phenotype",
            "role": "hard_fail",
            "field": "superiority_claim_basis",
            "kind": "value",
            "op": "in",
            "expected": [
              "established_by_scrna_phenotype",
              "established_by_reactivity_proxy_scores"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-5-codex",
      "modelName": "gpt-5.5",
      "harness": "openai-codex",
      "task": "10x_5p_melanoma_cluster_vs_cd39_tumor_reactivity_dominance",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 919,
      "cost": 1.051177,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "population_findings",
            "kind": "list",
            "matchKey": "capture_gate",
            "k": 4,
            "expected": [
              {
                "key": "tumor_cell_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "apc_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "combined_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "unselected_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "naive",
                      "central_memory",
                      "effector_memory"
                    ],
                    "canonical": "naive"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "low"
                    ],
                    "canonical": "low"
                  },
                  "purity_caveat": {
                    "op": "equals",
                    "values": [
                      "pure_T_cell"
                    ],
                    "canonical": "pure_T_cell"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "no_reactivity_evidence"
                    ],
                    "canonical": "no_reactivity_evidence"
                  }
                }
              },
              {
                "key": "marker_sorted_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "equals",
                    "values": [
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "terminal_exhausted_LAG3_high"
                  },
                  "clonal_expansion_level": {
                    "op": "in",
                    "values": [
                      "high",
                      "intermediate"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "marker_on_non_T_lineages",
                      "single_cell_gating_loses_target"
                    ],
                    "canonical": "marker_on_non_T_lineages"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "marker_or_exhaustion_defined_only"
                    ],
                    "canonical": "marker_or_exhaustion_defined_only"
                  }
                }
              }
            ]
          },
          {
            "name": "best_capture_approach_gate",
            "role": "gate",
            "field": "best_capture_approach",
            "kind": "value",
            "expected": "heterotypic_cell_conjugate"
          },
          {
            "name": "best_selection_marker_gate",
            "role": "gate",
            "field": "best_selection_marker",
            "kind": "value",
            "expected": "none"
          },
          {
            "name": "reactivity_criterion_gate",
            "role": "gate",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "clonal_expansion_and_physical_engagement"
          },
          {
            "name": "cross_patient_robustness_gate",
            "role": "gate",
            "field": "cross_patient_robustness",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_across_patients",
              "marginal_in_some_patients"
            ]
          },
          {
            "name": "marker_selection_consequence_gate",
            "role": "gate",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "discards_clonally_expanded_reactive_cells"
          },
          {
            "name": "superiority_claim_basis_gate",
            "role": "gate",
            "field": "superiority_claim_basis",
            "kind": "value",
            "expected": "requires_functional_reactivity_assay"
          },
          {
            "name": "conjugate_enriched_state_gate",
            "role": "gate",
            "field": "conjugate_enriched_state",
            "kind": "value",
            "expected": "progenitor_exhausted_TCF7_pos_stem_like"
          },
          {
            "name": "cf_circular_exhaustion_criterion",
            "role": "hard_fail",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "exhaustion_signature_intensity"
          },
          {
            "name": "cf_marker_sort_prior",
            "role": "hard_fail",
            "field": "best_selection_marker",
            "kind": "value",
            "op": "in",
            "expected": [
              "CD39",
              "PD1",
              "PD1_high",
              "TIM3",
              "PD1_CD39",
              "CD39_CD103"
            ]
          },
          {
            "name": "cf_single_cell_capture_prior",
            "role": "hard_fail",
            "field": "best_capture_approach",
            "kind": "value",
            "op": "in",
            "expected": [
              "dissociated_single_cell",
              "dissociated_bulk_population"
            ]
          },
          {
            "name": "cf_off_data_capture_gate",
            "role": "hard_fail",
            "field": "$.population_findings[*].capture_gate",
            "kind": "avoid",
            "expected": [
              "dissociated_bulk",
              "matched_peripheral_blood",
              "matched_lymph_node",
              "post_hoc_expanded_clonotype_group"
            ]
          },
          {
            "name": "cf_conjugate_mischaracterized",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_marker_singlet_called_engaged",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "contains",
            "matchKey": "capture_gate",
            "entity": "marker_sorted_singlet",
            "valueField": "reactivity_evidence",
            "expected": [
              "engagement_and_clonal_expansion"
            ]
          },
          {
            "name": "cf_no_engaged_reactive_population",
            "role": "hard_fail",
            "field": "$.population_findings[*].reactivity_evidence",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_marker_captures_all_reactive",
            "role": "hard_fail",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "captures_full_reactive_repertoire"
          },
          {
            "name": "cf_superiority_overclaimed_from_phenotype",
            "role": "hard_fail",
            "field": "superiority_claim_basis",
            "kind": "value",
            "op": "in",
            "expected": [
              "established_by_scrna_phenotype",
              "established_by_reactivity_proxy_scores"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-5-pi",
      "modelName": "gpt-5.5",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_vs_cd39_tumor_reactivity_dominance",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 1128,
      "cost": 2.049843,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "population_findings",
            "kind": "list",
            "matchKey": "capture_gate",
            "k": 4,
            "expected": [
              {
                "key": "tumor_cell_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "apc_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "combined_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "unselected_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "naive",
                      "central_memory",
                      "effector_memory"
                    ],
                    "canonical": "naive"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "low"
                    ],
                    "canonical": "low"
                  },
                  "purity_caveat": {
                    "op": "equals",
                    "values": [
                      "pure_T_cell"
                    ],
                    "canonical": "pure_T_cell"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "no_reactivity_evidence"
                    ],
                    "canonical": "no_reactivity_evidence"
                  }
                }
              },
              {
                "key": "marker_sorted_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "equals",
                    "values": [
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "terminal_exhausted_LAG3_high"
                  },
                  "clonal_expansion_level": {
                    "op": "in",
                    "values": [
                      "high",
                      "intermediate"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "marker_on_non_T_lineages",
                      "single_cell_gating_loses_target"
                    ],
                    "canonical": "marker_on_non_T_lineages"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "marker_or_exhaustion_defined_only"
                    ],
                    "canonical": "marker_or_exhaustion_defined_only"
                  }
                }
              }
            ]
          },
          {
            "name": "best_capture_approach_gate",
            "role": "gate",
            "field": "best_capture_approach",
            "kind": "value",
            "expected": "heterotypic_cell_conjugate"
          },
          {
            "name": "best_selection_marker_gate",
            "role": "gate",
            "field": "best_selection_marker",
            "kind": "value",
            "expected": "none"
          },
          {
            "name": "reactivity_criterion_gate",
            "role": "gate",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "clonal_expansion_and_physical_engagement"
          },
          {
            "name": "cross_patient_robustness_gate",
            "role": "gate",
            "field": "cross_patient_robustness",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_across_patients",
              "marginal_in_some_patients"
            ]
          },
          {
            "name": "marker_selection_consequence_gate",
            "role": "gate",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "discards_clonally_expanded_reactive_cells"
          },
          {
            "name": "superiority_claim_basis_gate",
            "role": "gate",
            "field": "superiority_claim_basis",
            "kind": "value",
            "expected": "requires_functional_reactivity_assay"
          },
          {
            "name": "conjugate_enriched_state_gate",
            "role": "gate",
            "field": "conjugate_enriched_state",
            "kind": "value",
            "expected": "progenitor_exhausted_TCF7_pos_stem_like"
          },
          {
            "name": "cf_circular_exhaustion_criterion",
            "role": "hard_fail",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "exhaustion_signature_intensity"
          },
          {
            "name": "cf_marker_sort_prior",
            "role": "hard_fail",
            "field": "best_selection_marker",
            "kind": "value",
            "op": "in",
            "expected": [
              "CD39",
              "PD1",
              "PD1_high",
              "TIM3",
              "PD1_CD39",
              "CD39_CD103"
            ]
          },
          {
            "name": "cf_single_cell_capture_prior",
            "role": "hard_fail",
            "field": "best_capture_approach",
            "kind": "value",
            "op": "in",
            "expected": [
              "dissociated_single_cell",
              "dissociated_bulk_population"
            ]
          },
          {
            "name": "cf_off_data_capture_gate",
            "role": "hard_fail",
            "field": "$.population_findings[*].capture_gate",
            "kind": "avoid",
            "expected": [
              "dissociated_bulk",
              "matched_peripheral_blood",
              "matched_lymph_node",
              "post_hoc_expanded_clonotype_group"
            ]
          },
          {
            "name": "cf_conjugate_mischaracterized",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_marker_singlet_called_engaged",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "contains",
            "matchKey": "capture_gate",
            "entity": "marker_sorted_singlet",
            "valueField": "reactivity_evidence",
            "expected": [
              "engagement_and_clonal_expansion"
            ]
          },
          {
            "name": "cf_no_engaged_reactive_population",
            "role": "hard_fail",
            "field": "$.population_findings[*].reactivity_evidence",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_marker_captures_all_reactive",
            "role": "hard_fail",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "captures_full_reactive_repertoire"
          },
          {
            "name": "cf_superiority_overclaimed_from_phenotype",
            "role": "hard_fail",
            "field": "superiority_claim_basis",
            "kind": "value",
            "op": "in",
            "expected": [
              "established_by_scrna_phenotype",
              "established_by_reactivity_proxy_scores"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-5-pi",
      "modelName": "gpt-5.5",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_vs_cd39_tumor_reactivity_dominance",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 699,
      "cost": 1.216539,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "population_findings",
            "kind": "list",
            "matchKey": "capture_gate",
            "k": 4,
            "expected": [
              {
                "key": "tumor_cell_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "apc_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "combined_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "unselected_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "naive",
                      "central_memory",
                      "effector_memory"
                    ],
                    "canonical": "naive"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "low"
                    ],
                    "canonical": "low"
                  },
                  "purity_caveat": {
                    "op": "equals",
                    "values": [
                      "pure_T_cell"
                    ],
                    "canonical": "pure_T_cell"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "no_reactivity_evidence"
                    ],
                    "canonical": "no_reactivity_evidence"
                  }
                }
              },
              {
                "key": "marker_sorted_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "equals",
                    "values": [
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "terminal_exhausted_LAG3_high"
                  },
                  "clonal_expansion_level": {
                    "op": "in",
                    "values": [
                      "high",
                      "intermediate"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "marker_on_non_T_lineages",
                      "single_cell_gating_loses_target"
                    ],
                    "canonical": "marker_on_non_T_lineages"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "marker_or_exhaustion_defined_only"
                    ],
                    "canonical": "marker_or_exhaustion_defined_only"
                  }
                }
              }
            ]
          },
          {
            "name": "best_capture_approach_gate",
            "role": "gate",
            "field": "best_capture_approach",
            "kind": "value",
            "expected": "heterotypic_cell_conjugate"
          },
          {
            "name": "best_selection_marker_gate",
            "role": "gate",
            "field": "best_selection_marker",
            "kind": "value",
            "expected": "none"
          },
          {
            "name": "reactivity_criterion_gate",
            "role": "gate",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "clonal_expansion_and_physical_engagement"
          },
          {
            "name": "cross_patient_robustness_gate",
            "role": "gate",
            "field": "cross_patient_robustness",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_across_patients",
              "marginal_in_some_patients"
            ]
          },
          {
            "name": "marker_selection_consequence_gate",
            "role": "gate",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "discards_clonally_expanded_reactive_cells"
          },
          {
            "name": "superiority_claim_basis_gate",
            "role": "gate",
            "field": "superiority_claim_basis",
            "kind": "value",
            "expected": "requires_functional_reactivity_assay"
          },
          {
            "name": "conjugate_enriched_state_gate",
            "role": "gate",
            "field": "conjugate_enriched_state",
            "kind": "value",
            "expected": "progenitor_exhausted_TCF7_pos_stem_like"
          },
          {
            "name": "cf_circular_exhaustion_criterion",
            "role": "hard_fail",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "exhaustion_signature_intensity"
          },
          {
            "name": "cf_marker_sort_prior",
            "role": "hard_fail",
            "field": "best_selection_marker",
            "kind": "value",
            "op": "in",
            "expected": [
              "CD39",
              "PD1",
              "PD1_high",
              "TIM3",
              "PD1_CD39",
              "CD39_CD103"
            ]
          },
          {
            "name": "cf_single_cell_capture_prior",
            "role": "hard_fail",
            "field": "best_capture_approach",
            "kind": "value",
            "op": "in",
            "expected": [
              "dissociated_single_cell",
              "dissociated_bulk_population"
            ]
          },
          {
            "name": "cf_off_data_capture_gate",
            "role": "hard_fail",
            "field": "$.population_findings[*].capture_gate",
            "kind": "avoid",
            "expected": [
              "dissociated_bulk",
              "matched_peripheral_blood",
              "matched_lymph_node",
              "post_hoc_expanded_clonotype_group"
            ]
          },
          {
            "name": "cf_conjugate_mischaracterized",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_marker_singlet_called_engaged",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "contains",
            "matchKey": "capture_gate",
            "entity": "marker_sorted_singlet",
            "valueField": "reactivity_evidence",
            "expected": [
              "engagement_and_clonal_expansion"
            ]
          },
          {
            "name": "cf_no_engaged_reactive_population",
            "role": "hard_fail",
            "field": "$.population_findings[*].reactivity_evidence",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_marker_captures_all_reactive",
            "role": "hard_fail",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "captures_full_reactive_repertoire"
          },
          {
            "name": "cf_superiority_overclaimed_from_phenotype",
            "role": "hard_fail",
            "field": "superiority_claim_basis",
            "kind": "value",
            "op": "in",
            "expected": [
              "established_by_scrna_phenotype",
              "established_by_reactivity_proxy_scores"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-5-pi",
      "modelName": "gpt-5.5",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_vs_cd39_tumor_reactivity_dominance",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 967,
      "cost": 1.876495,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "population_findings",
            "kind": "list",
            "matchKey": "capture_gate",
            "k": 4,
            "expected": [
              {
                "key": "tumor_cell_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "apc_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "combined_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "unselected_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "naive",
                      "central_memory",
                      "effector_memory"
                    ],
                    "canonical": "naive"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "low"
                    ],
                    "canonical": "low"
                  },
                  "purity_caveat": {
                    "op": "equals",
                    "values": [
                      "pure_T_cell"
                    ],
                    "canonical": "pure_T_cell"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "no_reactivity_evidence"
                    ],
                    "canonical": "no_reactivity_evidence"
                  }
                }
              },
              {
                "key": "marker_sorted_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "equals",
                    "values": [
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "terminal_exhausted_LAG3_high"
                  },
                  "clonal_expansion_level": {
                    "op": "in",
                    "values": [
                      "high",
                      "intermediate"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "marker_on_non_T_lineages",
                      "single_cell_gating_loses_target"
                    ],
                    "canonical": "marker_on_non_T_lineages"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "marker_or_exhaustion_defined_only"
                    ],
                    "canonical": "marker_or_exhaustion_defined_only"
                  }
                }
              }
            ]
          },
          {
            "name": "best_capture_approach_gate",
            "role": "gate",
            "field": "best_capture_approach",
            "kind": "value",
            "expected": "heterotypic_cell_conjugate"
          },
          {
            "name": "best_selection_marker_gate",
            "role": "gate",
            "field": "best_selection_marker",
            "kind": "value",
            "expected": "none"
          },
          {
            "name": "reactivity_criterion_gate",
            "role": "gate",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "clonal_expansion_and_physical_engagement"
          },
          {
            "name": "cross_patient_robustness_gate",
            "role": "gate",
            "field": "cross_patient_robustness",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_across_patients",
              "marginal_in_some_patients"
            ]
          },
          {
            "name": "marker_selection_consequence_gate",
            "role": "gate",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "discards_clonally_expanded_reactive_cells"
          },
          {
            "name": "superiority_claim_basis_gate",
            "role": "gate",
            "field": "superiority_claim_basis",
            "kind": "value",
            "expected": "requires_functional_reactivity_assay"
          },
          {
            "name": "conjugate_enriched_state_gate",
            "role": "gate",
            "field": "conjugate_enriched_state",
            "kind": "value",
            "expected": "progenitor_exhausted_TCF7_pos_stem_like"
          },
          {
            "name": "cf_circular_exhaustion_criterion",
            "role": "hard_fail",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "exhaustion_signature_intensity"
          },
          {
            "name": "cf_marker_sort_prior",
            "role": "hard_fail",
            "field": "best_selection_marker",
            "kind": "value",
            "op": "in",
            "expected": [
              "CD39",
              "PD1",
              "PD1_high",
              "TIM3",
              "PD1_CD39",
              "CD39_CD103"
            ]
          },
          {
            "name": "cf_single_cell_capture_prior",
            "role": "hard_fail",
            "field": "best_capture_approach",
            "kind": "value",
            "op": "in",
            "expected": [
              "dissociated_single_cell",
              "dissociated_bulk_population"
            ]
          },
          {
            "name": "cf_off_data_capture_gate",
            "role": "hard_fail",
            "field": "$.population_findings[*].capture_gate",
            "kind": "avoid",
            "expected": [
              "dissociated_bulk",
              "matched_peripheral_blood",
              "matched_lymph_node",
              "post_hoc_expanded_clonotype_group"
            ]
          },
          {
            "name": "cf_conjugate_mischaracterized",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_marker_singlet_called_engaged",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "contains",
            "matchKey": "capture_gate",
            "entity": "marker_sorted_singlet",
            "valueField": "reactivity_evidence",
            "expected": [
              "engagement_and_clonal_expansion"
            ]
          },
          {
            "name": "cf_no_engaged_reactive_population",
            "role": "hard_fail",
            "field": "$.population_findings[*].reactivity_evidence",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_marker_captures_all_reactive",
            "role": "hard_fail",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "captures_full_reactive_repertoire"
          },
          {
            "name": "cf_superiority_overclaimed_from_phenotype",
            "role": "hard_fail",
            "field": "superiority_claim_basis",
            "kind": "value",
            "op": "in",
            "expected": [
              "established_by_scrna_phenotype",
              "established_by_reactivity_proxy_scores"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-20-beta-reasoning-pi",
      "modelName": "grok-4.20-beta-0309-reasoning",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_vs_cd39_tumor_reactivity_dominance",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 414,
      "cost": 0.069321,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "population_findings",
            "kind": "list",
            "matchKey": "capture_gate",
            "k": 4,
            "expected": [
              {
                "key": "tumor_cell_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "apc_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "combined_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "unselected_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "naive",
                      "central_memory",
                      "effector_memory"
                    ],
                    "canonical": "naive"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "low"
                    ],
                    "canonical": "low"
                  },
                  "purity_caveat": {
                    "op": "equals",
                    "values": [
                      "pure_T_cell"
                    ],
                    "canonical": "pure_T_cell"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "no_reactivity_evidence"
                    ],
                    "canonical": "no_reactivity_evidence"
                  }
                }
              },
              {
                "key": "marker_sorted_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "equals",
                    "values": [
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "terminal_exhausted_LAG3_high"
                  },
                  "clonal_expansion_level": {
                    "op": "in",
                    "values": [
                      "high",
                      "intermediate"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "marker_on_non_T_lineages",
                      "single_cell_gating_loses_target"
                    ],
                    "canonical": "marker_on_non_T_lineages"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "marker_or_exhaustion_defined_only"
                    ],
                    "canonical": "marker_or_exhaustion_defined_only"
                  }
                }
              }
            ]
          },
          {
            "name": "best_capture_approach_gate",
            "role": "gate",
            "field": "best_capture_approach",
            "kind": "value",
            "expected": "heterotypic_cell_conjugate"
          },
          {
            "name": "best_selection_marker_gate",
            "role": "gate",
            "field": "best_selection_marker",
            "kind": "value",
            "expected": "none"
          },
          {
            "name": "reactivity_criterion_gate",
            "role": "gate",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "clonal_expansion_and_physical_engagement"
          },
          {
            "name": "cross_patient_robustness_gate",
            "role": "gate",
            "field": "cross_patient_robustness",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_across_patients",
              "marginal_in_some_patients"
            ]
          },
          {
            "name": "marker_selection_consequence_gate",
            "role": "gate",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "discards_clonally_expanded_reactive_cells"
          },
          {
            "name": "superiority_claim_basis_gate",
            "role": "gate",
            "field": "superiority_claim_basis",
            "kind": "value",
            "expected": "requires_functional_reactivity_assay"
          },
          {
            "name": "conjugate_enriched_state_gate",
            "role": "gate",
            "field": "conjugate_enriched_state",
            "kind": "value",
            "expected": "progenitor_exhausted_TCF7_pos_stem_like"
          },
          {
            "name": "cf_circular_exhaustion_criterion",
            "role": "hard_fail",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "exhaustion_signature_intensity"
          },
          {
            "name": "cf_marker_sort_prior",
            "role": "hard_fail",
            "field": "best_selection_marker",
            "kind": "value",
            "op": "in",
            "expected": [
              "CD39",
              "PD1",
              "PD1_high",
              "TIM3",
              "PD1_CD39",
              "CD39_CD103"
            ]
          },
          {
            "name": "cf_single_cell_capture_prior",
            "role": "hard_fail",
            "field": "best_capture_approach",
            "kind": "value",
            "op": "in",
            "expected": [
              "dissociated_single_cell",
              "dissociated_bulk_population"
            ]
          },
          {
            "name": "cf_off_data_capture_gate",
            "role": "hard_fail",
            "field": "$.population_findings[*].capture_gate",
            "kind": "avoid",
            "expected": [
              "dissociated_bulk",
              "matched_peripheral_blood",
              "matched_lymph_node",
              "post_hoc_expanded_clonotype_group"
            ]
          },
          {
            "name": "cf_conjugate_mischaracterized",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_marker_singlet_called_engaged",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "contains",
            "matchKey": "capture_gate",
            "entity": "marker_sorted_singlet",
            "valueField": "reactivity_evidence",
            "expected": [
              "engagement_and_clonal_expansion"
            ]
          },
          {
            "name": "cf_no_engaged_reactive_population",
            "role": "hard_fail",
            "field": "$.population_findings[*].reactivity_evidence",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_marker_captures_all_reactive",
            "role": "hard_fail",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "captures_full_reactive_repertoire"
          },
          {
            "name": "cf_superiority_overclaimed_from_phenotype",
            "role": "hard_fail",
            "field": "superiority_claim_basis",
            "kind": "value",
            "op": "in",
            "expected": [
              "established_by_scrna_phenotype",
              "established_by_reactivity_proxy_scores"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-20-beta-reasoning-pi",
      "modelName": "grok-4.20-beta-0309-reasoning",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_vs_cd39_tumor_reactivity_dominance",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 228,
      "cost": 0.057830900000000005,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "population_findings",
            "kind": "list",
            "matchKey": "capture_gate",
            "k": 4,
            "expected": [
              {
                "key": "tumor_cell_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "apc_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "combined_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "unselected_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "naive",
                      "central_memory",
                      "effector_memory"
                    ],
                    "canonical": "naive"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "low"
                    ],
                    "canonical": "low"
                  },
                  "purity_caveat": {
                    "op": "equals",
                    "values": [
                      "pure_T_cell"
                    ],
                    "canonical": "pure_T_cell"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "no_reactivity_evidence"
                    ],
                    "canonical": "no_reactivity_evidence"
                  }
                }
              },
              {
                "key": "marker_sorted_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "equals",
                    "values": [
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "terminal_exhausted_LAG3_high"
                  },
                  "clonal_expansion_level": {
                    "op": "in",
                    "values": [
                      "high",
                      "intermediate"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "marker_on_non_T_lineages",
                      "single_cell_gating_loses_target"
                    ],
                    "canonical": "marker_on_non_T_lineages"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "marker_or_exhaustion_defined_only"
                    ],
                    "canonical": "marker_or_exhaustion_defined_only"
                  }
                }
              }
            ]
          },
          {
            "name": "best_capture_approach_gate",
            "role": "gate",
            "field": "best_capture_approach",
            "kind": "value",
            "expected": "heterotypic_cell_conjugate"
          },
          {
            "name": "best_selection_marker_gate",
            "role": "gate",
            "field": "best_selection_marker",
            "kind": "value",
            "expected": "none"
          },
          {
            "name": "reactivity_criterion_gate",
            "role": "gate",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "clonal_expansion_and_physical_engagement"
          },
          {
            "name": "cross_patient_robustness_gate",
            "role": "gate",
            "field": "cross_patient_robustness",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_across_patients",
              "marginal_in_some_patients"
            ]
          },
          {
            "name": "marker_selection_consequence_gate",
            "role": "gate",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "discards_clonally_expanded_reactive_cells"
          },
          {
            "name": "superiority_claim_basis_gate",
            "role": "gate",
            "field": "superiority_claim_basis",
            "kind": "value",
            "expected": "requires_functional_reactivity_assay"
          },
          {
            "name": "conjugate_enriched_state_gate",
            "role": "gate",
            "field": "conjugate_enriched_state",
            "kind": "value",
            "expected": "progenitor_exhausted_TCF7_pos_stem_like"
          },
          {
            "name": "cf_circular_exhaustion_criterion",
            "role": "hard_fail",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "exhaustion_signature_intensity"
          },
          {
            "name": "cf_marker_sort_prior",
            "role": "hard_fail",
            "field": "best_selection_marker",
            "kind": "value",
            "op": "in",
            "expected": [
              "CD39",
              "PD1",
              "PD1_high",
              "TIM3",
              "PD1_CD39",
              "CD39_CD103"
            ]
          },
          {
            "name": "cf_single_cell_capture_prior",
            "role": "hard_fail",
            "field": "best_capture_approach",
            "kind": "value",
            "op": "in",
            "expected": [
              "dissociated_single_cell",
              "dissociated_bulk_population"
            ]
          },
          {
            "name": "cf_off_data_capture_gate",
            "role": "hard_fail",
            "field": "$.population_findings[*].capture_gate",
            "kind": "avoid",
            "expected": [
              "dissociated_bulk",
              "matched_peripheral_blood",
              "matched_lymph_node",
              "post_hoc_expanded_clonotype_group"
            ]
          },
          {
            "name": "cf_conjugate_mischaracterized",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_marker_singlet_called_engaged",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "contains",
            "matchKey": "capture_gate",
            "entity": "marker_sorted_singlet",
            "valueField": "reactivity_evidence",
            "expected": [
              "engagement_and_clonal_expansion"
            ]
          },
          {
            "name": "cf_no_engaged_reactive_population",
            "role": "hard_fail",
            "field": "$.population_findings[*].reactivity_evidence",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_marker_captures_all_reactive",
            "role": "hard_fail",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "captures_full_reactive_repertoire"
          },
          {
            "name": "cf_superiority_overclaimed_from_phenotype",
            "role": "hard_fail",
            "field": "superiority_claim_basis",
            "kind": "value",
            "op": "in",
            "expected": [
              "established_by_scrna_phenotype",
              "established_by_reactivity_proxy_scores"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-20-beta-reasoning-pi",
      "modelName": "grok-4.20-beta-0309-reasoning",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_vs_cd39_tumor_reactivity_dominance",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 303,
      "cost": 0.05355805,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "population_findings",
            "kind": "list",
            "matchKey": "capture_gate",
            "k": 4,
            "expected": [
              {
                "key": "tumor_cell_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "apc_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "combined_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "unselected_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "naive",
                      "central_memory",
                      "effector_memory"
                    ],
                    "canonical": "naive"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "low"
                    ],
                    "canonical": "low"
                  },
                  "purity_caveat": {
                    "op": "equals",
                    "values": [
                      "pure_T_cell"
                    ],
                    "canonical": "pure_T_cell"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "no_reactivity_evidence"
                    ],
                    "canonical": "no_reactivity_evidence"
                  }
                }
              },
              {
                "key": "marker_sorted_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "equals",
                    "values": [
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "terminal_exhausted_LAG3_high"
                  },
                  "clonal_expansion_level": {
                    "op": "in",
                    "values": [
                      "high",
                      "intermediate"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "marker_on_non_T_lineages",
                      "single_cell_gating_loses_target"
                    ],
                    "canonical": "marker_on_non_T_lineages"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "marker_or_exhaustion_defined_only"
                    ],
                    "canonical": "marker_or_exhaustion_defined_only"
                  }
                }
              }
            ]
          },
          {
            "name": "best_capture_approach_gate",
            "role": "gate",
            "field": "best_capture_approach",
            "kind": "value",
            "expected": "heterotypic_cell_conjugate"
          },
          {
            "name": "best_selection_marker_gate",
            "role": "gate",
            "field": "best_selection_marker",
            "kind": "value",
            "expected": "none"
          },
          {
            "name": "reactivity_criterion_gate",
            "role": "gate",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "clonal_expansion_and_physical_engagement"
          },
          {
            "name": "cross_patient_robustness_gate",
            "role": "gate",
            "field": "cross_patient_robustness",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_across_patients",
              "marginal_in_some_patients"
            ]
          },
          {
            "name": "marker_selection_consequence_gate",
            "role": "gate",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "discards_clonally_expanded_reactive_cells"
          },
          {
            "name": "superiority_claim_basis_gate",
            "role": "gate",
            "field": "superiority_claim_basis",
            "kind": "value",
            "expected": "requires_functional_reactivity_assay"
          },
          {
            "name": "conjugate_enriched_state_gate",
            "role": "gate",
            "field": "conjugate_enriched_state",
            "kind": "value",
            "expected": "progenitor_exhausted_TCF7_pos_stem_like"
          },
          {
            "name": "cf_circular_exhaustion_criterion",
            "role": "hard_fail",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "exhaustion_signature_intensity"
          },
          {
            "name": "cf_marker_sort_prior",
            "role": "hard_fail",
            "field": "best_selection_marker",
            "kind": "value",
            "op": "in",
            "expected": [
              "CD39",
              "PD1",
              "PD1_high",
              "TIM3",
              "PD1_CD39",
              "CD39_CD103"
            ]
          },
          {
            "name": "cf_single_cell_capture_prior",
            "role": "hard_fail",
            "field": "best_capture_approach",
            "kind": "value",
            "op": "in",
            "expected": [
              "dissociated_single_cell",
              "dissociated_bulk_population"
            ]
          },
          {
            "name": "cf_off_data_capture_gate",
            "role": "hard_fail",
            "field": "$.population_findings[*].capture_gate",
            "kind": "avoid",
            "expected": [
              "dissociated_bulk",
              "matched_peripheral_blood",
              "matched_lymph_node",
              "post_hoc_expanded_clonotype_group"
            ]
          },
          {
            "name": "cf_conjugate_mischaracterized",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_marker_singlet_called_engaged",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "contains",
            "matchKey": "capture_gate",
            "entity": "marker_sorted_singlet",
            "valueField": "reactivity_evidence",
            "expected": [
              "engagement_and_clonal_expansion"
            ]
          },
          {
            "name": "cf_no_engaged_reactive_population",
            "role": "hard_fail",
            "field": "$.population_findings[*].reactivity_evidence",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_marker_captures_all_reactive",
            "role": "hard_fail",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "captures_full_reactive_repertoire"
          },
          {
            "name": "cf_superiority_overclaimed_from_phenotype",
            "role": "hard_fail",
            "field": "superiority_claim_basis",
            "kind": "value",
            "op": "in",
            "expected": [
              "established_by_scrna_phenotype",
              "established_by_reactivity_proxy_scores"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-3-pi",
      "modelName": "grok-4.3",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_vs_cd39_tumor_reactivity_dominance",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 216,
      "cost": 0.0233947,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "population_findings",
            "kind": "list",
            "matchKey": "capture_gate",
            "k": 4,
            "expected": [
              {
                "key": "tumor_cell_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "apc_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "combined_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "unselected_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "naive",
                      "central_memory",
                      "effector_memory"
                    ],
                    "canonical": "naive"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "low"
                    ],
                    "canonical": "low"
                  },
                  "purity_caveat": {
                    "op": "equals",
                    "values": [
                      "pure_T_cell"
                    ],
                    "canonical": "pure_T_cell"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "no_reactivity_evidence"
                    ],
                    "canonical": "no_reactivity_evidence"
                  }
                }
              },
              {
                "key": "marker_sorted_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "equals",
                    "values": [
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "terminal_exhausted_LAG3_high"
                  },
                  "clonal_expansion_level": {
                    "op": "in",
                    "values": [
                      "high",
                      "intermediate"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "marker_on_non_T_lineages",
                      "single_cell_gating_loses_target"
                    ],
                    "canonical": "marker_on_non_T_lineages"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "marker_or_exhaustion_defined_only"
                    ],
                    "canonical": "marker_or_exhaustion_defined_only"
                  }
                }
              }
            ]
          },
          {
            "name": "best_capture_approach_gate",
            "role": "gate",
            "field": "best_capture_approach",
            "kind": "value",
            "expected": "heterotypic_cell_conjugate"
          },
          {
            "name": "best_selection_marker_gate",
            "role": "gate",
            "field": "best_selection_marker",
            "kind": "value",
            "expected": "none"
          },
          {
            "name": "reactivity_criterion_gate",
            "role": "gate",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "clonal_expansion_and_physical_engagement"
          },
          {
            "name": "cross_patient_robustness_gate",
            "role": "gate",
            "field": "cross_patient_robustness",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_across_patients",
              "marginal_in_some_patients"
            ]
          },
          {
            "name": "marker_selection_consequence_gate",
            "role": "gate",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "discards_clonally_expanded_reactive_cells"
          },
          {
            "name": "superiority_claim_basis_gate",
            "role": "gate",
            "field": "superiority_claim_basis",
            "kind": "value",
            "expected": "requires_functional_reactivity_assay"
          },
          {
            "name": "conjugate_enriched_state_gate",
            "role": "gate",
            "field": "conjugate_enriched_state",
            "kind": "value",
            "expected": "progenitor_exhausted_TCF7_pos_stem_like"
          },
          {
            "name": "cf_circular_exhaustion_criterion",
            "role": "hard_fail",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "exhaustion_signature_intensity"
          },
          {
            "name": "cf_marker_sort_prior",
            "role": "hard_fail",
            "field": "best_selection_marker",
            "kind": "value",
            "op": "in",
            "expected": [
              "CD39",
              "PD1",
              "PD1_high",
              "TIM3",
              "PD1_CD39",
              "CD39_CD103"
            ]
          },
          {
            "name": "cf_single_cell_capture_prior",
            "role": "hard_fail",
            "field": "best_capture_approach",
            "kind": "value",
            "op": "in",
            "expected": [
              "dissociated_single_cell",
              "dissociated_bulk_population"
            ]
          },
          {
            "name": "cf_off_data_capture_gate",
            "role": "hard_fail",
            "field": "$.population_findings[*].capture_gate",
            "kind": "avoid",
            "expected": [
              "dissociated_bulk",
              "matched_peripheral_blood",
              "matched_lymph_node",
              "post_hoc_expanded_clonotype_group"
            ]
          },
          {
            "name": "cf_conjugate_mischaracterized",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_marker_singlet_called_engaged",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "contains",
            "matchKey": "capture_gate",
            "entity": "marker_sorted_singlet",
            "valueField": "reactivity_evidence",
            "expected": [
              "engagement_and_clonal_expansion"
            ]
          },
          {
            "name": "cf_no_engaged_reactive_population",
            "role": "hard_fail",
            "field": "$.population_findings[*].reactivity_evidence",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_marker_captures_all_reactive",
            "role": "hard_fail",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "captures_full_reactive_repertoire"
          },
          {
            "name": "cf_superiority_overclaimed_from_phenotype",
            "role": "hard_fail",
            "field": "superiority_claim_basis",
            "kind": "value",
            "op": "in",
            "expected": [
              "established_by_scrna_phenotype",
              "established_by_reactivity_proxy_scores"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-3-pi",
      "modelName": "grok-4.3",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_vs_cd39_tumor_reactivity_dominance",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 200,
      "cost": 0.021112750000000003,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "population_findings",
            "kind": "list",
            "matchKey": "capture_gate",
            "k": 4,
            "expected": [
              {
                "key": "tumor_cell_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "apc_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "combined_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "unselected_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "naive",
                      "central_memory",
                      "effector_memory"
                    ],
                    "canonical": "naive"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "low"
                    ],
                    "canonical": "low"
                  },
                  "purity_caveat": {
                    "op": "equals",
                    "values": [
                      "pure_T_cell"
                    ],
                    "canonical": "pure_T_cell"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "no_reactivity_evidence"
                    ],
                    "canonical": "no_reactivity_evidence"
                  }
                }
              },
              {
                "key": "marker_sorted_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "equals",
                    "values": [
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "terminal_exhausted_LAG3_high"
                  },
                  "clonal_expansion_level": {
                    "op": "in",
                    "values": [
                      "high",
                      "intermediate"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "marker_on_non_T_lineages",
                      "single_cell_gating_loses_target"
                    ],
                    "canonical": "marker_on_non_T_lineages"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "marker_or_exhaustion_defined_only"
                    ],
                    "canonical": "marker_or_exhaustion_defined_only"
                  }
                }
              }
            ]
          },
          {
            "name": "best_capture_approach_gate",
            "role": "gate",
            "field": "best_capture_approach",
            "kind": "value",
            "expected": "heterotypic_cell_conjugate"
          },
          {
            "name": "best_selection_marker_gate",
            "role": "gate",
            "field": "best_selection_marker",
            "kind": "value",
            "expected": "none"
          },
          {
            "name": "reactivity_criterion_gate",
            "role": "gate",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "clonal_expansion_and_physical_engagement"
          },
          {
            "name": "cross_patient_robustness_gate",
            "role": "gate",
            "field": "cross_patient_robustness",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_across_patients",
              "marginal_in_some_patients"
            ]
          },
          {
            "name": "marker_selection_consequence_gate",
            "role": "gate",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "discards_clonally_expanded_reactive_cells"
          },
          {
            "name": "superiority_claim_basis_gate",
            "role": "gate",
            "field": "superiority_claim_basis",
            "kind": "value",
            "expected": "requires_functional_reactivity_assay"
          },
          {
            "name": "conjugate_enriched_state_gate",
            "role": "gate",
            "field": "conjugate_enriched_state",
            "kind": "value",
            "expected": "progenitor_exhausted_TCF7_pos_stem_like"
          },
          {
            "name": "cf_circular_exhaustion_criterion",
            "role": "hard_fail",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "exhaustion_signature_intensity"
          },
          {
            "name": "cf_marker_sort_prior",
            "role": "hard_fail",
            "field": "best_selection_marker",
            "kind": "value",
            "op": "in",
            "expected": [
              "CD39",
              "PD1",
              "PD1_high",
              "TIM3",
              "PD1_CD39",
              "CD39_CD103"
            ]
          },
          {
            "name": "cf_single_cell_capture_prior",
            "role": "hard_fail",
            "field": "best_capture_approach",
            "kind": "value",
            "op": "in",
            "expected": [
              "dissociated_single_cell",
              "dissociated_bulk_population"
            ]
          },
          {
            "name": "cf_off_data_capture_gate",
            "role": "hard_fail",
            "field": "$.population_findings[*].capture_gate",
            "kind": "avoid",
            "expected": [
              "dissociated_bulk",
              "matched_peripheral_blood",
              "matched_lymph_node",
              "post_hoc_expanded_clonotype_group"
            ]
          },
          {
            "name": "cf_conjugate_mischaracterized",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_marker_singlet_called_engaged",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "contains",
            "matchKey": "capture_gate",
            "entity": "marker_sorted_singlet",
            "valueField": "reactivity_evidence",
            "expected": [
              "engagement_and_clonal_expansion"
            ]
          },
          {
            "name": "cf_no_engaged_reactive_population",
            "role": "hard_fail",
            "field": "$.population_findings[*].reactivity_evidence",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_marker_captures_all_reactive",
            "role": "hard_fail",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "captures_full_reactive_repertoire"
          },
          {
            "name": "cf_superiority_overclaimed_from_phenotype",
            "role": "hard_fail",
            "field": "superiority_claim_basis",
            "kind": "value",
            "op": "in",
            "expected": [
              "established_by_scrna_phenotype",
              "established_by_reactivity_proxy_scores"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-3-pi",
      "modelName": "grok-4.3",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_vs_cd39_tumor_reactivity_dominance",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 176,
      "cost": 0.013364500000000001,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "population_findings",
            "kind": "list",
            "matchKey": "capture_gate",
            "k": 4,
            "expected": [
              {
                "key": "tumor_cell_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "apc_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "combined_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "unselected_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "naive",
                      "central_memory",
                      "effector_memory"
                    ],
                    "canonical": "naive"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "low"
                    ],
                    "canonical": "low"
                  },
                  "purity_caveat": {
                    "op": "equals",
                    "values": [
                      "pure_T_cell"
                    ],
                    "canonical": "pure_T_cell"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "no_reactivity_evidence"
                    ],
                    "canonical": "no_reactivity_evidence"
                  }
                }
              },
              {
                "key": "marker_sorted_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "equals",
                    "values": [
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "terminal_exhausted_LAG3_high"
                  },
                  "clonal_expansion_level": {
                    "op": "in",
                    "values": [
                      "high",
                      "intermediate"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "marker_on_non_T_lineages",
                      "single_cell_gating_loses_target"
                    ],
                    "canonical": "marker_on_non_T_lineages"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "marker_or_exhaustion_defined_only"
                    ],
                    "canonical": "marker_or_exhaustion_defined_only"
                  }
                }
              }
            ]
          },
          {
            "name": "best_capture_approach_gate",
            "role": "gate",
            "field": "best_capture_approach",
            "kind": "value",
            "expected": "heterotypic_cell_conjugate"
          },
          {
            "name": "best_selection_marker_gate",
            "role": "gate",
            "field": "best_selection_marker",
            "kind": "value",
            "expected": "none"
          },
          {
            "name": "reactivity_criterion_gate",
            "role": "gate",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "clonal_expansion_and_physical_engagement"
          },
          {
            "name": "cross_patient_robustness_gate",
            "role": "gate",
            "field": "cross_patient_robustness",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_across_patients",
              "marginal_in_some_patients"
            ]
          },
          {
            "name": "marker_selection_consequence_gate",
            "role": "gate",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "discards_clonally_expanded_reactive_cells"
          },
          {
            "name": "superiority_claim_basis_gate",
            "role": "gate",
            "field": "superiority_claim_basis",
            "kind": "value",
            "expected": "requires_functional_reactivity_assay"
          },
          {
            "name": "conjugate_enriched_state_gate",
            "role": "gate",
            "field": "conjugate_enriched_state",
            "kind": "value",
            "expected": "progenitor_exhausted_TCF7_pos_stem_like"
          },
          {
            "name": "cf_circular_exhaustion_criterion",
            "role": "hard_fail",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "exhaustion_signature_intensity"
          },
          {
            "name": "cf_marker_sort_prior",
            "role": "hard_fail",
            "field": "best_selection_marker",
            "kind": "value",
            "op": "in",
            "expected": [
              "CD39",
              "PD1",
              "PD1_high",
              "TIM3",
              "PD1_CD39",
              "CD39_CD103"
            ]
          },
          {
            "name": "cf_single_cell_capture_prior",
            "role": "hard_fail",
            "field": "best_capture_approach",
            "kind": "value",
            "op": "in",
            "expected": [
              "dissociated_single_cell",
              "dissociated_bulk_population"
            ]
          },
          {
            "name": "cf_off_data_capture_gate",
            "role": "hard_fail",
            "field": "$.population_findings[*].capture_gate",
            "kind": "avoid",
            "expected": [
              "dissociated_bulk",
              "matched_peripheral_blood",
              "matched_lymph_node",
              "post_hoc_expanded_clonotype_group"
            ]
          },
          {
            "name": "cf_conjugate_mischaracterized",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_marker_singlet_called_engaged",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "contains",
            "matchKey": "capture_gate",
            "entity": "marker_sorted_singlet",
            "valueField": "reactivity_evidence",
            "expected": [
              "engagement_and_clonal_expansion"
            ]
          },
          {
            "name": "cf_no_engaged_reactive_population",
            "role": "hard_fail",
            "field": "$.population_findings[*].reactivity_evidence",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_marker_captures_all_reactive",
            "role": "hard_fail",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "captures_full_reactive_repertoire"
          },
          {
            "name": "cf_superiority_overclaimed_from_phenotype",
            "role": "hard_fail",
            "field": "superiority_claim_basis",
            "kind": "value",
            "op": "in",
            "expected": [
              "established_by_scrna_phenotype",
              "established_by_reactivity_proxy_scores"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-kimi-k2p6-pi",
      "modelName": "kimi-k2p6",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_vs_cd39_tumor_reactivity_dominance",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 1568,
      "cost": 0.65089599,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "population_findings",
            "kind": "list",
            "matchKey": "capture_gate",
            "k": 4,
            "expected": [
              {
                "key": "tumor_cell_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "apc_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "combined_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "unselected_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "naive",
                      "central_memory",
                      "effector_memory"
                    ],
                    "canonical": "naive"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "low"
                    ],
                    "canonical": "low"
                  },
                  "purity_caveat": {
                    "op": "equals",
                    "values": [
                      "pure_T_cell"
                    ],
                    "canonical": "pure_T_cell"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "no_reactivity_evidence"
                    ],
                    "canonical": "no_reactivity_evidence"
                  }
                }
              },
              {
                "key": "marker_sorted_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "equals",
                    "values": [
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "terminal_exhausted_LAG3_high"
                  },
                  "clonal_expansion_level": {
                    "op": "in",
                    "values": [
                      "high",
                      "intermediate"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "marker_on_non_T_lineages",
                      "single_cell_gating_loses_target"
                    ],
                    "canonical": "marker_on_non_T_lineages"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "marker_or_exhaustion_defined_only"
                    ],
                    "canonical": "marker_or_exhaustion_defined_only"
                  }
                }
              }
            ]
          },
          {
            "name": "best_capture_approach_gate",
            "role": "gate",
            "field": "best_capture_approach",
            "kind": "value",
            "expected": "heterotypic_cell_conjugate"
          },
          {
            "name": "best_selection_marker_gate",
            "role": "gate",
            "field": "best_selection_marker",
            "kind": "value",
            "expected": "none"
          },
          {
            "name": "reactivity_criterion_gate",
            "role": "gate",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "clonal_expansion_and_physical_engagement"
          },
          {
            "name": "cross_patient_robustness_gate",
            "role": "gate",
            "field": "cross_patient_robustness",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_across_patients",
              "marginal_in_some_patients"
            ]
          },
          {
            "name": "marker_selection_consequence_gate",
            "role": "gate",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "discards_clonally_expanded_reactive_cells"
          },
          {
            "name": "superiority_claim_basis_gate",
            "role": "gate",
            "field": "superiority_claim_basis",
            "kind": "value",
            "expected": "requires_functional_reactivity_assay"
          },
          {
            "name": "conjugate_enriched_state_gate",
            "role": "gate",
            "field": "conjugate_enriched_state",
            "kind": "value",
            "expected": "progenitor_exhausted_TCF7_pos_stem_like"
          },
          {
            "name": "cf_circular_exhaustion_criterion",
            "role": "hard_fail",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "exhaustion_signature_intensity"
          },
          {
            "name": "cf_marker_sort_prior",
            "role": "hard_fail",
            "field": "best_selection_marker",
            "kind": "value",
            "op": "in",
            "expected": [
              "CD39",
              "PD1",
              "PD1_high",
              "TIM3",
              "PD1_CD39",
              "CD39_CD103"
            ]
          },
          {
            "name": "cf_single_cell_capture_prior",
            "role": "hard_fail",
            "field": "best_capture_approach",
            "kind": "value",
            "op": "in",
            "expected": [
              "dissociated_single_cell",
              "dissociated_bulk_population"
            ]
          },
          {
            "name": "cf_off_data_capture_gate",
            "role": "hard_fail",
            "field": "$.population_findings[*].capture_gate",
            "kind": "avoid",
            "expected": [
              "dissociated_bulk",
              "matched_peripheral_blood",
              "matched_lymph_node",
              "post_hoc_expanded_clonotype_group"
            ]
          },
          {
            "name": "cf_conjugate_mischaracterized",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_marker_singlet_called_engaged",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "contains",
            "matchKey": "capture_gate",
            "entity": "marker_sorted_singlet",
            "valueField": "reactivity_evidence",
            "expected": [
              "engagement_and_clonal_expansion"
            ]
          },
          {
            "name": "cf_no_engaged_reactive_population",
            "role": "hard_fail",
            "field": "$.population_findings[*].reactivity_evidence",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_marker_captures_all_reactive",
            "role": "hard_fail",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "captures_full_reactive_repertoire"
          },
          {
            "name": "cf_superiority_overclaimed_from_phenotype",
            "role": "hard_fail",
            "field": "superiority_claim_basis",
            "kind": "value",
            "op": "in",
            "expected": [
              "established_by_scrna_phenotype",
              "established_by_reactivity_proxy_scores"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-kimi-k2p6-pi",
      "modelName": "kimi-k2p6",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_vs_cd39_tumor_reactivity_dominance",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 1325,
      "cost": 0.56777667,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "population_findings",
            "kind": "list",
            "matchKey": "capture_gate",
            "k": 4,
            "expected": [
              {
                "key": "tumor_cell_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "apc_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "combined_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "unselected_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "naive",
                      "central_memory",
                      "effector_memory"
                    ],
                    "canonical": "naive"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "low"
                    ],
                    "canonical": "low"
                  },
                  "purity_caveat": {
                    "op": "equals",
                    "values": [
                      "pure_T_cell"
                    ],
                    "canonical": "pure_T_cell"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "no_reactivity_evidence"
                    ],
                    "canonical": "no_reactivity_evidence"
                  }
                }
              },
              {
                "key": "marker_sorted_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "equals",
                    "values": [
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "terminal_exhausted_LAG3_high"
                  },
                  "clonal_expansion_level": {
                    "op": "in",
                    "values": [
                      "high",
                      "intermediate"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "marker_on_non_T_lineages",
                      "single_cell_gating_loses_target"
                    ],
                    "canonical": "marker_on_non_T_lineages"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "marker_or_exhaustion_defined_only"
                    ],
                    "canonical": "marker_or_exhaustion_defined_only"
                  }
                }
              }
            ]
          },
          {
            "name": "best_capture_approach_gate",
            "role": "gate",
            "field": "best_capture_approach",
            "kind": "value",
            "expected": "heterotypic_cell_conjugate"
          },
          {
            "name": "best_selection_marker_gate",
            "role": "gate",
            "field": "best_selection_marker",
            "kind": "value",
            "expected": "none"
          },
          {
            "name": "reactivity_criterion_gate",
            "role": "gate",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "clonal_expansion_and_physical_engagement"
          },
          {
            "name": "cross_patient_robustness_gate",
            "role": "gate",
            "field": "cross_patient_robustness",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_across_patients",
              "marginal_in_some_patients"
            ]
          },
          {
            "name": "marker_selection_consequence_gate",
            "role": "gate",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "discards_clonally_expanded_reactive_cells"
          },
          {
            "name": "superiority_claim_basis_gate",
            "role": "gate",
            "field": "superiority_claim_basis",
            "kind": "value",
            "expected": "requires_functional_reactivity_assay"
          },
          {
            "name": "conjugate_enriched_state_gate",
            "role": "gate",
            "field": "conjugate_enriched_state",
            "kind": "value",
            "expected": "progenitor_exhausted_TCF7_pos_stem_like"
          },
          {
            "name": "cf_circular_exhaustion_criterion",
            "role": "hard_fail",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "exhaustion_signature_intensity"
          },
          {
            "name": "cf_marker_sort_prior",
            "role": "hard_fail",
            "field": "best_selection_marker",
            "kind": "value",
            "op": "in",
            "expected": [
              "CD39",
              "PD1",
              "PD1_high",
              "TIM3",
              "PD1_CD39",
              "CD39_CD103"
            ]
          },
          {
            "name": "cf_single_cell_capture_prior",
            "role": "hard_fail",
            "field": "best_capture_approach",
            "kind": "value",
            "op": "in",
            "expected": [
              "dissociated_single_cell",
              "dissociated_bulk_population"
            ]
          },
          {
            "name": "cf_off_data_capture_gate",
            "role": "hard_fail",
            "field": "$.population_findings[*].capture_gate",
            "kind": "avoid",
            "expected": [
              "dissociated_bulk",
              "matched_peripheral_blood",
              "matched_lymph_node",
              "post_hoc_expanded_clonotype_group"
            ]
          },
          {
            "name": "cf_conjugate_mischaracterized",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_marker_singlet_called_engaged",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "contains",
            "matchKey": "capture_gate",
            "entity": "marker_sorted_singlet",
            "valueField": "reactivity_evidence",
            "expected": [
              "engagement_and_clonal_expansion"
            ]
          },
          {
            "name": "cf_no_engaged_reactive_population",
            "role": "hard_fail",
            "field": "$.population_findings[*].reactivity_evidence",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_marker_captures_all_reactive",
            "role": "hard_fail",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "captures_full_reactive_repertoire"
          },
          {
            "name": "cf_superiority_overclaimed_from_phenotype",
            "role": "hard_fail",
            "field": "superiority_claim_basis",
            "kind": "value",
            "op": "in",
            "expected": [
              "established_by_scrna_phenotype",
              "established_by_reactivity_proxy_scores"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-kimi-k2p6-pi",
      "modelName": "kimi-k2p6",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_vs_cd39_tumor_reactivity_dominance",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1252,
      "cost": 0.54844167,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "population_findings",
            "kind": "list",
            "matchKey": "capture_gate",
            "k": 4,
            "expected": [
              {
                "key": "tumor_cell_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "apc_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "combined_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "unselected_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "naive",
                      "central_memory",
                      "effector_memory"
                    ],
                    "canonical": "naive"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "low"
                    ],
                    "canonical": "low"
                  },
                  "purity_caveat": {
                    "op": "equals",
                    "values": [
                      "pure_T_cell"
                    ],
                    "canonical": "pure_T_cell"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "no_reactivity_evidence"
                    ],
                    "canonical": "no_reactivity_evidence"
                  }
                }
              },
              {
                "key": "marker_sorted_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "equals",
                    "values": [
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "terminal_exhausted_LAG3_high"
                  },
                  "clonal_expansion_level": {
                    "op": "in",
                    "values": [
                      "high",
                      "intermediate"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "marker_on_non_T_lineages",
                      "single_cell_gating_loses_target"
                    ],
                    "canonical": "marker_on_non_T_lineages"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "marker_or_exhaustion_defined_only"
                    ],
                    "canonical": "marker_or_exhaustion_defined_only"
                  }
                }
              }
            ]
          },
          {
            "name": "best_capture_approach_gate",
            "role": "gate",
            "field": "best_capture_approach",
            "kind": "value",
            "expected": "heterotypic_cell_conjugate"
          },
          {
            "name": "best_selection_marker_gate",
            "role": "gate",
            "field": "best_selection_marker",
            "kind": "value",
            "expected": "none"
          },
          {
            "name": "reactivity_criterion_gate",
            "role": "gate",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "clonal_expansion_and_physical_engagement"
          },
          {
            "name": "cross_patient_robustness_gate",
            "role": "gate",
            "field": "cross_patient_robustness",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_across_patients",
              "marginal_in_some_patients"
            ]
          },
          {
            "name": "marker_selection_consequence_gate",
            "role": "gate",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "discards_clonally_expanded_reactive_cells"
          },
          {
            "name": "superiority_claim_basis_gate",
            "role": "gate",
            "field": "superiority_claim_basis",
            "kind": "value",
            "expected": "requires_functional_reactivity_assay"
          },
          {
            "name": "conjugate_enriched_state_gate",
            "role": "gate",
            "field": "conjugate_enriched_state",
            "kind": "value",
            "expected": "progenitor_exhausted_TCF7_pos_stem_like"
          },
          {
            "name": "cf_circular_exhaustion_criterion",
            "role": "hard_fail",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "exhaustion_signature_intensity"
          },
          {
            "name": "cf_marker_sort_prior",
            "role": "hard_fail",
            "field": "best_selection_marker",
            "kind": "value",
            "op": "in",
            "expected": [
              "CD39",
              "PD1",
              "PD1_high",
              "TIM3",
              "PD1_CD39",
              "CD39_CD103"
            ]
          },
          {
            "name": "cf_single_cell_capture_prior",
            "role": "hard_fail",
            "field": "best_capture_approach",
            "kind": "value",
            "op": "in",
            "expected": [
              "dissociated_single_cell",
              "dissociated_bulk_population"
            ]
          },
          {
            "name": "cf_off_data_capture_gate",
            "role": "hard_fail",
            "field": "$.population_findings[*].capture_gate",
            "kind": "avoid",
            "expected": [
              "dissociated_bulk",
              "matched_peripheral_blood",
              "matched_lymph_node",
              "post_hoc_expanded_clonotype_group"
            ]
          },
          {
            "name": "cf_conjugate_mischaracterized",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_marker_singlet_called_engaged",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "contains",
            "matchKey": "capture_gate",
            "entity": "marker_sorted_singlet",
            "valueField": "reactivity_evidence",
            "expected": [
              "engagement_and_clonal_expansion"
            ]
          },
          {
            "name": "cf_no_engaged_reactive_population",
            "role": "hard_fail",
            "field": "$.population_findings[*].reactivity_evidence",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_marker_captures_all_reactive",
            "role": "hard_fail",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "captures_full_reactive_repertoire"
          },
          {
            "name": "cf_superiority_overclaimed_from_phenotype",
            "role": "hard_fail",
            "field": "superiority_claim_basis",
            "kind": "value",
            "op": "in",
            "expected": [
              "established_by_scrna_phenotype",
              "established_by_reactivity_proxy_scores"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-6-claude-code",
      "modelName": "claude-opus-4-6",
      "harness": "claude-code",
      "task": "10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 1,
      "passed": true,
      "score": 15,
      "durationS": 1027.28,
      "cost": 1.6065779999999998,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_singlet_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "TOXhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "LAG3high_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "GZMKhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "intermediate_exhausted_gzmk"
                    ],
                    "canonical": "intermediate_exhausted_gzmk"
                  }
                }
              },
              {
                "key": "TCF7_stem_like_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "stem_like_exhausted"
                    ],
                    "canonical": "stem_like_exhausted"
                  }
                }
              },
              {
                "key": "MKI67_pos_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              },
              {
                "key": "MKI67high_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_tumor_cluster_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "Tn",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "Tn_Tmem",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "early_Tem_GZMK",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              },
              {
                "key": "Tem_GZMH",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              }
            ]
          },
          {
            "name": "clonotype_expansion_direction_data_true",
            "role": "gate",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_substantially_more_expanded_than_cluster",
              "singlet_modestly_more_expanded_than_cluster"
            ]
          },
          {
            "name": "biological_inference_data_true",
            "role": "gate",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_antigen_experienced_cluster_bystander"
            ]
          },
          {
            "name": "singlet_top3_contains_naive_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Tn",
              "Tn_Tmem",
              "early_Tem_GZMK",
              "Tem_GZMH"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "TOXhigh_Tex",
              "LAG3high_Tex",
              "GZMKhigh_Tex",
              "TCF7_stem_like_Tex",
              "MKI67_pos_Tex_Tprol",
              "MKI67high_Tex_Tprol"
            ]
          },
          {
            "name": "singlet_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "TOXhigh_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "TCF7_stem_like_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "MKI67_state_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "Tn_or_Tn_Tmem_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_stemlike_precursor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_reported_canonical_expansion_direction",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_substantially_more_expanded_than_singlet",
              "cluster_modestly_more_expanded_than_singlet"
            ]
          },
          {
            "name": "cf_expansion_lazy_defer",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "expansion_pattern_not_detectable"
            ]
          },
          {
            "name": "cf_reported_canonical_expected_biology",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_antigen_experienced_singlet_bystander"
            ]
          },
          {
            "name": "cf_biological_inference_null_artifact_or_lineage",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "differences_reflect_dissociation_or_sorting_artifact",
              "cluster_and_singlet_capture_same_biology",
              "cluster_and_singlet_capture_distinct_cd8_lineages"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "biological_relations",
            "kind": "dict",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_greater_than",
                  "greater_than"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "less_than",
                  "much_less_than"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than",
                  "approximately_equal"
                ]
              },
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal"
                ]
              },
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal",
                  "greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_exhaustion_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_naive_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_proliferation_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_expansion_richness_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_multichain_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_intermediate_gzmk_overgeneralized",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_melanocytic_conjugate_overcall",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "values": [
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_relations_mass_hedge",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              }
            }
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: PASS\n  + list_match: list_match (answer_field='cd8_states_enriched_in_singlet_fraction'): PASS  (score=3.0)\n  + list_match: list_match (answer_field='cd8_states_enriched_in_tumor_cluster_fraction'): PASS  (score=3.0)\n  + clonotype_expansion_direction_data_true: passed=True, score=1.0\n  + biological_inference_data_true: passed=True, score=1.0\n  + dict_match: dict_match (answer_field='biological_relations'): PASS  (score=7.0)\n  + hard_fail 'singlet_top3_contains_naive_state_prior_pushing': not triggered\n  + hard_fail 'tumor_cluster_top3_contains_exhausted_state_prior_pushing': not triggered\n  + hard_fail 'singlet_top3_contains_decoy_token': not triggered\n  + hard_fail 'tumor_cluster_top3_contains_decoy_token': not triggered\n  + hard_fail 'TOXhigh_Tex_wrong_functional_category': not triggered\n  + hard_fail 'TCF7_stem_like_Tex_wrong_functional_category': not triggered\n  + hard_fail 'MKI67_state_wrong_functional_category': not triggered\n  + hard_fail 'Tn_or_Tn_Tmem_wrong_functional_category': not triggered\n  + hard_fail 'missing_exhausted_or_proliferating_anchor_in_singlet_top3': not triggered\n  + hard_fail 'missing_stemlike_precursor_in_singlet_top3': not triggered\n  + hard_fail 'missing_naive_or_early_memory_anchor_in_tumor_cluster_top3': not triggered\n  + hard_fail 'cf_reported_canonical_expansion_direction': not triggered\n  + hard_fail 'cf_expansion_lazy_defer': not triggered\n  + hard_fail 'cf_reported_canonical_expected_biology': not triggered\n  + hard_fail 'cf_biological_inference_null_artifact_or_lineage': not triggered\n  + hard_fail 'cf_exhaustion_inverted_prior_push': not triggered\n  + hard_fail 'cf_naive_inverted_prior_push': not triggered\n  + hard_fail 'cf_proliferation_inverted_prior_push': not triggered\n  + hard_fail 'cf_expansion_richness_inverted_prior_push': not triggered\n  + hard_fail 'cf_multichain_inverted_prior_push': not triggered\n  + hard_fail 'cf_intermediate_gzmk_overgeneralized': not triggered\n  + hard_fail 'cf_melanocytic_conjugate_overcall': not triggered\n  + hard_fail 'cf_relations_mass_hedge': not triggered",
      "metrics": {
        "hard_fail_triggered": [],
        "pass_rule": "all",
        "scoring_count": 5,
        "scoring_passed": 5,
        "scoring_total_score": 15
      },
      "fieldScores": {
        "biological_inference_data_true": 1,
        "clonotype_expansion_direction_data_true": 1,
        "dict_match": 7,
        "list_match": 3
      },
      "graderChecks": [
        {
          "name": "cd8_states_enriched_in_singlet_fraction",
          "kind": "list_match",
          "passed": true,
          "score": 3
        },
        {
          "name": "cd8_states_enriched_in_tumor_cluster_fraction",
          "kind": "list_match",
          "passed": true,
          "score": 3
        },
        {
          "name": "clonotype_expansion_direction_data_true",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "biological_inference_data_true",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "biological_relations",
          "kind": "list_match",
          "passed": true,
          "score": 7
        },
        {
          "name": "singlet_top3_contains_naive_state_prior_pushing",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "singlet_top3_contains_decoy_token",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "tumor_cluster_top3_contains_decoy_token",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "TOXhigh_Tex_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "TCF7_stem_like_Tex_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "MKI67_state_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "Tn_or_Tn_Tmem_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_stemlike_precursor_in_singlet_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_reported_canonical_expansion_direction",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_expansion_lazy_defer",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_reported_canonical_expected_biology",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_biological_inference_null_artifact_or_lineage",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_exhaustion_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_naive_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_proliferation_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_expansion_richness_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_multichain_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_intermediate_gzmk_overgeneralized",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_melanocytic_conjugate_overcall",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_relations_mass_hedge",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "biological_inference": "singlet_antigen_experienced_cluster_bystander",
        "biological_relations": {
          "clonal_expansion_richness_in_singlets_vs_tumor_cluster": "greater_than",
          "exhaustion_program_in_singlets_vs_tumor_cluster": "greater_than",
          "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": "approximately_equal",
          "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": "greater_than",
          "multichain_tcr_rate_in_singlets_vs_tumor_cluster": "greater_than",
          "naive_memory_program_in_singlets_vs_tumor_cluster": "less_than",
          "proliferation_program_in_singlets_vs_tumor_cluster": "greater_than"
        },
        "cd8_states_enriched_in_singlet_fraction": [
          {
            "functional_category": "stem_like_exhausted",
            "state_label": "TCF7_stem_like_Tex"
          },
          {
            "functional_category": "terminal_exhausted",
            "state_label": "TOXhigh_Tex"
          },
          {
            "functional_category": "proliferating",
            "state_label": "MKI67high_Tex_Tprol"
          }
        ],
        "cd8_states_enriched_in_tumor_cluster_fraction": [
          {
            "functional_category": "naive_memory",
            "state_label": "Tn"
          },
          {
            "functional_category": "naive_memory",
            "state_label": "Tn_Tmem"
          },
          {
            "functional_category": "early_effector_memory",
            "state_label": "early_Tem_GZMK"
          }
        ],
        "clonotype_expansion_finding": "singlet_modestly_more_expanded_than_cluster"
      },
      "nTurns": 17,
      "checkpointSignals": {
        "clustering": true,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": false,
        "nerveJointAnchor": false,
        "multiomeAssembly": false,
        "depthNormalizedAtac": false,
        "regulonInference": false,
        "cd8SubsetIntegration": true,
        "perPatientEnrichment": true,
        "tcrRichnessMetrics": true,
        "orthologProjection": false,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": true,
        "mousePseudobulkDesign": false,
        "cancerEpithelialDenominator": false,
        "geneLevelDe": false,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "trajectoryFile": "scbench-long/trajectories/10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1__claude-opus-4-6__claude-code__r1.json",
      "runId": "claude-opus-4-6__claude-code__10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1__r1",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-opus-4-6-claude-code",
      "modelName": "claude-opus-4-6",
      "harness": "claude-code",
      "task": "10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 2,
      "passed": true,
      "score": 15,
      "durationS": 2030.48,
      "cost": 2.6382179999999993,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_singlet_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "TOXhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "LAG3high_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "GZMKhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "intermediate_exhausted_gzmk"
                    ],
                    "canonical": "intermediate_exhausted_gzmk"
                  }
                }
              },
              {
                "key": "TCF7_stem_like_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "stem_like_exhausted"
                    ],
                    "canonical": "stem_like_exhausted"
                  }
                }
              },
              {
                "key": "MKI67_pos_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              },
              {
                "key": "MKI67high_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_tumor_cluster_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "Tn",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "Tn_Tmem",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "early_Tem_GZMK",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              },
              {
                "key": "Tem_GZMH",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              }
            ]
          },
          {
            "name": "clonotype_expansion_direction_data_true",
            "role": "gate",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_substantially_more_expanded_than_cluster",
              "singlet_modestly_more_expanded_than_cluster"
            ]
          },
          {
            "name": "biological_inference_data_true",
            "role": "gate",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_antigen_experienced_cluster_bystander"
            ]
          },
          {
            "name": "singlet_top3_contains_naive_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Tn",
              "Tn_Tmem",
              "early_Tem_GZMK",
              "Tem_GZMH"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "TOXhigh_Tex",
              "LAG3high_Tex",
              "GZMKhigh_Tex",
              "TCF7_stem_like_Tex",
              "MKI67_pos_Tex_Tprol",
              "MKI67high_Tex_Tprol"
            ]
          },
          {
            "name": "singlet_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "TOXhigh_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "TCF7_stem_like_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "MKI67_state_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "Tn_or_Tn_Tmem_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_stemlike_precursor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_reported_canonical_expansion_direction",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_substantially_more_expanded_than_singlet",
              "cluster_modestly_more_expanded_than_singlet"
            ]
          },
          {
            "name": "cf_expansion_lazy_defer",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "expansion_pattern_not_detectable"
            ]
          },
          {
            "name": "cf_reported_canonical_expected_biology",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_antigen_experienced_singlet_bystander"
            ]
          },
          {
            "name": "cf_biological_inference_null_artifact_or_lineage",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "differences_reflect_dissociation_or_sorting_artifact",
              "cluster_and_singlet_capture_same_biology",
              "cluster_and_singlet_capture_distinct_cd8_lineages"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "biological_relations",
            "kind": "dict",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_greater_than",
                  "greater_than"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "less_than",
                  "much_less_than"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than",
                  "approximately_equal"
                ]
              },
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal"
                ]
              },
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal",
                  "greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_exhaustion_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_naive_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_proliferation_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_expansion_richness_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_multichain_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_intermediate_gzmk_overgeneralized",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_melanocytic_conjugate_overcall",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "values": [
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_relations_mass_hedge",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              }
            }
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: PASS\n  + list_match: list_match (answer_field='cd8_states_enriched_in_singlet_fraction'): PASS  (score=3.0)\n  + list_match: list_match (answer_field='cd8_states_enriched_in_tumor_cluster_fraction'): PASS  (score=3.0)\n  + clonotype_expansion_direction_data_true: passed=True, score=1.0\n  + biological_inference_data_true: passed=True, score=1.0\n  + dict_match: dict_match (answer_field='biological_relations'): PASS  (score=7.0)\n  + hard_fail 'singlet_top3_contains_naive_state_prior_pushing': not triggered\n  + hard_fail 'tumor_cluster_top3_contains_exhausted_state_prior_pushing': not triggered\n  + hard_fail 'singlet_top3_contains_decoy_token': not triggered\n  + hard_fail 'tumor_cluster_top3_contains_decoy_token': not triggered\n  + hard_fail 'TOXhigh_Tex_wrong_functional_category': not triggered\n  + hard_fail 'TCF7_stem_like_Tex_wrong_functional_category': not triggered\n  + hard_fail 'MKI67_state_wrong_functional_category': not triggered\n  + hard_fail 'Tn_or_Tn_Tmem_wrong_functional_category': not triggered\n  + hard_fail 'missing_exhausted_or_proliferating_anchor_in_singlet_top3': not triggered\n  + hard_fail 'missing_stemlike_precursor_in_singlet_top3': not triggered\n  + hard_fail 'missing_naive_or_early_memory_anchor_in_tumor_cluster_top3': not triggered\n  + hard_fail 'cf_reported_canonical_expansion_direction': not triggered\n  + hard_fail 'cf_expansion_lazy_defer': not triggered\n  + hard_fail 'cf_reported_canonical_expected_biology': not triggered\n  + hard_fail 'cf_biological_inference_null_artifact_or_lineage': not triggered\n  + hard_fail 'cf_exhaustion_inverted_prior_push': not triggered\n  + hard_fail 'cf_naive_inverted_prior_push': not triggered\n  + hard_fail 'cf_proliferation_inverted_prior_push': not triggered\n  + hard_fail 'cf_expansion_richness_inverted_prior_push': not triggered\n  + hard_fail 'cf_multichain_inverted_prior_push': not triggered\n  + hard_fail 'cf_intermediate_gzmk_overgeneralized': not triggered\n  + hard_fail 'cf_melanocytic_conjugate_overcall': not triggered\n  + hard_fail 'cf_relations_mass_hedge': not triggered",
      "metrics": {
        "hard_fail_triggered": [],
        "pass_rule": "all",
        "scoring_count": 5,
        "scoring_passed": 5,
        "scoring_total_score": 15
      },
      "fieldScores": {
        "biological_inference_data_true": 1,
        "clonotype_expansion_direction_data_true": 1,
        "dict_match": 7,
        "list_match": 3
      },
      "graderChecks": [
        {
          "name": "cd8_states_enriched_in_singlet_fraction",
          "kind": "list_match",
          "passed": true,
          "score": 3
        },
        {
          "name": "cd8_states_enriched_in_tumor_cluster_fraction",
          "kind": "list_match",
          "passed": true,
          "score": 3
        },
        {
          "name": "clonotype_expansion_direction_data_true",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "biological_inference_data_true",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "biological_relations",
          "kind": "list_match",
          "passed": true,
          "score": 7
        },
        {
          "name": "singlet_top3_contains_naive_state_prior_pushing",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "singlet_top3_contains_decoy_token",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "tumor_cluster_top3_contains_decoy_token",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "TOXhigh_Tex_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "TCF7_stem_like_Tex_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "MKI67_state_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "Tn_or_Tn_Tmem_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_stemlike_precursor_in_singlet_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_reported_canonical_expansion_direction",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_expansion_lazy_defer",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_reported_canonical_expected_biology",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_biological_inference_null_artifact_or_lineage",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_exhaustion_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_naive_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_proliferation_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_expansion_richness_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_multichain_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_intermediate_gzmk_overgeneralized",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_melanocytic_conjugate_overcall",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_relations_mass_hedge",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "biological_inference": "singlet_antigen_experienced_cluster_bystander",
        "biological_relations": {
          "clonal_expansion_richness_in_singlets_vs_tumor_cluster": "greater_than",
          "exhaustion_program_in_singlets_vs_tumor_cluster": "greater_than",
          "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": "approximately_equal",
          "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": "approximately_equal",
          "multichain_tcr_rate_in_singlets_vs_tumor_cluster": "greater_than",
          "naive_memory_program_in_singlets_vs_tumor_cluster": "less_than",
          "proliferation_program_in_singlets_vs_tumor_cluster": "greater_than"
        },
        "cd8_states_enriched_in_singlet_fraction": [
          {
            "functional_category": "stem_like_exhausted",
            "state_label": "TCF7_stem_like_Tex"
          },
          {
            "functional_category": "terminal_exhausted",
            "state_label": "TOXhigh_Tex"
          },
          {
            "functional_category": "proliferating",
            "state_label": "MKI67high_Tex_Tprol"
          }
        ],
        "cd8_states_enriched_in_tumor_cluster_fraction": [
          {
            "functional_category": "naive_memory",
            "state_label": "Tn"
          },
          {
            "functional_category": "naive_memory",
            "state_label": "Tn_Tmem"
          },
          {
            "functional_category": "early_effector_memory",
            "state_label": "early_Tem_GZMK"
          }
        ],
        "clonotype_expansion_finding": "singlet_modestly_more_expanded_than_cluster"
      },
      "nTurns": 19,
      "checkpointSignals": {
        "clustering": true,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": false,
        "nerveJointAnchor": false,
        "multiomeAssembly": false,
        "depthNormalizedAtac": false,
        "regulonInference": false,
        "cd8SubsetIntegration": true,
        "perPatientEnrichment": true,
        "tcrRichnessMetrics": true,
        "orthologProjection": false,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": true,
        "mousePseudobulkDesign": false,
        "cancerEpithelialDenominator": false,
        "geneLevelDe": false,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "trajectoryFile": "scbench-long/trajectories/10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1__claude-opus-4-6__claude-code__r2.json",
      "runId": "claude-opus-4-6__claude-code__10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1__r2",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-opus-4-6-claude-code",
      "modelName": "claude-opus-4-6",
      "harness": "claude-code",
      "task": "10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": false,
      "score": 13,
      "durationS": 1879.78,
      "cost": 2.3290217500000003,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_singlet_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "TOXhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "LAG3high_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "GZMKhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "intermediate_exhausted_gzmk"
                    ],
                    "canonical": "intermediate_exhausted_gzmk"
                  }
                }
              },
              {
                "key": "TCF7_stem_like_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "stem_like_exhausted"
                    ],
                    "canonical": "stem_like_exhausted"
                  }
                }
              },
              {
                "key": "MKI67_pos_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              },
              {
                "key": "MKI67high_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_tumor_cluster_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "Tn",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "Tn_Tmem",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "early_Tem_GZMK",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              },
              {
                "key": "Tem_GZMH",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              }
            ]
          },
          {
            "name": "clonotype_expansion_direction_data_true",
            "role": "gate",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_substantially_more_expanded_than_cluster",
              "singlet_modestly_more_expanded_than_cluster"
            ]
          },
          {
            "name": "biological_inference_data_true",
            "role": "gate",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_antigen_experienced_cluster_bystander"
            ]
          },
          {
            "name": "singlet_top3_contains_naive_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Tn",
              "Tn_Tmem",
              "early_Tem_GZMK",
              "Tem_GZMH"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "TOXhigh_Tex",
              "LAG3high_Tex",
              "GZMKhigh_Tex",
              "TCF7_stem_like_Tex",
              "MKI67_pos_Tex_Tprol",
              "MKI67high_Tex_Tprol"
            ]
          },
          {
            "name": "singlet_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "TOXhigh_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "TCF7_stem_like_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "MKI67_state_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "Tn_or_Tn_Tmem_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_stemlike_precursor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_reported_canonical_expansion_direction",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_substantially_more_expanded_than_singlet",
              "cluster_modestly_more_expanded_than_singlet"
            ]
          },
          {
            "name": "cf_expansion_lazy_defer",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "expansion_pattern_not_detectable"
            ]
          },
          {
            "name": "cf_reported_canonical_expected_biology",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_antigen_experienced_singlet_bystander"
            ]
          },
          {
            "name": "cf_biological_inference_null_artifact_or_lineage",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "differences_reflect_dissociation_or_sorting_artifact",
              "cluster_and_singlet_capture_same_biology",
              "cluster_and_singlet_capture_distinct_cd8_lineages"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "biological_relations",
            "kind": "dict",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_greater_than",
                  "greater_than"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "less_than",
                  "much_less_than"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than",
                  "approximately_equal"
                ]
              },
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal"
                ]
              },
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal",
                  "greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_exhaustion_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_naive_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_proliferation_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_expansion_richness_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_multichain_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_intermediate_gzmk_overgeneralized",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_melanocytic_conjugate_overcall",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "values": [
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_relations_mass_hedge",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              }
            }
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: FAIL\n  + list_match: list_match (answer_field='cd8_states_enriched_in_singlet_fraction'): PASS  (score=3.0)\n  + list_match: list_match (answer_field='cd8_states_enriched_in_tumor_cluster_fraction'): PASS  (score=2.0)\n  + clonotype_expansion_direction_data_true: passed=True, score=1.0\n  + biological_inference_data_true: passed=True, score=1.0\n  x dict_match: dict_match (answer_field='biological_relations'): FAIL  (score=6.0)\n  + hard_fail 'singlet_top3_contains_naive_state_prior_pushing': not triggered\n  + hard_fail 'tumor_cluster_top3_contains_exhausted_state_prior_pushing': not triggered\n  + hard_fail 'singlet_top3_contains_decoy_token': not triggered\n  + hard_fail 'tumor_cluster_top3_contains_decoy_token': not triggered\n  + hard_fail 'TOXhigh_Tex_wrong_functional_category': not triggered\n  + hard_fail 'TCF7_stem_like_Tex_wrong_functional_category': not triggered\n  + hard_fail 'MKI67_state_wrong_functional_category': not triggered\n  + hard_fail 'Tn_or_Tn_Tmem_wrong_functional_category': not triggered\n  + hard_fail 'missing_exhausted_or_proliferating_anchor_in_singlet_top3': not triggered\n  + hard_fail 'missing_stemlike_precursor_in_singlet_top3': not triggered\n  + hard_fail 'missing_naive_or_early_memory_anchor_in_tumor_cluster_top3': not triggered\n  + hard_fail 'cf_reported_canonical_expansion_direction': not triggered\n  + hard_fail 'cf_expansion_lazy_defer': not triggered\n  + hard_fail 'cf_reported_canonical_expected_biology': not triggered\n  + hard_fail 'cf_biological_inference_null_artifact_or_lineage': not triggered\n  + hard_fail 'cf_exhaustion_inverted_prior_push': not triggered\n  + hard_fail 'cf_naive_inverted_prior_push': not triggered\n  + hard_fail 'cf_proliferation_inverted_prior_push': not triggered\n  + hard_fail 'cf_expansion_richness_inverted_prior_push': not triggered\n  + hard_fail 'cf_multichain_inverted_prior_push': not triggered\n  + hard_fail 'cf_intermediate_gzmk_overgeneralized': not triggered\n  + hard_fail 'cf_melanocytic_conjugate_overcall': not triggered\n  + hard_fail 'cf_relations_mass_hedge': not triggered",
      "metrics": {
        "hard_fail_triggered": [],
        "pass_rule": "all",
        "scoring_count": 5,
        "scoring_passed": 4,
        "scoring_total_score": 13
      },
      "fieldScores": {
        "biological_inference_data_true": 1,
        "clonotype_expansion_direction_data_true": 1,
        "dict_match": 6,
        "list_match": 2
      },
      "graderChecks": [
        {
          "name": "cd8_states_enriched_in_singlet_fraction",
          "kind": "list_match",
          "passed": true,
          "score": 3
        },
        {
          "name": "cd8_states_enriched_in_tumor_cluster_fraction",
          "kind": "list_match",
          "passed": true,
          "score": 2
        },
        {
          "name": "clonotype_expansion_direction_data_true",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "biological_inference_data_true",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "biological_relations",
          "kind": "list_match",
          "passed": false,
          "score": 6
        },
        {
          "name": "singlet_top3_contains_naive_state_prior_pushing",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "singlet_top3_contains_decoy_token",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "tumor_cluster_top3_contains_decoy_token",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "TOXhigh_Tex_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "TCF7_stem_like_Tex_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "MKI67_state_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "Tn_or_Tn_Tmem_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_stemlike_precursor_in_singlet_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_reported_canonical_expansion_direction",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_expansion_lazy_defer",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_reported_canonical_expected_biology",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_biological_inference_null_artifact_or_lineage",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_exhaustion_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_naive_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_proliferation_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_expansion_richness_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_multichain_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_intermediate_gzmk_overgeneralized",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_melanocytic_conjugate_overcall",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_relations_mass_hedge",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "biological_inference": "singlet_antigen_experienced_cluster_bystander",
        "biological_relations": {
          "clonal_expansion_richness_in_singlets_vs_tumor_cluster": "greater_than",
          "exhaustion_program_in_singlets_vs_tumor_cluster": "greater_than",
          "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": "approximately_equal",
          "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": "greater_than",
          "multichain_tcr_rate_in_singlets_vs_tumor_cluster": "greater_than",
          "naive_memory_program_in_singlets_vs_tumor_cluster": "much_less_than",
          "proliferation_program_in_singlets_vs_tumor_cluster": "approximately_equal"
        },
        "cd8_states_enriched_in_singlet_fraction": [
          {
            "functional_category": "stem_like_exhausted",
            "state_label": "TCF7_stem_like_Tex"
          },
          {
            "functional_category": "intermediate_exhausted_gzmk",
            "state_label": "GZMKhigh_Tex"
          },
          {
            "functional_category": "terminal_exhausted",
            "state_label": "TOXhigh_Tex"
          }
        ],
        "cd8_states_enriched_in_tumor_cluster_fraction": [
          {
            "functional_category": "naive_memory",
            "state_label": "Tn_Tmem"
          },
          {
            "functional_category": "early_effector_memory",
            "state_label": "early_Tem_GZMK"
          },
          {
            "functional_category": "early_effector_memory",
            "state_label": "Tem_NK_like"
          }
        ],
        "clonotype_expansion_finding": "singlet_modestly_more_expanded_than_cluster"
      },
      "nTurns": 19,
      "checkpointSignals": {
        "clustering": false,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": false,
        "nerveJointAnchor": false,
        "multiomeAssembly": false,
        "depthNormalizedAtac": false,
        "regulonInference": false,
        "cd8SubsetIntegration": true,
        "perPatientEnrichment": true,
        "tcrRichnessMetrics": true,
        "orthologProjection": false,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": true,
        "mousePseudobulkDesign": false,
        "cancerEpithelialDenominator": false,
        "geneLevelDe": false,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "trajectoryFile": "scbench-long/trajectories/10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1__claude-opus-4-6__claude-code__r3.json",
      "runId": "claude-opus-4-6__claude-code__10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1__r3",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-opus-4-6-pi",
      "modelName": "claude-opus-4-6",
      "harness": "pi",
      "task": "10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 1,
      "passed": false,
      "score": 14,
      "durationS": 1576.6,
      "cost": 2.558829,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_singlet_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "TOXhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "LAG3high_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "GZMKhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "intermediate_exhausted_gzmk"
                    ],
                    "canonical": "intermediate_exhausted_gzmk"
                  }
                }
              },
              {
                "key": "TCF7_stem_like_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "stem_like_exhausted"
                    ],
                    "canonical": "stem_like_exhausted"
                  }
                }
              },
              {
                "key": "MKI67_pos_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              },
              {
                "key": "MKI67high_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_tumor_cluster_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "Tn",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "Tn_Tmem",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "early_Tem_GZMK",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              },
              {
                "key": "Tem_GZMH",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              }
            ]
          },
          {
            "name": "clonotype_expansion_direction_data_true",
            "role": "gate",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_substantially_more_expanded_than_cluster",
              "singlet_modestly_more_expanded_than_cluster"
            ]
          },
          {
            "name": "biological_inference_data_true",
            "role": "gate",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_antigen_experienced_cluster_bystander"
            ]
          },
          {
            "name": "singlet_top3_contains_naive_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Tn",
              "Tn_Tmem",
              "early_Tem_GZMK",
              "Tem_GZMH"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "TOXhigh_Tex",
              "LAG3high_Tex",
              "GZMKhigh_Tex",
              "TCF7_stem_like_Tex",
              "MKI67_pos_Tex_Tprol",
              "MKI67high_Tex_Tprol"
            ]
          },
          {
            "name": "singlet_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "TOXhigh_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "TCF7_stem_like_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "MKI67_state_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "Tn_or_Tn_Tmem_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_stemlike_precursor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_reported_canonical_expansion_direction",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_substantially_more_expanded_than_singlet",
              "cluster_modestly_more_expanded_than_singlet"
            ]
          },
          {
            "name": "cf_expansion_lazy_defer",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "expansion_pattern_not_detectable"
            ]
          },
          {
            "name": "cf_reported_canonical_expected_biology",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_antigen_experienced_singlet_bystander"
            ]
          },
          {
            "name": "cf_biological_inference_null_artifact_or_lineage",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "differences_reflect_dissociation_or_sorting_artifact",
              "cluster_and_singlet_capture_same_biology",
              "cluster_and_singlet_capture_distinct_cd8_lineages"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "biological_relations",
            "kind": "dict",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_greater_than",
                  "greater_than"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "less_than",
                  "much_less_than"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than",
                  "approximately_equal"
                ]
              },
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal"
                ]
              },
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal",
                  "greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_exhaustion_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_naive_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_proliferation_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_expansion_richness_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_multichain_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_intermediate_gzmk_overgeneralized",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_melanocytic_conjugate_overcall",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "values": [
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_relations_mass_hedge",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              }
            }
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: FAIL\n  + list_match: list_match (answer_field='cd8_states_enriched_in_singlet_fraction'): PASS  (score=3.0)\n  + list_match: list_match (answer_field='cd8_states_enriched_in_tumor_cluster_fraction'): PASS  (score=3.0)\n  + clonotype_expansion_direction_data_true: passed=True, score=1.0\n  + biological_inference_data_true: passed=True, score=1.0\n  x dict_match: dict_match (answer_field='biological_relations'): FAIL  (score=6.0)\n  + hard_fail 'singlet_top3_contains_naive_state_prior_pushing': not triggered\n  + hard_fail 'tumor_cluster_top3_contains_exhausted_state_prior_pushing': not triggered\n  + hard_fail 'singlet_top3_contains_decoy_token': not triggered\n  + hard_fail 'tumor_cluster_top3_contains_decoy_token': not triggered\n  + hard_fail 'TOXhigh_Tex_wrong_functional_category': not triggered\n  + hard_fail 'TCF7_stem_like_Tex_wrong_functional_category': not triggered\n  + hard_fail 'MKI67_state_wrong_functional_category': not triggered\n  + hard_fail 'Tn_or_Tn_Tmem_wrong_functional_category': not triggered\n  + hard_fail 'missing_exhausted_or_proliferating_anchor_in_singlet_top3': not triggered\n  + hard_fail 'missing_stemlike_precursor_in_singlet_top3': not triggered\n  + hard_fail 'missing_naive_or_early_memory_anchor_in_tumor_cluster_top3': not triggered\n  + hard_fail 'cf_reported_canonical_expansion_direction': not triggered\n  + hard_fail 'cf_expansion_lazy_defer': not triggered\n  + hard_fail 'cf_reported_canonical_expected_biology': not triggered\n  + hard_fail 'cf_biological_inference_null_artifact_or_lineage': not triggered\n  + hard_fail 'cf_exhaustion_inverted_prior_push': not triggered\n  + hard_fail 'cf_naive_inverted_prior_push': not triggered\n  + hard_fail 'cf_proliferation_inverted_prior_push': not triggered\n  + hard_fail 'cf_expansion_richness_inverted_prior_push': not triggered\n  + hard_fail 'cf_multichain_inverted_prior_push': not triggered\n  x hard_fail 'cf_intermediate_gzmk_overgeneralized': TRIGGERED\n  + hard_fail 'cf_melanocytic_conjugate_overcall': not triggered\n  + hard_fail 'cf_relations_mass_hedge': not triggered",
      "metrics": {
        "hard_fail_triggered": [
          "cf_intermediate_gzmk_overgeneralized"
        ],
        "pass_rule": "all",
        "scoring_count": 5,
        "scoring_passed": 4,
        "scoring_total_score": 14
      },
      "fieldScores": {
        "biological_inference_data_true": 1,
        "clonotype_expansion_direction_data_true": 1,
        "dict_match": 6,
        "list_match": 3
      },
      "graderChecks": [
        {
          "name": "cd8_states_enriched_in_singlet_fraction",
          "kind": "list_match",
          "passed": true,
          "score": 3
        },
        {
          "name": "cd8_states_enriched_in_tumor_cluster_fraction",
          "kind": "list_match",
          "passed": true,
          "score": 3
        },
        {
          "name": "clonotype_expansion_direction_data_true",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "biological_inference_data_true",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "biological_relations",
          "kind": "list_match",
          "passed": false,
          "score": 6
        },
        {
          "name": "singlet_top3_contains_naive_state_prior_pushing",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "singlet_top3_contains_decoy_token",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "tumor_cluster_top3_contains_decoy_token",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "TOXhigh_Tex_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "TCF7_stem_like_Tex_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "MKI67_state_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "Tn_or_Tn_Tmem_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_stemlike_precursor_in_singlet_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_reported_canonical_expansion_direction",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_expansion_lazy_defer",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_reported_canonical_expected_biology",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_biological_inference_null_artifact_or_lineage",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_exhaustion_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_naive_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_proliferation_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_expansion_richness_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_multichain_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_intermediate_gzmk_overgeneralized",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "cf_melanocytic_conjugate_overcall",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_relations_mass_hedge",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "biological_inference": "singlet_antigen_experienced_cluster_bystander",
        "biological_relations": {
          "clonal_expansion_richness_in_singlets_vs_tumor_cluster": "greater_than",
          "exhaustion_program_in_singlets_vs_tumor_cluster": "greater_than",
          "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": "greater_than",
          "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": "greater_than",
          "multichain_tcr_rate_in_singlets_vs_tumor_cluster": "greater_than",
          "naive_memory_program_in_singlets_vs_tumor_cluster": "less_than",
          "proliferation_program_in_singlets_vs_tumor_cluster": "greater_than"
        },
        "cd8_states_enriched_in_singlet_fraction": [
          {
            "functional_category": "stem_like_exhausted",
            "state_label": "TCF7_stem_like_Tex"
          },
          {
            "functional_category": "intermediate_exhausted_gzmk",
            "state_label": "GZMKhigh_Tex"
          },
          {
            "functional_category": "terminal_exhausted",
            "state_label": "TOXhigh_Tex"
          }
        ],
        "cd8_states_enriched_in_tumor_cluster_fraction": [
          {
            "functional_category": "naive_memory",
            "state_label": "Tn_Tmem"
          },
          {
            "functional_category": "early_effector_memory",
            "state_label": "early_Tem_GZMK"
          },
          {
            "functional_category": "early_effector_memory",
            "state_label": "Tem_GZMH"
          }
        ],
        "clonotype_expansion_finding": "singlet_modestly_more_expanded_than_cluster"
      },
      "nTurns": 23,
      "checkpointSignals": {
        "clustering": true,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": false,
        "nerveJointAnchor": false,
        "multiomeAssembly": false,
        "depthNormalizedAtac": false,
        "regulonInference": false,
        "cd8SubsetIntegration": true,
        "perPatientEnrichment": true,
        "tcrRichnessMetrics": true,
        "orthologProjection": false,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": true,
        "enrichmentFamilyRanking": true,
        "mousePseudobulkDesign": false,
        "cancerEpithelialDenominator": false,
        "geneLevelDe": false,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "trajectoryFile": "scbench-long/trajectories/10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1__claude-opus-4-6__pi__r1.json",
      "runId": "claude-opus-4-6__pi__10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1__r1",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-opus-4-6-pi",
      "modelName": "claude-opus-4-6",
      "harness": "pi",
      "task": "10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 2,
      "passed": true,
      "score": 15,
      "durationS": 2506.07,
      "cost": 3.09249775,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_singlet_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "TOXhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "LAG3high_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "GZMKhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "intermediate_exhausted_gzmk"
                    ],
                    "canonical": "intermediate_exhausted_gzmk"
                  }
                }
              },
              {
                "key": "TCF7_stem_like_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "stem_like_exhausted"
                    ],
                    "canonical": "stem_like_exhausted"
                  }
                }
              },
              {
                "key": "MKI67_pos_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              },
              {
                "key": "MKI67high_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_tumor_cluster_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "Tn",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "Tn_Tmem",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "early_Tem_GZMK",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              },
              {
                "key": "Tem_GZMH",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              }
            ]
          },
          {
            "name": "clonotype_expansion_direction_data_true",
            "role": "gate",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_substantially_more_expanded_than_cluster",
              "singlet_modestly_more_expanded_than_cluster"
            ]
          },
          {
            "name": "biological_inference_data_true",
            "role": "gate",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_antigen_experienced_cluster_bystander"
            ]
          },
          {
            "name": "singlet_top3_contains_naive_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Tn",
              "Tn_Tmem",
              "early_Tem_GZMK",
              "Tem_GZMH"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "TOXhigh_Tex",
              "LAG3high_Tex",
              "GZMKhigh_Tex",
              "TCF7_stem_like_Tex",
              "MKI67_pos_Tex_Tprol",
              "MKI67high_Tex_Tprol"
            ]
          },
          {
            "name": "singlet_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "TOXhigh_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "TCF7_stem_like_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "MKI67_state_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "Tn_or_Tn_Tmem_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_stemlike_precursor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_reported_canonical_expansion_direction",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_substantially_more_expanded_than_singlet",
              "cluster_modestly_more_expanded_than_singlet"
            ]
          },
          {
            "name": "cf_expansion_lazy_defer",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "expansion_pattern_not_detectable"
            ]
          },
          {
            "name": "cf_reported_canonical_expected_biology",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_antigen_experienced_singlet_bystander"
            ]
          },
          {
            "name": "cf_biological_inference_null_artifact_or_lineage",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "differences_reflect_dissociation_or_sorting_artifact",
              "cluster_and_singlet_capture_same_biology",
              "cluster_and_singlet_capture_distinct_cd8_lineages"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "biological_relations",
            "kind": "dict",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_greater_than",
                  "greater_than"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "less_than",
                  "much_less_than"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than",
                  "approximately_equal"
                ]
              },
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal"
                ]
              },
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal",
                  "greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_exhaustion_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_naive_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_proliferation_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_expansion_richness_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_multichain_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_intermediate_gzmk_overgeneralized",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_melanocytic_conjugate_overcall",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "values": [
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_relations_mass_hedge",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              }
            }
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: PASS\n  + list_match: list_match (answer_field='cd8_states_enriched_in_singlet_fraction'): PASS  (score=3.0)\n  + list_match: list_match (answer_field='cd8_states_enriched_in_tumor_cluster_fraction'): PASS  (score=3.0)\n  + clonotype_expansion_direction_data_true: passed=True, score=1.0\n  + biological_inference_data_true: passed=True, score=1.0\n  + dict_match: dict_match (answer_field='biological_relations'): PASS  (score=7.0)\n  + hard_fail 'singlet_top3_contains_naive_state_prior_pushing': not triggered\n  + hard_fail 'tumor_cluster_top3_contains_exhausted_state_prior_pushing': not triggered\n  + hard_fail 'singlet_top3_contains_decoy_token': not triggered\n  + hard_fail 'tumor_cluster_top3_contains_decoy_token': not triggered\n  + hard_fail 'TOXhigh_Tex_wrong_functional_category': not triggered\n  + hard_fail 'TCF7_stem_like_Tex_wrong_functional_category': not triggered\n  + hard_fail 'MKI67_state_wrong_functional_category': not triggered\n  + hard_fail 'Tn_or_Tn_Tmem_wrong_functional_category': not triggered\n  + hard_fail 'missing_exhausted_or_proliferating_anchor_in_singlet_top3': not triggered\n  + hard_fail 'missing_stemlike_precursor_in_singlet_top3': not triggered\n  + hard_fail 'missing_naive_or_early_memory_anchor_in_tumor_cluster_top3': not triggered\n  + hard_fail 'cf_reported_canonical_expansion_direction': not triggered\n  + hard_fail 'cf_expansion_lazy_defer': not triggered\n  + hard_fail 'cf_reported_canonical_expected_biology': not triggered\n  + hard_fail 'cf_biological_inference_null_artifact_or_lineage': not triggered\n  + hard_fail 'cf_exhaustion_inverted_prior_push': not triggered\n  + hard_fail 'cf_naive_inverted_prior_push': not triggered\n  + hard_fail 'cf_proliferation_inverted_prior_push': not triggered\n  + hard_fail 'cf_expansion_richness_inverted_prior_push': not triggered\n  + hard_fail 'cf_multichain_inverted_prior_push': not triggered\n  + hard_fail 'cf_intermediate_gzmk_overgeneralized': not triggered\n  + hard_fail 'cf_melanocytic_conjugate_overcall': not triggered\n  + hard_fail 'cf_relations_mass_hedge': not triggered",
      "metrics": {
        "hard_fail_triggered": [],
        "pass_rule": "all",
        "scoring_count": 5,
        "scoring_passed": 5,
        "scoring_total_score": 15
      },
      "fieldScores": {
        "biological_inference_data_true": 1,
        "clonotype_expansion_direction_data_true": 1,
        "dict_match": 7,
        "list_match": 3
      },
      "graderChecks": [
        {
          "name": "cd8_states_enriched_in_singlet_fraction",
          "kind": "list_match",
          "passed": true,
          "score": 3
        },
        {
          "name": "cd8_states_enriched_in_tumor_cluster_fraction",
          "kind": "list_match",
          "passed": true,
          "score": 3
        },
        {
          "name": "clonotype_expansion_direction_data_true",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "biological_inference_data_true",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "biological_relations",
          "kind": "list_match",
          "passed": true,
          "score": 7
        },
        {
          "name": "singlet_top3_contains_naive_state_prior_pushing",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "singlet_top3_contains_decoy_token",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "tumor_cluster_top3_contains_decoy_token",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "TOXhigh_Tex_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "TCF7_stem_like_Tex_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "MKI67_state_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "Tn_or_Tn_Tmem_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_stemlike_precursor_in_singlet_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_reported_canonical_expansion_direction",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_expansion_lazy_defer",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_reported_canonical_expected_biology",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_biological_inference_null_artifact_or_lineage",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_exhaustion_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_naive_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_proliferation_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_expansion_richness_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_multichain_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_intermediate_gzmk_overgeneralized",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_melanocytic_conjugate_overcall",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_relations_mass_hedge",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "biological_inference": "singlet_antigen_experienced_cluster_bystander",
        "biological_relations": {
          "clonal_expansion_richness_in_singlets_vs_tumor_cluster": "greater_than",
          "exhaustion_program_in_singlets_vs_tumor_cluster": "greater_than",
          "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": "approximately_equal",
          "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": "greater_than",
          "multichain_tcr_rate_in_singlets_vs_tumor_cluster": "greater_than",
          "naive_memory_program_in_singlets_vs_tumor_cluster": "less_than",
          "proliferation_program_in_singlets_vs_tumor_cluster": "greater_than"
        },
        "cd8_states_enriched_in_singlet_fraction": [
          {
            "functional_category": "terminal_exhausted",
            "state_label": "TOXhigh_Tex"
          },
          {
            "functional_category": "stem_like_exhausted",
            "state_label": "TCF7_stem_like_Tex"
          },
          {
            "functional_category": "proliferating",
            "state_label": "MKI67high_Tex_Tprol"
          }
        ],
        "cd8_states_enriched_in_tumor_cluster_fraction": [
          {
            "functional_category": "naive_memory",
            "state_label": "Tn_Tmem"
          },
          {
            "functional_category": "early_effector_memory",
            "state_label": "early_Tem_GZMK"
          },
          {
            "functional_category": "naive_memory",
            "state_label": "Tn"
          }
        ],
        "clonotype_expansion_finding": "singlet_modestly_more_expanded_than_cluster"
      },
      "nTurns": 22,
      "checkpointSignals": {
        "clustering": true,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": false,
        "nerveJointAnchor": false,
        "multiomeAssembly": false,
        "depthNormalizedAtac": false,
        "regulonInference": false,
        "cd8SubsetIntegration": true,
        "perPatientEnrichment": true,
        "tcrRichnessMetrics": true,
        "orthologProjection": false,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": true,
        "mousePseudobulkDesign": false,
        "cancerEpithelialDenominator": false,
        "geneLevelDe": false,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "trajectoryFile": "scbench-long/trajectories/10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1__claude-opus-4-6__pi__r2.json",
      "runId": "claude-opus-4-6__pi__10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1__r2",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-opus-4-6-pi",
      "modelName": "claude-opus-4-6",
      "harness": "pi",
      "task": "10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": true,
      "score": 15,
      "durationS": 1889.35,
      "cost": 2.64664875,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_singlet_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "TOXhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "LAG3high_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "GZMKhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "intermediate_exhausted_gzmk"
                    ],
                    "canonical": "intermediate_exhausted_gzmk"
                  }
                }
              },
              {
                "key": "TCF7_stem_like_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "stem_like_exhausted"
                    ],
                    "canonical": "stem_like_exhausted"
                  }
                }
              },
              {
                "key": "MKI67_pos_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              },
              {
                "key": "MKI67high_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_tumor_cluster_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "Tn",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "Tn_Tmem",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "early_Tem_GZMK",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              },
              {
                "key": "Tem_GZMH",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              }
            ]
          },
          {
            "name": "clonotype_expansion_direction_data_true",
            "role": "gate",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_substantially_more_expanded_than_cluster",
              "singlet_modestly_more_expanded_than_cluster"
            ]
          },
          {
            "name": "biological_inference_data_true",
            "role": "gate",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_antigen_experienced_cluster_bystander"
            ]
          },
          {
            "name": "singlet_top3_contains_naive_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Tn",
              "Tn_Tmem",
              "early_Tem_GZMK",
              "Tem_GZMH"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "TOXhigh_Tex",
              "LAG3high_Tex",
              "GZMKhigh_Tex",
              "TCF7_stem_like_Tex",
              "MKI67_pos_Tex_Tprol",
              "MKI67high_Tex_Tprol"
            ]
          },
          {
            "name": "singlet_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "TOXhigh_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "TCF7_stem_like_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "MKI67_state_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "Tn_or_Tn_Tmem_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_stemlike_precursor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_reported_canonical_expansion_direction",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_substantially_more_expanded_than_singlet",
              "cluster_modestly_more_expanded_than_singlet"
            ]
          },
          {
            "name": "cf_expansion_lazy_defer",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "expansion_pattern_not_detectable"
            ]
          },
          {
            "name": "cf_reported_canonical_expected_biology",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_antigen_experienced_singlet_bystander"
            ]
          },
          {
            "name": "cf_biological_inference_null_artifact_or_lineage",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "differences_reflect_dissociation_or_sorting_artifact",
              "cluster_and_singlet_capture_same_biology",
              "cluster_and_singlet_capture_distinct_cd8_lineages"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "biological_relations",
            "kind": "dict",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_greater_than",
                  "greater_than"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "less_than",
                  "much_less_than"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than",
                  "approximately_equal"
                ]
              },
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal"
                ]
              },
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal",
                  "greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_exhaustion_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_naive_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_proliferation_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_expansion_richness_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_multichain_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_intermediate_gzmk_overgeneralized",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_melanocytic_conjugate_overcall",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "values": [
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_relations_mass_hedge",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              }
            }
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: PASS\n  + list_match: list_match (answer_field='cd8_states_enriched_in_singlet_fraction'): PASS  (score=3.0)\n  + list_match: list_match (answer_field='cd8_states_enriched_in_tumor_cluster_fraction'): PASS  (score=3.0)\n  + clonotype_expansion_direction_data_true: passed=True, score=1.0\n  + biological_inference_data_true: passed=True, score=1.0\n  + dict_match: dict_match (answer_field='biological_relations'): PASS  (score=7.0)\n  + hard_fail 'singlet_top3_contains_naive_state_prior_pushing': not triggered\n  + hard_fail 'tumor_cluster_top3_contains_exhausted_state_prior_pushing': not triggered\n  + hard_fail 'singlet_top3_contains_decoy_token': not triggered\n  + hard_fail 'tumor_cluster_top3_contains_decoy_token': not triggered\n  + hard_fail 'TOXhigh_Tex_wrong_functional_category': not triggered\n  + hard_fail 'TCF7_stem_like_Tex_wrong_functional_category': not triggered\n  + hard_fail 'MKI67_state_wrong_functional_category': not triggered\n  + hard_fail 'Tn_or_Tn_Tmem_wrong_functional_category': not triggered\n  + hard_fail 'missing_exhausted_or_proliferating_anchor_in_singlet_top3': not triggered\n  + hard_fail 'missing_stemlike_precursor_in_singlet_top3': not triggered\n  + hard_fail 'missing_naive_or_early_memory_anchor_in_tumor_cluster_top3': not triggered\n  + hard_fail 'cf_reported_canonical_expansion_direction': not triggered\n  + hard_fail 'cf_expansion_lazy_defer': not triggered\n  + hard_fail 'cf_reported_canonical_expected_biology': not triggered\n  + hard_fail 'cf_biological_inference_null_artifact_or_lineage': not triggered\n  + hard_fail 'cf_exhaustion_inverted_prior_push': not triggered\n  + hard_fail 'cf_naive_inverted_prior_push': not triggered\n  + hard_fail 'cf_proliferation_inverted_prior_push': not triggered\n  + hard_fail 'cf_expansion_richness_inverted_prior_push': not triggered\n  + hard_fail 'cf_multichain_inverted_prior_push': not triggered\n  + hard_fail 'cf_intermediate_gzmk_overgeneralized': not triggered\n  + hard_fail 'cf_melanocytic_conjugate_overcall': not triggered\n  + hard_fail 'cf_relations_mass_hedge': not triggered",
      "metrics": {
        "hard_fail_triggered": [],
        "pass_rule": "all",
        "scoring_count": 5,
        "scoring_passed": 5,
        "scoring_total_score": 15
      },
      "fieldScores": {
        "biological_inference_data_true": 1,
        "clonotype_expansion_direction_data_true": 1,
        "dict_match": 7,
        "list_match": 3
      },
      "graderChecks": [
        {
          "name": "cd8_states_enriched_in_singlet_fraction",
          "kind": "list_match",
          "passed": true,
          "score": 3
        },
        {
          "name": "cd8_states_enriched_in_tumor_cluster_fraction",
          "kind": "list_match",
          "passed": true,
          "score": 3
        },
        {
          "name": "clonotype_expansion_direction_data_true",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "biological_inference_data_true",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "biological_relations",
          "kind": "list_match",
          "passed": true,
          "score": 7
        },
        {
          "name": "singlet_top3_contains_naive_state_prior_pushing",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "singlet_top3_contains_decoy_token",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "tumor_cluster_top3_contains_decoy_token",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "TOXhigh_Tex_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "TCF7_stem_like_Tex_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "MKI67_state_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "Tn_or_Tn_Tmem_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_stemlike_precursor_in_singlet_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_reported_canonical_expansion_direction",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_expansion_lazy_defer",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_reported_canonical_expected_biology",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_biological_inference_null_artifact_or_lineage",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_exhaustion_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_naive_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_proliferation_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_expansion_richness_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_multichain_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_intermediate_gzmk_overgeneralized",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_melanocytic_conjugate_overcall",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_relations_mass_hedge",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "biological_inference": "singlet_antigen_experienced_cluster_bystander",
        "biological_relations": {
          "clonal_expansion_richness_in_singlets_vs_tumor_cluster": "greater_than",
          "exhaustion_program_in_singlets_vs_tumor_cluster": "greater_than",
          "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": "approximately_equal",
          "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": "greater_than",
          "multichain_tcr_rate_in_singlets_vs_tumor_cluster": "greater_than",
          "naive_memory_program_in_singlets_vs_tumor_cluster": "less_than",
          "proliferation_program_in_singlets_vs_tumor_cluster": "greater_than"
        },
        "cd8_states_enriched_in_singlet_fraction": [
          {
            "functional_category": "stem_like_exhausted",
            "state_label": "TCF7_stem_like_Tex"
          },
          {
            "functional_category": "terminal_exhausted",
            "state_label": "LAG3high_Tex"
          },
          {
            "functional_category": "proliferating",
            "state_label": "MKI67high_Tex_Tprol"
          }
        ],
        "cd8_states_enriched_in_tumor_cluster_fraction": [
          {
            "functional_category": "naive_memory",
            "state_label": "Tn"
          },
          {
            "functional_category": "naive_memory",
            "state_label": "Tn_Tmem"
          },
          {
            "functional_category": "early_effector_memory",
            "state_label": "early_Tem_GZMK"
          }
        ],
        "clonotype_expansion_finding": "singlet_modestly_more_expanded_than_cluster"
      },
      "nTurns": 25,
      "checkpointSignals": {
        "clustering": true,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": false,
        "nerveJointAnchor": false,
        "multiomeAssembly": false,
        "depthNormalizedAtac": false,
        "regulonInference": false,
        "cd8SubsetIntegration": true,
        "perPatientEnrichment": true,
        "tcrRichnessMetrics": true,
        "orthologProjection": false,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": true,
        "mousePseudobulkDesign": false,
        "cancerEpithelialDenominator": false,
        "geneLevelDe": false,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "trajectoryFile": "scbench-long/trajectories/10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1__claude-opus-4-6__pi__r3.json",
      "runId": "claude-opus-4-6__pi__10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1__r3",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-opus-4-7-claude-code",
      "modelName": "claude-opus-4-7",
      "harness": "claude-code",
      "task": "10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 1,
      "passed": false,
      "score": 14,
      "durationS": 2055.92,
      "cost": 6.07067975,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_singlet_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "TOXhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "LAG3high_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "GZMKhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "intermediate_exhausted_gzmk"
                    ],
                    "canonical": "intermediate_exhausted_gzmk"
                  }
                }
              },
              {
                "key": "TCF7_stem_like_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "stem_like_exhausted"
                    ],
                    "canonical": "stem_like_exhausted"
                  }
                }
              },
              {
                "key": "MKI67_pos_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              },
              {
                "key": "MKI67high_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_tumor_cluster_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "Tn",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "Tn_Tmem",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "early_Tem_GZMK",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              },
              {
                "key": "Tem_GZMH",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              }
            ]
          },
          {
            "name": "clonotype_expansion_direction_data_true",
            "role": "gate",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_substantially_more_expanded_than_cluster",
              "singlet_modestly_more_expanded_than_cluster"
            ]
          },
          {
            "name": "biological_inference_data_true",
            "role": "gate",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_antigen_experienced_cluster_bystander"
            ]
          },
          {
            "name": "singlet_top3_contains_naive_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Tn",
              "Tn_Tmem",
              "early_Tem_GZMK",
              "Tem_GZMH"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "TOXhigh_Tex",
              "LAG3high_Tex",
              "GZMKhigh_Tex",
              "TCF7_stem_like_Tex",
              "MKI67_pos_Tex_Tprol",
              "MKI67high_Tex_Tprol"
            ]
          },
          {
            "name": "singlet_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "TOXhigh_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "TCF7_stem_like_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "MKI67_state_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "Tn_or_Tn_Tmem_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_stemlike_precursor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_reported_canonical_expansion_direction",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_substantially_more_expanded_than_singlet",
              "cluster_modestly_more_expanded_than_singlet"
            ]
          },
          {
            "name": "cf_expansion_lazy_defer",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "expansion_pattern_not_detectable"
            ]
          },
          {
            "name": "cf_reported_canonical_expected_biology",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_antigen_experienced_singlet_bystander"
            ]
          },
          {
            "name": "cf_biological_inference_null_artifact_or_lineage",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "differences_reflect_dissociation_or_sorting_artifact",
              "cluster_and_singlet_capture_same_biology",
              "cluster_and_singlet_capture_distinct_cd8_lineages"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "biological_relations",
            "kind": "dict",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_greater_than",
                  "greater_than"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "less_than",
                  "much_less_than"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than",
                  "approximately_equal"
                ]
              },
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal"
                ]
              },
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal",
                  "greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_exhaustion_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_naive_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_proliferation_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_expansion_richness_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_multichain_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_intermediate_gzmk_overgeneralized",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_melanocytic_conjugate_overcall",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "values": [
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_relations_mass_hedge",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              }
            }
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: FAIL\n  + list_match: list_match (answer_field='cd8_states_enriched_in_singlet_fraction'): PASS  (score=3.0)\n  + list_match: list_match (answer_field='cd8_states_enriched_in_tumor_cluster_fraction'): PASS  (score=3.0)\n  + clonotype_expansion_direction_data_true: passed=True, score=1.0\n  + biological_inference_data_true: passed=True, score=1.0\n  x dict_match: dict_match (answer_field='biological_relations'): FAIL  (score=6.0)\n  + hard_fail 'singlet_top3_contains_naive_state_prior_pushing': not triggered\n  + hard_fail 'tumor_cluster_top3_contains_exhausted_state_prior_pushing': not triggered\n  + hard_fail 'singlet_top3_contains_decoy_token': not triggered\n  + hard_fail 'tumor_cluster_top3_contains_decoy_token': not triggered\n  + hard_fail 'TOXhigh_Tex_wrong_functional_category': not triggered\n  + hard_fail 'TCF7_stem_like_Tex_wrong_functional_category': not triggered\n  + hard_fail 'MKI67_state_wrong_functional_category': not triggered\n  + hard_fail 'Tn_or_Tn_Tmem_wrong_functional_category': not triggered\n  + hard_fail 'missing_exhausted_or_proliferating_anchor_in_singlet_top3': not triggered\n  + hard_fail 'missing_stemlike_precursor_in_singlet_top3': not triggered\n  + hard_fail 'missing_naive_or_early_memory_anchor_in_tumor_cluster_top3': not triggered\n  + hard_fail 'cf_reported_canonical_expansion_direction': not triggered\n  + hard_fail 'cf_expansion_lazy_defer': not triggered\n  + hard_fail 'cf_reported_canonical_expected_biology': not triggered\n  + hard_fail 'cf_biological_inference_null_artifact_or_lineage': not triggered\n  + hard_fail 'cf_exhaustion_inverted_prior_push': not triggered\n  + hard_fail 'cf_naive_inverted_prior_push': not triggered\n  + hard_fail 'cf_proliferation_inverted_prior_push': not triggered\n  + hard_fail 'cf_expansion_richness_inverted_prior_push': not triggered\n  + hard_fail 'cf_multichain_inverted_prior_push': not triggered\n  x hard_fail 'cf_intermediate_gzmk_overgeneralized': TRIGGERED\n  + hard_fail 'cf_melanocytic_conjugate_overcall': not triggered\n  + hard_fail 'cf_relations_mass_hedge': not triggered",
      "metrics": {
        "hard_fail_triggered": [
          "cf_intermediate_gzmk_overgeneralized"
        ],
        "pass_rule": "all",
        "scoring_count": 5,
        "scoring_passed": 4,
        "scoring_total_score": 14
      },
      "fieldScores": {
        "biological_inference_data_true": 1,
        "clonotype_expansion_direction_data_true": 1,
        "dict_match": 6,
        "list_match": 3
      },
      "graderChecks": [
        {
          "name": "cd8_states_enriched_in_singlet_fraction",
          "kind": "list_match",
          "passed": true,
          "score": 3
        },
        {
          "name": "cd8_states_enriched_in_tumor_cluster_fraction",
          "kind": "list_match",
          "passed": true,
          "score": 3
        },
        {
          "name": "clonotype_expansion_direction_data_true",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "biological_inference_data_true",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "biological_relations",
          "kind": "list_match",
          "passed": false,
          "score": 6
        },
        {
          "name": "singlet_top3_contains_naive_state_prior_pushing",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "singlet_top3_contains_decoy_token",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "tumor_cluster_top3_contains_decoy_token",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "TOXhigh_Tex_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "TCF7_stem_like_Tex_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "MKI67_state_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "Tn_or_Tn_Tmem_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_stemlike_precursor_in_singlet_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_reported_canonical_expansion_direction",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_expansion_lazy_defer",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_reported_canonical_expected_biology",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_biological_inference_null_artifact_or_lineage",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_exhaustion_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_naive_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_proliferation_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_expansion_richness_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_multichain_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_intermediate_gzmk_overgeneralized",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "cf_melanocytic_conjugate_overcall",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_relations_mass_hedge",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "biological_inference": "singlet_antigen_experienced_cluster_bystander",
        "biological_relations": {
          "clonal_expansion_richness_in_singlets_vs_tumor_cluster": "greater_than",
          "exhaustion_program_in_singlets_vs_tumor_cluster": "much_greater_than",
          "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": "greater_than",
          "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": "approximately_equal",
          "multichain_tcr_rate_in_singlets_vs_tumor_cluster": "greater_than",
          "naive_memory_program_in_singlets_vs_tumor_cluster": "much_less_than",
          "proliferation_program_in_singlets_vs_tumor_cluster": "greater_than"
        },
        "cd8_states_enriched_in_singlet_fraction": [
          {
            "functional_category": "stem_like_exhausted",
            "state_label": "TCF7_stem_like_Tex"
          },
          {
            "functional_category": "intermediate_exhausted_gzmk",
            "state_label": "GZMKhigh_Tex"
          },
          {
            "functional_category": "terminal_exhausted",
            "state_label": "LAG3high_Tex"
          }
        ],
        "cd8_states_enriched_in_tumor_cluster_fraction": [
          {
            "functional_category": "naive_memory",
            "state_label": "Tn"
          },
          {
            "functional_category": "naive_memory",
            "state_label": "Tn_Tmem"
          },
          {
            "functional_category": "early_effector_memory",
            "state_label": "early_Tem_GZMK"
          }
        ],
        "clonotype_expansion_finding": "singlet_modestly_more_expanded_than_cluster"
      },
      "nTurns": 48,
      "checkpointSignals": {
        "clustering": true,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": true,
        "nerveJointAnchor": false,
        "multiomeAssembly": false,
        "depthNormalizedAtac": false,
        "regulonInference": false,
        "cd8SubsetIntegration": true,
        "perPatientEnrichment": true,
        "tcrRichnessMetrics": true,
        "orthologProjection": false,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": true,
        "mousePseudobulkDesign": false,
        "cancerEpithelialDenominator": false,
        "geneLevelDe": false,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "trajectoryFile": "scbench-long/trajectories/10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1__claude-opus-4-7__claude-code__r1.json",
      "runId": "claude-opus-4-7__claude-code__10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1__r1",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-opus-4-7-claude-code",
      "modelName": "claude-opus-4-7",
      "harness": "claude-code",
      "task": "10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 2,
      "passed": false,
      "score": 12,
      "durationS": 3051.89,
      "cost": 8.32972075,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_singlet_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "TOXhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "LAG3high_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "GZMKhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "intermediate_exhausted_gzmk"
                    ],
                    "canonical": "intermediate_exhausted_gzmk"
                  }
                }
              },
              {
                "key": "TCF7_stem_like_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "stem_like_exhausted"
                    ],
                    "canonical": "stem_like_exhausted"
                  }
                }
              },
              {
                "key": "MKI67_pos_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              },
              {
                "key": "MKI67high_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_tumor_cluster_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "Tn",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "Tn_Tmem",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "early_Tem_GZMK",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              },
              {
                "key": "Tem_GZMH",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              }
            ]
          },
          {
            "name": "clonotype_expansion_direction_data_true",
            "role": "gate",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_substantially_more_expanded_than_cluster",
              "singlet_modestly_more_expanded_than_cluster"
            ]
          },
          {
            "name": "biological_inference_data_true",
            "role": "gate",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_antigen_experienced_cluster_bystander"
            ]
          },
          {
            "name": "singlet_top3_contains_naive_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Tn",
              "Tn_Tmem",
              "early_Tem_GZMK",
              "Tem_GZMH"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "TOXhigh_Tex",
              "LAG3high_Tex",
              "GZMKhigh_Tex",
              "TCF7_stem_like_Tex",
              "MKI67_pos_Tex_Tprol",
              "MKI67high_Tex_Tprol"
            ]
          },
          {
            "name": "singlet_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "TOXhigh_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "TCF7_stem_like_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "MKI67_state_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "Tn_or_Tn_Tmem_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_stemlike_precursor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_reported_canonical_expansion_direction",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_substantially_more_expanded_than_singlet",
              "cluster_modestly_more_expanded_than_singlet"
            ]
          },
          {
            "name": "cf_expansion_lazy_defer",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "expansion_pattern_not_detectable"
            ]
          },
          {
            "name": "cf_reported_canonical_expected_biology",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_antigen_experienced_singlet_bystander"
            ]
          },
          {
            "name": "cf_biological_inference_null_artifact_or_lineage",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "differences_reflect_dissociation_or_sorting_artifact",
              "cluster_and_singlet_capture_same_biology",
              "cluster_and_singlet_capture_distinct_cd8_lineages"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "biological_relations",
            "kind": "dict",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_greater_than",
                  "greater_than"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "less_than",
                  "much_less_than"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than",
                  "approximately_equal"
                ]
              },
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal"
                ]
              },
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal",
                  "greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_exhaustion_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_naive_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_proliferation_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_expansion_richness_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_multichain_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_intermediate_gzmk_overgeneralized",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_melanocytic_conjugate_overcall",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "values": [
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_relations_mass_hedge",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              }
            }
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: FAIL\n  + list_match: list_match (answer_field='cd8_states_enriched_in_singlet_fraction'): PASS  (score=3.0)\n  + list_match: list_match (answer_field='cd8_states_enriched_in_tumor_cluster_fraction'): PASS  (score=3.0)\n  + clonotype_expansion_direction_data_true: passed=True, score=1.0\n  + biological_inference_data_true: passed=True, score=1.0\n  x dict_match: dict_match (answer_field='biological_relations'): FAIL  (score=4.0)\n  + hard_fail 'singlet_top3_contains_naive_state_prior_pushing': not triggered\n  + hard_fail 'tumor_cluster_top3_contains_exhausted_state_prior_pushing': not triggered\n  + hard_fail 'singlet_top3_contains_decoy_token': not triggered\n  + hard_fail 'tumor_cluster_top3_contains_decoy_token': not triggered\n  + hard_fail 'TOXhigh_Tex_wrong_functional_category': not triggered\n  + hard_fail 'TCF7_stem_like_Tex_wrong_functional_category': not triggered\n  + hard_fail 'MKI67_state_wrong_functional_category': not triggered\n  + hard_fail 'Tn_or_Tn_Tmem_wrong_functional_category': not triggered\n  + hard_fail 'missing_exhausted_or_proliferating_anchor_in_singlet_top3': not triggered\n  + hard_fail 'missing_stemlike_precursor_in_singlet_top3': not triggered\n  + hard_fail 'missing_naive_or_early_memory_anchor_in_tumor_cluster_top3': not triggered\n  + hard_fail 'cf_reported_canonical_expansion_direction': not triggered\n  + hard_fail 'cf_expansion_lazy_defer': not triggered\n  + hard_fail 'cf_reported_canonical_expected_biology': not triggered\n  + hard_fail 'cf_biological_inference_null_artifact_or_lineage': not triggered\n  + hard_fail 'cf_exhaustion_inverted_prior_push': not triggered\n  + hard_fail 'cf_naive_inverted_prior_push': not triggered\n  + hard_fail 'cf_proliferation_inverted_prior_push': not triggered\n  + hard_fail 'cf_expansion_richness_inverted_prior_push': not triggered\n  + hard_fail 'cf_multichain_inverted_prior_push': not triggered\n  x hard_fail 'cf_intermediate_gzmk_overgeneralized': TRIGGERED\n  + hard_fail 'cf_melanocytic_conjugate_overcall': not triggered\n  + hard_fail 'cf_relations_mass_hedge': not triggered",
      "metrics": {
        "hard_fail_triggered": [
          "cf_intermediate_gzmk_overgeneralized"
        ],
        "pass_rule": "all",
        "scoring_count": 5,
        "scoring_passed": 4,
        "scoring_total_score": 12
      },
      "fieldScores": {
        "biological_inference_data_true": 1,
        "clonotype_expansion_direction_data_true": 1,
        "dict_match": 4,
        "list_match": 3
      },
      "graderChecks": [
        {
          "name": "cd8_states_enriched_in_singlet_fraction",
          "kind": "list_match",
          "passed": true,
          "score": 3
        },
        {
          "name": "cd8_states_enriched_in_tumor_cluster_fraction",
          "kind": "list_match",
          "passed": true,
          "score": 3
        },
        {
          "name": "clonotype_expansion_direction_data_true",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "biological_inference_data_true",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "biological_relations",
          "kind": "list_match",
          "passed": false,
          "score": 4
        },
        {
          "name": "singlet_top3_contains_naive_state_prior_pushing",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "singlet_top3_contains_decoy_token",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "tumor_cluster_top3_contains_decoy_token",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "TOXhigh_Tex_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "TCF7_stem_like_Tex_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "MKI67_state_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "Tn_or_Tn_Tmem_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_stemlike_precursor_in_singlet_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_reported_canonical_expansion_direction",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_expansion_lazy_defer",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_reported_canonical_expected_biology",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_biological_inference_null_artifact_or_lineage",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_exhaustion_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_naive_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_proliferation_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_expansion_richness_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_multichain_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_intermediate_gzmk_overgeneralized",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "cf_melanocytic_conjugate_overcall",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_relations_mass_hedge",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "biological_inference": "singlet_antigen_experienced_cluster_bystander",
        "biological_relations": {
          "clonal_expansion_richness_in_singlets_vs_tumor_cluster": "greater_than",
          "exhaustion_program_in_singlets_vs_tumor_cluster": "approximately_equal",
          "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": "greater_than",
          "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": "greater_than",
          "multichain_tcr_rate_in_singlets_vs_tumor_cluster": "greater_than",
          "naive_memory_program_in_singlets_vs_tumor_cluster": "less_than",
          "proliferation_program_in_singlets_vs_tumor_cluster": "approximately_equal"
        },
        "cd8_states_enriched_in_singlet_fraction": [
          {
            "functional_category": "stem_like_exhausted",
            "state_label": "TCF7_stem_like_Tex"
          },
          {
            "functional_category": "intermediate_exhausted_gzmk",
            "state_label": "GZMKhigh_Tex"
          },
          {
            "functional_category": "terminal_exhausted",
            "state_label": "LAG3high_Tex"
          }
        ],
        "cd8_states_enriched_in_tumor_cluster_fraction": [
          {
            "functional_category": "naive_memory",
            "state_label": "Tn"
          },
          {
            "functional_category": "naive_memory",
            "state_label": "Tn_Tmem"
          },
          {
            "functional_category": "early_effector_memory",
            "state_label": "early_Tem_GZMK"
          }
        ],
        "clonotype_expansion_finding": "singlet_modestly_more_expanded_than_cluster"
      },
      "nTurns": 59,
      "checkpointSignals": {
        "clustering": false,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": false,
        "nerveJointAnchor": false,
        "multiomeAssembly": false,
        "depthNormalizedAtac": false,
        "regulonInference": false,
        "cd8SubsetIntegration": true,
        "perPatientEnrichment": true,
        "tcrRichnessMetrics": true,
        "orthologProjection": false,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": true,
        "mousePseudobulkDesign": false,
        "cancerEpithelialDenominator": false,
        "geneLevelDe": false,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "trajectoryFile": "scbench-long/trajectories/10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1__claude-opus-4-7__claude-code__r2.json",
      "runId": "claude-opus-4-7__claude-code__10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1__r2",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-opus-4-7-claude-code",
      "modelName": "claude-opus-4-7",
      "harness": "claude-code",
      "task": "10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": true,
      "score": 15,
      "durationS": 2356.75,
      "cost": 7.047943999999999,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_singlet_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "TOXhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "LAG3high_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "GZMKhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "intermediate_exhausted_gzmk"
                    ],
                    "canonical": "intermediate_exhausted_gzmk"
                  }
                }
              },
              {
                "key": "TCF7_stem_like_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "stem_like_exhausted"
                    ],
                    "canonical": "stem_like_exhausted"
                  }
                }
              },
              {
                "key": "MKI67_pos_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              },
              {
                "key": "MKI67high_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_tumor_cluster_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "Tn",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "Tn_Tmem",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "early_Tem_GZMK",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              },
              {
                "key": "Tem_GZMH",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              }
            ]
          },
          {
            "name": "clonotype_expansion_direction_data_true",
            "role": "gate",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_substantially_more_expanded_than_cluster",
              "singlet_modestly_more_expanded_than_cluster"
            ]
          },
          {
            "name": "biological_inference_data_true",
            "role": "gate",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_antigen_experienced_cluster_bystander"
            ]
          },
          {
            "name": "singlet_top3_contains_naive_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Tn",
              "Tn_Tmem",
              "early_Tem_GZMK",
              "Tem_GZMH"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "TOXhigh_Tex",
              "LAG3high_Tex",
              "GZMKhigh_Tex",
              "TCF7_stem_like_Tex",
              "MKI67_pos_Tex_Tprol",
              "MKI67high_Tex_Tprol"
            ]
          },
          {
            "name": "singlet_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "TOXhigh_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "TCF7_stem_like_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "MKI67_state_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "Tn_or_Tn_Tmem_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_stemlike_precursor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_reported_canonical_expansion_direction",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_substantially_more_expanded_than_singlet",
              "cluster_modestly_more_expanded_than_singlet"
            ]
          },
          {
            "name": "cf_expansion_lazy_defer",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "expansion_pattern_not_detectable"
            ]
          },
          {
            "name": "cf_reported_canonical_expected_biology",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_antigen_experienced_singlet_bystander"
            ]
          },
          {
            "name": "cf_biological_inference_null_artifact_or_lineage",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "differences_reflect_dissociation_or_sorting_artifact",
              "cluster_and_singlet_capture_same_biology",
              "cluster_and_singlet_capture_distinct_cd8_lineages"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "biological_relations",
            "kind": "dict",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_greater_than",
                  "greater_than"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "less_than",
                  "much_less_than"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than",
                  "approximately_equal"
                ]
              },
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal"
                ]
              },
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal",
                  "greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_exhaustion_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_naive_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_proliferation_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_expansion_richness_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_multichain_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_intermediate_gzmk_overgeneralized",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_melanocytic_conjugate_overcall",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "values": [
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_relations_mass_hedge",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              }
            }
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: PASS\n  + list_match: list_match (answer_field='cd8_states_enriched_in_singlet_fraction'): PASS  (score=3.0)\n  + list_match: list_match (answer_field='cd8_states_enriched_in_tumor_cluster_fraction'): PASS  (score=3.0)\n  + clonotype_expansion_direction_data_true: passed=True, score=1.0\n  + biological_inference_data_true: passed=True, score=1.0\n  + dict_match: dict_match (answer_field='biological_relations'): PASS  (score=7.0)\n  + hard_fail 'singlet_top3_contains_naive_state_prior_pushing': not triggered\n  + hard_fail 'tumor_cluster_top3_contains_exhausted_state_prior_pushing': not triggered\n  + hard_fail 'singlet_top3_contains_decoy_token': not triggered\n  + hard_fail 'tumor_cluster_top3_contains_decoy_token': not triggered\n  + hard_fail 'TOXhigh_Tex_wrong_functional_category': not triggered\n  + hard_fail 'TCF7_stem_like_Tex_wrong_functional_category': not triggered\n  + hard_fail 'MKI67_state_wrong_functional_category': not triggered\n  + hard_fail 'Tn_or_Tn_Tmem_wrong_functional_category': not triggered\n  + hard_fail 'missing_exhausted_or_proliferating_anchor_in_singlet_top3': not triggered\n  + hard_fail 'missing_stemlike_precursor_in_singlet_top3': not triggered\n  + hard_fail 'missing_naive_or_early_memory_anchor_in_tumor_cluster_top3': not triggered\n  + hard_fail 'cf_reported_canonical_expansion_direction': not triggered\n  + hard_fail 'cf_expansion_lazy_defer': not triggered\n  + hard_fail 'cf_reported_canonical_expected_biology': not triggered\n  + hard_fail 'cf_biological_inference_null_artifact_or_lineage': not triggered\n  + hard_fail 'cf_exhaustion_inverted_prior_push': not triggered\n  + hard_fail 'cf_naive_inverted_prior_push': not triggered\n  + hard_fail 'cf_proliferation_inverted_prior_push': not triggered\n  + hard_fail 'cf_expansion_richness_inverted_prior_push': not triggered\n  + hard_fail 'cf_multichain_inverted_prior_push': not triggered\n  + hard_fail 'cf_intermediate_gzmk_overgeneralized': not triggered\n  + hard_fail 'cf_melanocytic_conjugate_overcall': not triggered\n  + hard_fail 'cf_relations_mass_hedge': not triggered",
      "metrics": {
        "hard_fail_triggered": [],
        "pass_rule": "all",
        "scoring_count": 5,
        "scoring_passed": 5,
        "scoring_total_score": 15
      },
      "fieldScores": {
        "biological_inference_data_true": 1,
        "clonotype_expansion_direction_data_true": 1,
        "dict_match": 7,
        "list_match": 3
      },
      "graderChecks": [
        {
          "name": "cd8_states_enriched_in_singlet_fraction",
          "kind": "list_match",
          "passed": true,
          "score": 3
        },
        {
          "name": "cd8_states_enriched_in_tumor_cluster_fraction",
          "kind": "list_match",
          "passed": true,
          "score": 3
        },
        {
          "name": "clonotype_expansion_direction_data_true",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "biological_inference_data_true",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "biological_relations",
          "kind": "list_match",
          "passed": true,
          "score": 7
        },
        {
          "name": "singlet_top3_contains_naive_state_prior_pushing",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "singlet_top3_contains_decoy_token",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "tumor_cluster_top3_contains_decoy_token",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "TOXhigh_Tex_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "TCF7_stem_like_Tex_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "MKI67_state_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "Tn_or_Tn_Tmem_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_stemlike_precursor_in_singlet_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_reported_canonical_expansion_direction",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_expansion_lazy_defer",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_reported_canonical_expected_biology",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_biological_inference_null_artifact_or_lineage",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_exhaustion_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_naive_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_proliferation_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_expansion_richness_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_multichain_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_intermediate_gzmk_overgeneralized",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_melanocytic_conjugate_overcall",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_relations_mass_hedge",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "biological_inference": "singlet_antigen_experienced_cluster_bystander",
        "biological_relations": {
          "clonal_expansion_richness_in_singlets_vs_tumor_cluster": "greater_than",
          "exhaustion_program_in_singlets_vs_tumor_cluster": "greater_than",
          "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": "less_than",
          "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": "approximately_equal",
          "multichain_tcr_rate_in_singlets_vs_tumor_cluster": "greater_than",
          "naive_memory_program_in_singlets_vs_tumor_cluster": "less_than",
          "proliferation_program_in_singlets_vs_tumor_cluster": "greater_than"
        },
        "cd8_states_enriched_in_singlet_fraction": [
          {
            "functional_category": "stem_like_exhausted",
            "state_label": "TCF7_stem_like_Tex"
          },
          {
            "functional_category": "terminal_exhausted",
            "state_label": "TOXhigh_Tex"
          },
          {
            "functional_category": "proliferating",
            "state_label": "MKI67_pos_Tex_Tprol"
          }
        ],
        "cd8_states_enriched_in_tumor_cluster_fraction": [
          {
            "functional_category": "naive_memory",
            "state_label": "Tn"
          },
          {
            "functional_category": "naive_memory",
            "state_label": "Tn_Tmem"
          },
          {
            "functional_category": "early_effector_memory",
            "state_label": "early_Tem_GZMK"
          }
        ],
        "clonotype_expansion_finding": "singlet_substantially_more_expanded_than_cluster"
      },
      "nTurns": 71,
      "checkpointSignals": {
        "clustering": true,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": false,
        "nerveJointAnchor": false,
        "multiomeAssembly": false,
        "depthNormalizedAtac": true,
        "regulonInference": false,
        "cd8SubsetIntegration": true,
        "perPatientEnrichment": true,
        "tcrRichnessMetrics": true,
        "orthologProjection": false,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": false,
        "mousePseudobulkDesign": false,
        "cancerEpithelialDenominator": false,
        "geneLevelDe": false,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "trajectoryFile": "scbench-long/trajectories/10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1__claude-opus-4-7__claude-code__r3.json",
      "runId": "claude-opus-4-7__claude-code__10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1__r3",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-opus-4-7-pi",
      "modelName": "claude-opus-4-7",
      "harness": "pi",
      "task": "10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 1,
      "passed": true,
      "score": 15,
      "durationS": 4266.17,
      "cost": null,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_singlet_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "TOXhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "LAG3high_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "GZMKhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "intermediate_exhausted_gzmk"
                    ],
                    "canonical": "intermediate_exhausted_gzmk"
                  }
                }
              },
              {
                "key": "TCF7_stem_like_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "stem_like_exhausted"
                    ],
                    "canonical": "stem_like_exhausted"
                  }
                }
              },
              {
                "key": "MKI67_pos_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              },
              {
                "key": "MKI67high_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_tumor_cluster_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "Tn",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "Tn_Tmem",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "early_Tem_GZMK",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              },
              {
                "key": "Tem_GZMH",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              }
            ]
          },
          {
            "name": "clonotype_expansion_direction_data_true",
            "role": "gate",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_substantially_more_expanded_than_cluster",
              "singlet_modestly_more_expanded_than_cluster"
            ]
          },
          {
            "name": "biological_inference_data_true",
            "role": "gate",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_antigen_experienced_cluster_bystander"
            ]
          },
          {
            "name": "singlet_top3_contains_naive_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Tn",
              "Tn_Tmem",
              "early_Tem_GZMK",
              "Tem_GZMH"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "TOXhigh_Tex",
              "LAG3high_Tex",
              "GZMKhigh_Tex",
              "TCF7_stem_like_Tex",
              "MKI67_pos_Tex_Tprol",
              "MKI67high_Tex_Tprol"
            ]
          },
          {
            "name": "singlet_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "TOXhigh_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "TCF7_stem_like_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "MKI67_state_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "Tn_or_Tn_Tmem_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_stemlike_precursor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_reported_canonical_expansion_direction",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_substantially_more_expanded_than_singlet",
              "cluster_modestly_more_expanded_than_singlet"
            ]
          },
          {
            "name": "cf_expansion_lazy_defer",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "expansion_pattern_not_detectable"
            ]
          },
          {
            "name": "cf_reported_canonical_expected_biology",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_antigen_experienced_singlet_bystander"
            ]
          },
          {
            "name": "cf_biological_inference_null_artifact_or_lineage",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "differences_reflect_dissociation_or_sorting_artifact",
              "cluster_and_singlet_capture_same_biology",
              "cluster_and_singlet_capture_distinct_cd8_lineages"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "biological_relations",
            "kind": "dict",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_greater_than",
                  "greater_than"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "less_than",
                  "much_less_than"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than",
                  "approximately_equal"
                ]
              },
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal"
                ]
              },
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal",
                  "greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_exhaustion_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_naive_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_proliferation_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_expansion_richness_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_multichain_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_intermediate_gzmk_overgeneralized",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_melanocytic_conjugate_overcall",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "values": [
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_relations_mass_hedge",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              }
            }
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: PASS\n  + list_match: list_match (answer_field='cd8_states_enriched_in_singlet_fraction'): PASS  (score=3.0)\n  + list_match: list_match (answer_field='cd8_states_enriched_in_tumor_cluster_fraction'): PASS  (score=3.0)\n  + clonotype_expansion_direction_data_true: passed=True, score=1.0\n  + biological_inference_data_true: passed=True, score=1.0\n  + dict_match: dict_match (answer_field='biological_relations'): PASS  (score=7.0)\n  + hard_fail 'singlet_top3_contains_naive_state_prior_pushing': not triggered\n  + hard_fail 'tumor_cluster_top3_contains_exhausted_state_prior_pushing': not triggered\n  + hard_fail 'singlet_top3_contains_decoy_token': not triggered\n  + hard_fail 'tumor_cluster_top3_contains_decoy_token': not triggered\n  + hard_fail 'TOXhigh_Tex_wrong_functional_category': not triggered\n  + hard_fail 'TCF7_stem_like_Tex_wrong_functional_category': not triggered\n  + hard_fail 'MKI67_state_wrong_functional_category': not triggered\n  + hard_fail 'Tn_or_Tn_Tmem_wrong_functional_category': not triggered\n  + hard_fail 'missing_exhausted_or_proliferating_anchor_in_singlet_top3': not triggered\n  + hard_fail 'missing_stemlike_precursor_in_singlet_top3': not triggered\n  + hard_fail 'missing_naive_or_early_memory_anchor_in_tumor_cluster_top3': not triggered\n  + hard_fail 'cf_reported_canonical_expansion_direction': not triggered\n  + hard_fail 'cf_expansion_lazy_defer': not triggered\n  + hard_fail 'cf_reported_canonical_expected_biology': not triggered\n  + hard_fail 'cf_biological_inference_null_artifact_or_lineage': not triggered\n  + hard_fail 'cf_exhaustion_inverted_prior_push': not triggered\n  + hard_fail 'cf_naive_inverted_prior_push': not triggered\n  + hard_fail 'cf_proliferation_inverted_prior_push': not triggered\n  + hard_fail 'cf_expansion_richness_inverted_prior_push': not triggered\n  + hard_fail 'cf_multichain_inverted_prior_push': not triggered\n  + hard_fail 'cf_intermediate_gzmk_overgeneralized': not triggered\n  + hard_fail 'cf_melanocytic_conjugate_overcall': not triggered\n  + hard_fail 'cf_relations_mass_hedge': not triggered",
      "metrics": {
        "hard_fail_triggered": [],
        "pass_rule": "all",
        "scoring_count": 5,
        "scoring_passed": 5,
        "scoring_total_score": 15
      },
      "fieldScores": {
        "biological_inference_data_true": 1,
        "clonotype_expansion_direction_data_true": 1,
        "dict_match": 7,
        "list_match": 3
      },
      "graderChecks": [
        {
          "name": "cd8_states_enriched_in_singlet_fraction",
          "kind": "list_match",
          "passed": true,
          "score": 3
        },
        {
          "name": "cd8_states_enriched_in_tumor_cluster_fraction",
          "kind": "list_match",
          "passed": true,
          "score": 3
        },
        {
          "name": "clonotype_expansion_direction_data_true",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "biological_inference_data_true",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "biological_relations",
          "kind": "list_match",
          "passed": true,
          "score": 7
        },
        {
          "name": "singlet_top3_contains_naive_state_prior_pushing",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "singlet_top3_contains_decoy_token",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "tumor_cluster_top3_contains_decoy_token",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "TOXhigh_Tex_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "TCF7_stem_like_Tex_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "MKI67_state_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "Tn_or_Tn_Tmem_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_stemlike_precursor_in_singlet_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_reported_canonical_expansion_direction",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_expansion_lazy_defer",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_reported_canonical_expected_biology",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_biological_inference_null_artifact_or_lineage",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_exhaustion_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_naive_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_proliferation_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_expansion_richness_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_multichain_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_intermediate_gzmk_overgeneralized",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_melanocytic_conjugate_overcall",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_relations_mass_hedge",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "biological_inference": "singlet_antigen_experienced_cluster_bystander",
        "biological_relations": {
          "clonal_expansion_richness_in_singlets_vs_tumor_cluster": "greater_than",
          "exhaustion_program_in_singlets_vs_tumor_cluster": "greater_than",
          "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": "approximately_equal",
          "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": "approximately_equal",
          "multichain_tcr_rate_in_singlets_vs_tumor_cluster": "greater_than",
          "naive_memory_program_in_singlets_vs_tumor_cluster": "less_than",
          "proliferation_program_in_singlets_vs_tumor_cluster": "greater_than"
        },
        "cd8_states_enriched_in_singlet_fraction": [
          {
            "functional_category": "stem_like_exhausted",
            "state_label": "TCF7_stem_like_Tex"
          },
          {
            "functional_category": "terminal_exhausted",
            "state_label": "TOXhigh_Tex"
          },
          {
            "functional_category": "proliferating",
            "state_label": "MKI67high_Tex_Tprol"
          }
        ],
        "cd8_states_enriched_in_tumor_cluster_fraction": [
          {
            "functional_category": "naive_memory",
            "state_label": "Tn_Tmem"
          },
          {
            "functional_category": "early_effector_memory",
            "state_label": "early_Tem_GZMK"
          },
          {
            "functional_category": "early_effector_memory",
            "state_label": "Tem_GZMH"
          }
        ],
        "clonotype_expansion_finding": "singlet_modestly_more_expanded_than_cluster"
      },
      "nTurns": 58,
      "checkpointSignals": {
        "clustering": true,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": false,
        "nerveJointAnchor": false,
        "multiomeAssembly": false,
        "depthNormalizedAtac": false,
        "regulonInference": false,
        "cd8SubsetIntegration": true,
        "perPatientEnrichment": true,
        "tcrRichnessMetrics": true,
        "orthologProjection": false,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": true,
        "mousePseudobulkDesign": false,
        "cancerEpithelialDenominator": false,
        "geneLevelDe": false,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "trajectoryFile": "scbench-long/trajectories/10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1__claude-opus-4-7__pi__r1.json",
      "runId": "claude-opus-4-7__pi__10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1__r1",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-opus-4-7-pi",
      "modelName": "claude-opus-4-7",
      "harness": "pi",
      "task": "10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 2,
      "passed": false,
      "score": 14,
      "durationS": 2796.34,
      "cost": null,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_singlet_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "TOXhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "LAG3high_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "GZMKhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "intermediate_exhausted_gzmk"
                    ],
                    "canonical": "intermediate_exhausted_gzmk"
                  }
                }
              },
              {
                "key": "TCF7_stem_like_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "stem_like_exhausted"
                    ],
                    "canonical": "stem_like_exhausted"
                  }
                }
              },
              {
                "key": "MKI67_pos_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              },
              {
                "key": "MKI67high_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_tumor_cluster_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "Tn",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "Tn_Tmem",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "early_Tem_GZMK",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              },
              {
                "key": "Tem_GZMH",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              }
            ]
          },
          {
            "name": "clonotype_expansion_direction_data_true",
            "role": "gate",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_substantially_more_expanded_than_cluster",
              "singlet_modestly_more_expanded_than_cluster"
            ]
          },
          {
            "name": "biological_inference_data_true",
            "role": "gate",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_antigen_experienced_cluster_bystander"
            ]
          },
          {
            "name": "singlet_top3_contains_naive_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Tn",
              "Tn_Tmem",
              "early_Tem_GZMK",
              "Tem_GZMH"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "TOXhigh_Tex",
              "LAG3high_Tex",
              "GZMKhigh_Tex",
              "TCF7_stem_like_Tex",
              "MKI67_pos_Tex_Tprol",
              "MKI67high_Tex_Tprol"
            ]
          },
          {
            "name": "singlet_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "TOXhigh_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "TCF7_stem_like_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "MKI67_state_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "Tn_or_Tn_Tmem_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_stemlike_precursor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_reported_canonical_expansion_direction",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_substantially_more_expanded_than_singlet",
              "cluster_modestly_more_expanded_than_singlet"
            ]
          },
          {
            "name": "cf_expansion_lazy_defer",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "expansion_pattern_not_detectable"
            ]
          },
          {
            "name": "cf_reported_canonical_expected_biology",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_antigen_experienced_singlet_bystander"
            ]
          },
          {
            "name": "cf_biological_inference_null_artifact_or_lineage",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "differences_reflect_dissociation_or_sorting_artifact",
              "cluster_and_singlet_capture_same_biology",
              "cluster_and_singlet_capture_distinct_cd8_lineages"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "biological_relations",
            "kind": "dict",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_greater_than",
                  "greater_than"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "less_than",
                  "much_less_than"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than",
                  "approximately_equal"
                ]
              },
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal"
                ]
              },
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal",
                  "greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_exhaustion_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_naive_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_proliferation_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_expansion_richness_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_multichain_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_intermediate_gzmk_overgeneralized",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_melanocytic_conjugate_overcall",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "values": [
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_relations_mass_hedge",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              }
            }
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: FAIL\n  + list_match: list_match (answer_field='cd8_states_enriched_in_singlet_fraction'): PASS  (score=3.0)\n  + list_match: list_match (answer_field='cd8_states_enriched_in_tumor_cluster_fraction'): PASS  (score=3.0)\n  + clonotype_expansion_direction_data_true: passed=True, score=1.0\n  + biological_inference_data_true: passed=True, score=1.0\n  x dict_match: dict_match (answer_field='biological_relations'): FAIL  (score=6.0)\n  + hard_fail 'singlet_top3_contains_naive_state_prior_pushing': not triggered\n  + hard_fail 'tumor_cluster_top3_contains_exhausted_state_prior_pushing': not triggered\n  + hard_fail 'singlet_top3_contains_decoy_token': not triggered\n  + hard_fail 'tumor_cluster_top3_contains_decoy_token': not triggered\n  + hard_fail 'TOXhigh_Tex_wrong_functional_category': not triggered\n  + hard_fail 'TCF7_stem_like_Tex_wrong_functional_category': not triggered\n  + hard_fail 'MKI67_state_wrong_functional_category': not triggered\n  + hard_fail 'Tn_or_Tn_Tmem_wrong_functional_category': not triggered\n  + hard_fail 'missing_exhausted_or_proliferating_anchor_in_singlet_top3': not triggered\n  + hard_fail 'missing_stemlike_precursor_in_singlet_top3': not triggered\n  + hard_fail 'missing_naive_or_early_memory_anchor_in_tumor_cluster_top3': not triggered\n  + hard_fail 'cf_reported_canonical_expansion_direction': not triggered\n  + hard_fail 'cf_expansion_lazy_defer': not triggered\n  + hard_fail 'cf_reported_canonical_expected_biology': not triggered\n  + hard_fail 'cf_biological_inference_null_artifact_or_lineage': not triggered\n  + hard_fail 'cf_exhaustion_inverted_prior_push': not triggered\n  + hard_fail 'cf_naive_inverted_prior_push': not triggered\n  + hard_fail 'cf_proliferation_inverted_prior_push': not triggered\n  + hard_fail 'cf_expansion_richness_inverted_prior_push': not triggered\n  + hard_fail 'cf_multichain_inverted_prior_push': not triggered\n  x hard_fail 'cf_intermediate_gzmk_overgeneralized': TRIGGERED\n  + hard_fail 'cf_melanocytic_conjugate_overcall': not triggered\n  + hard_fail 'cf_relations_mass_hedge': not triggered",
      "metrics": {
        "hard_fail_triggered": [
          "cf_intermediate_gzmk_overgeneralized"
        ],
        "pass_rule": "all",
        "scoring_count": 5,
        "scoring_passed": 4,
        "scoring_total_score": 14
      },
      "fieldScores": {
        "biological_inference_data_true": 1,
        "clonotype_expansion_direction_data_true": 1,
        "dict_match": 6,
        "list_match": 3
      },
      "graderChecks": [
        {
          "name": "cd8_states_enriched_in_singlet_fraction",
          "kind": "list_match",
          "passed": true,
          "score": 3
        },
        {
          "name": "cd8_states_enriched_in_tumor_cluster_fraction",
          "kind": "list_match",
          "passed": true,
          "score": 3
        },
        {
          "name": "clonotype_expansion_direction_data_true",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "biological_inference_data_true",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "biological_relations",
          "kind": "list_match",
          "passed": false,
          "score": 6
        },
        {
          "name": "singlet_top3_contains_naive_state_prior_pushing",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "singlet_top3_contains_decoy_token",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "tumor_cluster_top3_contains_decoy_token",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "TOXhigh_Tex_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "TCF7_stem_like_Tex_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "MKI67_state_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "Tn_or_Tn_Tmem_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_stemlike_precursor_in_singlet_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_reported_canonical_expansion_direction",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_expansion_lazy_defer",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_reported_canonical_expected_biology",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_biological_inference_null_artifact_or_lineage",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_exhaustion_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_naive_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_proliferation_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_expansion_richness_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_multichain_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_intermediate_gzmk_overgeneralized",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "cf_melanocytic_conjugate_overcall",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_relations_mass_hedge",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "biological_inference": "singlet_antigen_experienced_cluster_bystander",
        "biological_relations": {
          "clonal_expansion_richness_in_singlets_vs_tumor_cluster": "greater_than",
          "exhaustion_program_in_singlets_vs_tumor_cluster": "greater_than",
          "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": "greater_than",
          "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": "approximately_equal",
          "multichain_tcr_rate_in_singlets_vs_tumor_cluster": "greater_than",
          "naive_memory_program_in_singlets_vs_tumor_cluster": "less_than",
          "proliferation_program_in_singlets_vs_tumor_cluster": "greater_than"
        },
        "cd8_states_enriched_in_singlet_fraction": [
          {
            "functional_category": "stem_like_exhausted",
            "state_label": "TCF7_stem_like_Tex"
          },
          {
            "functional_category": "intermediate_exhausted_gzmk",
            "state_label": "GZMKhigh_Tex"
          },
          {
            "functional_category": "terminal_exhausted",
            "state_label": "TOXhigh_Tex"
          }
        ],
        "cd8_states_enriched_in_tumor_cluster_fraction": [
          {
            "functional_category": "naive_memory",
            "state_label": "Tn"
          },
          {
            "functional_category": "early_effector_memory",
            "state_label": "early_Tem_GZMK"
          },
          {
            "functional_category": "early_effector_memory",
            "state_label": "Tem_GZMH"
          }
        ],
        "clonotype_expansion_finding": "singlet_modestly_more_expanded_than_cluster"
      },
      "nTurns": 40,
      "checkpointSignals": {
        "clustering": true,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": false,
        "nerveJointAnchor": false,
        "multiomeAssembly": false,
        "depthNormalizedAtac": false,
        "regulonInference": false,
        "cd8SubsetIntegration": true,
        "perPatientEnrichment": true,
        "tcrRichnessMetrics": true,
        "orthologProjection": false,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": true,
        "mousePseudobulkDesign": false,
        "cancerEpithelialDenominator": false,
        "geneLevelDe": false,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "trajectoryFile": "scbench-long/trajectories/10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1__claude-opus-4-7__pi__r2.json",
      "runId": "claude-opus-4-7__pi__10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1__r2",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-opus-4-7-pi",
      "modelName": "claude-opus-4-7",
      "harness": "pi",
      "task": "10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": true,
      "score": 15,
      "durationS": 2840.03,
      "cost": null,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_singlet_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "TOXhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "LAG3high_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "GZMKhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "intermediate_exhausted_gzmk"
                    ],
                    "canonical": "intermediate_exhausted_gzmk"
                  }
                }
              },
              {
                "key": "TCF7_stem_like_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "stem_like_exhausted"
                    ],
                    "canonical": "stem_like_exhausted"
                  }
                }
              },
              {
                "key": "MKI67_pos_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              },
              {
                "key": "MKI67high_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_tumor_cluster_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "Tn",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "Tn_Tmem",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "early_Tem_GZMK",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              },
              {
                "key": "Tem_GZMH",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              }
            ]
          },
          {
            "name": "clonotype_expansion_direction_data_true",
            "role": "gate",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_substantially_more_expanded_than_cluster",
              "singlet_modestly_more_expanded_than_cluster"
            ]
          },
          {
            "name": "biological_inference_data_true",
            "role": "gate",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_antigen_experienced_cluster_bystander"
            ]
          },
          {
            "name": "singlet_top3_contains_naive_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Tn",
              "Tn_Tmem",
              "early_Tem_GZMK",
              "Tem_GZMH"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "TOXhigh_Tex",
              "LAG3high_Tex",
              "GZMKhigh_Tex",
              "TCF7_stem_like_Tex",
              "MKI67_pos_Tex_Tprol",
              "MKI67high_Tex_Tprol"
            ]
          },
          {
            "name": "singlet_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "TOXhigh_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "TCF7_stem_like_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "MKI67_state_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "Tn_or_Tn_Tmem_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_stemlike_precursor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_reported_canonical_expansion_direction",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_substantially_more_expanded_than_singlet",
              "cluster_modestly_more_expanded_than_singlet"
            ]
          },
          {
            "name": "cf_expansion_lazy_defer",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "expansion_pattern_not_detectable"
            ]
          },
          {
            "name": "cf_reported_canonical_expected_biology",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_antigen_experienced_singlet_bystander"
            ]
          },
          {
            "name": "cf_biological_inference_null_artifact_or_lineage",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "differences_reflect_dissociation_or_sorting_artifact",
              "cluster_and_singlet_capture_same_biology",
              "cluster_and_singlet_capture_distinct_cd8_lineages"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "biological_relations",
            "kind": "dict",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_greater_than",
                  "greater_than"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "less_than",
                  "much_less_than"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than",
                  "approximately_equal"
                ]
              },
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal"
                ]
              },
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal",
                  "greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_exhaustion_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_naive_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_proliferation_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_expansion_richness_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_multichain_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_intermediate_gzmk_overgeneralized",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_melanocytic_conjugate_overcall",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "values": [
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_relations_mass_hedge",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              }
            }
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: PASS\n  + list_match: list_match (answer_field='cd8_states_enriched_in_singlet_fraction'): PASS  (score=3.0)\n  + list_match: list_match (answer_field='cd8_states_enriched_in_tumor_cluster_fraction'): PASS  (score=3.0)\n  + clonotype_expansion_direction_data_true: passed=True, score=1.0\n  + biological_inference_data_true: passed=True, score=1.0\n  + dict_match: dict_match (answer_field='biological_relations'): PASS  (score=7.0)\n  + hard_fail 'singlet_top3_contains_naive_state_prior_pushing': not triggered\n  + hard_fail 'tumor_cluster_top3_contains_exhausted_state_prior_pushing': not triggered\n  + hard_fail 'singlet_top3_contains_decoy_token': not triggered\n  + hard_fail 'tumor_cluster_top3_contains_decoy_token': not triggered\n  + hard_fail 'TOXhigh_Tex_wrong_functional_category': not triggered\n  + hard_fail 'TCF7_stem_like_Tex_wrong_functional_category': not triggered\n  + hard_fail 'MKI67_state_wrong_functional_category': not triggered\n  + hard_fail 'Tn_or_Tn_Tmem_wrong_functional_category': not triggered\n  + hard_fail 'missing_exhausted_or_proliferating_anchor_in_singlet_top3': not triggered\n  + hard_fail 'missing_stemlike_precursor_in_singlet_top3': not triggered\n  + hard_fail 'missing_naive_or_early_memory_anchor_in_tumor_cluster_top3': not triggered\n  + hard_fail 'cf_reported_canonical_expansion_direction': not triggered\n  + hard_fail 'cf_expansion_lazy_defer': not triggered\n  + hard_fail 'cf_reported_canonical_expected_biology': not triggered\n  + hard_fail 'cf_biological_inference_null_artifact_or_lineage': not triggered\n  + hard_fail 'cf_exhaustion_inverted_prior_push': not triggered\n  + hard_fail 'cf_naive_inverted_prior_push': not triggered\n  + hard_fail 'cf_proliferation_inverted_prior_push': not triggered\n  + hard_fail 'cf_expansion_richness_inverted_prior_push': not triggered\n  + hard_fail 'cf_multichain_inverted_prior_push': not triggered\n  + hard_fail 'cf_intermediate_gzmk_overgeneralized': not triggered\n  + hard_fail 'cf_melanocytic_conjugate_overcall': not triggered\n  + hard_fail 'cf_relations_mass_hedge': not triggered",
      "metrics": {
        "hard_fail_triggered": [],
        "pass_rule": "all",
        "scoring_count": 5,
        "scoring_passed": 5,
        "scoring_total_score": 15
      },
      "fieldScores": {
        "biological_inference_data_true": 1,
        "clonotype_expansion_direction_data_true": 1,
        "dict_match": 7,
        "list_match": 3
      },
      "graderChecks": [
        {
          "name": "cd8_states_enriched_in_singlet_fraction",
          "kind": "list_match",
          "passed": true,
          "score": 3
        },
        {
          "name": "cd8_states_enriched_in_tumor_cluster_fraction",
          "kind": "list_match",
          "passed": true,
          "score": 3
        },
        {
          "name": "clonotype_expansion_direction_data_true",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "biological_inference_data_true",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "biological_relations",
          "kind": "list_match",
          "passed": true,
          "score": 7
        },
        {
          "name": "singlet_top3_contains_naive_state_prior_pushing",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "singlet_top3_contains_decoy_token",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "tumor_cluster_top3_contains_decoy_token",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "TOXhigh_Tex_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "TCF7_stem_like_Tex_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "MKI67_state_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "Tn_or_Tn_Tmem_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_stemlike_precursor_in_singlet_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_reported_canonical_expansion_direction",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_expansion_lazy_defer",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_reported_canonical_expected_biology",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_biological_inference_null_artifact_or_lineage",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_exhaustion_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_naive_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_proliferation_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_expansion_richness_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_multichain_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_intermediate_gzmk_overgeneralized",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_melanocytic_conjugate_overcall",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_relations_mass_hedge",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "biological_inference": "singlet_antigen_experienced_cluster_bystander",
        "biological_relations": {
          "clonal_expansion_richness_in_singlets_vs_tumor_cluster": "greater_than",
          "exhaustion_program_in_singlets_vs_tumor_cluster": "greater_than",
          "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": "approximately_equal",
          "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": "greater_than",
          "multichain_tcr_rate_in_singlets_vs_tumor_cluster": "greater_than",
          "naive_memory_program_in_singlets_vs_tumor_cluster": "less_than",
          "proliferation_program_in_singlets_vs_tumor_cluster": "greater_than"
        },
        "cd8_states_enriched_in_singlet_fraction": [
          {
            "functional_category": "stem_like_exhausted",
            "state_label": "TCF7_stem_like_Tex"
          },
          {
            "functional_category": "intermediate_exhausted_gzmk",
            "state_label": "GZMKhigh_Tex"
          },
          {
            "functional_category": "terminal_exhausted",
            "state_label": "TOXhigh_Tex"
          }
        ],
        "cd8_states_enriched_in_tumor_cluster_fraction": [
          {
            "functional_category": "naive_memory",
            "state_label": "Tn"
          },
          {
            "functional_category": "naive_memory",
            "state_label": "Tn_Tmem"
          },
          {
            "functional_category": "early_effector_memory",
            "state_label": "early_Tem_GZMK"
          }
        ],
        "clonotype_expansion_finding": "singlet_modestly_more_expanded_than_cluster"
      },
      "nTurns": 42,
      "checkpointSignals": {
        "clustering": true,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": false,
        "nerveJointAnchor": false,
        "multiomeAssembly": false,
        "depthNormalizedAtac": false,
        "regulonInference": false,
        "cd8SubsetIntegration": true,
        "perPatientEnrichment": true,
        "tcrRichnessMetrics": true,
        "orthologProjection": false,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": true,
        "mousePseudobulkDesign": false,
        "cancerEpithelialDenominator": false,
        "geneLevelDe": false,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "trajectoryFile": "scbench-long/trajectories/10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1__claude-opus-4-7__pi__r3.json",
      "runId": "claude-opus-4-7__pi__10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1__r3",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-opus-4-8-claude-code",
      "modelName": "claude-opus-4-8",
      "harness": "claude-code",
      "task": "10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 1,
      "passed": false,
      "score": 14,
      "durationS": 1821.67,
      "cost": 5.031114500000001,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_singlet_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "TOXhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "LAG3high_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "GZMKhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "intermediate_exhausted_gzmk"
                    ],
                    "canonical": "intermediate_exhausted_gzmk"
                  }
                }
              },
              {
                "key": "TCF7_stem_like_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "stem_like_exhausted"
                    ],
                    "canonical": "stem_like_exhausted"
                  }
                }
              },
              {
                "key": "MKI67_pos_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              },
              {
                "key": "MKI67high_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_tumor_cluster_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "Tn",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "Tn_Tmem",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "early_Tem_GZMK",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              },
              {
                "key": "Tem_GZMH",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              }
            ]
          },
          {
            "name": "clonotype_expansion_direction_data_true",
            "role": "gate",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_substantially_more_expanded_than_cluster",
              "singlet_modestly_more_expanded_than_cluster"
            ]
          },
          {
            "name": "biological_inference_data_true",
            "role": "gate",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_antigen_experienced_cluster_bystander"
            ]
          },
          {
            "name": "singlet_top3_contains_naive_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Tn",
              "Tn_Tmem",
              "early_Tem_GZMK",
              "Tem_GZMH"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "TOXhigh_Tex",
              "LAG3high_Tex",
              "GZMKhigh_Tex",
              "TCF7_stem_like_Tex",
              "MKI67_pos_Tex_Tprol",
              "MKI67high_Tex_Tprol"
            ]
          },
          {
            "name": "singlet_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "TOXhigh_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "TCF7_stem_like_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "MKI67_state_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "Tn_or_Tn_Tmem_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_stemlike_precursor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_reported_canonical_expansion_direction",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_substantially_more_expanded_than_singlet",
              "cluster_modestly_more_expanded_than_singlet"
            ]
          },
          {
            "name": "cf_expansion_lazy_defer",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "expansion_pattern_not_detectable"
            ]
          },
          {
            "name": "cf_reported_canonical_expected_biology",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_antigen_experienced_singlet_bystander"
            ]
          },
          {
            "name": "cf_biological_inference_null_artifact_or_lineage",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "differences_reflect_dissociation_or_sorting_artifact",
              "cluster_and_singlet_capture_same_biology",
              "cluster_and_singlet_capture_distinct_cd8_lineages"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "biological_relations",
            "kind": "dict",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_greater_than",
                  "greater_than"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "less_than",
                  "much_less_than"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than",
                  "approximately_equal"
                ]
              },
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal"
                ]
              },
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal",
                  "greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_exhaustion_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_naive_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_proliferation_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_expansion_richness_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_multichain_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_intermediate_gzmk_overgeneralized",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_melanocytic_conjugate_overcall",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "values": [
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_relations_mass_hedge",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              }
            }
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: FAIL\n  + list_match: list_match (answer_field='cd8_states_enriched_in_singlet_fraction'): PASS  (score=3.0)\n  + list_match: list_match (answer_field='cd8_states_enriched_in_tumor_cluster_fraction'): PASS  (score=3.0)\n  + clonotype_expansion_direction_data_true: passed=True, score=1.0\n  + biological_inference_data_true: passed=True, score=1.0\n  x dict_match: dict_match (answer_field='biological_relations'): FAIL  (score=6.0)\n  + hard_fail 'singlet_top3_contains_naive_state_prior_pushing': not triggered\n  + hard_fail 'tumor_cluster_top3_contains_exhausted_state_prior_pushing': not triggered\n  + hard_fail 'singlet_top3_contains_decoy_token': not triggered\n  + hard_fail 'tumor_cluster_top3_contains_decoy_token': not triggered\n  + hard_fail 'TOXhigh_Tex_wrong_functional_category': not triggered\n  + hard_fail 'TCF7_stem_like_Tex_wrong_functional_category': not triggered\n  + hard_fail 'MKI67_state_wrong_functional_category': not triggered\n  + hard_fail 'Tn_or_Tn_Tmem_wrong_functional_category': not triggered\n  + hard_fail 'missing_exhausted_or_proliferating_anchor_in_singlet_top3': not triggered\n  + hard_fail 'missing_stemlike_precursor_in_singlet_top3': not triggered\n  + hard_fail 'missing_naive_or_early_memory_anchor_in_tumor_cluster_top3': not triggered\n  + hard_fail 'cf_reported_canonical_expansion_direction': not triggered\n  + hard_fail 'cf_expansion_lazy_defer': not triggered\n  + hard_fail 'cf_reported_canonical_expected_biology': not triggered\n  + hard_fail 'cf_biological_inference_null_artifact_or_lineage': not triggered\n  + hard_fail 'cf_exhaustion_inverted_prior_push': not triggered\n  + hard_fail 'cf_naive_inverted_prior_push': not triggered\n  + hard_fail 'cf_proliferation_inverted_prior_push': not triggered\n  + hard_fail 'cf_expansion_richness_inverted_prior_push': not triggered\n  + hard_fail 'cf_multichain_inverted_prior_push': not triggered\n  x hard_fail 'cf_intermediate_gzmk_overgeneralized': TRIGGERED\n  + hard_fail 'cf_melanocytic_conjugate_overcall': not triggered\n  + hard_fail 'cf_relations_mass_hedge': not triggered",
      "metrics": {
        "hard_fail_triggered": [
          "cf_intermediate_gzmk_overgeneralized"
        ],
        "pass_rule": "all",
        "scoring_count": 5,
        "scoring_passed": 4,
        "scoring_total_score": 14
      },
      "fieldScores": {
        "biological_inference_data_true": 1,
        "clonotype_expansion_direction_data_true": 1,
        "dict_match": 6,
        "list_match": 3
      },
      "graderChecks": [
        {
          "name": "cd8_states_enriched_in_singlet_fraction",
          "kind": "list_match",
          "passed": true,
          "score": 3
        },
        {
          "name": "cd8_states_enriched_in_tumor_cluster_fraction",
          "kind": "list_match",
          "passed": true,
          "score": 3
        },
        {
          "name": "clonotype_expansion_direction_data_true",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "biological_inference_data_true",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "biological_relations",
          "kind": "list_match",
          "passed": false,
          "score": 6
        },
        {
          "name": "singlet_top3_contains_naive_state_prior_pushing",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "singlet_top3_contains_decoy_token",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "tumor_cluster_top3_contains_decoy_token",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "TOXhigh_Tex_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "TCF7_stem_like_Tex_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "MKI67_state_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "Tn_or_Tn_Tmem_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_stemlike_precursor_in_singlet_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_reported_canonical_expansion_direction",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_expansion_lazy_defer",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_reported_canonical_expected_biology",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_biological_inference_null_artifact_or_lineage",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_exhaustion_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_naive_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_proliferation_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_expansion_richness_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_multichain_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_intermediate_gzmk_overgeneralized",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "cf_melanocytic_conjugate_overcall",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_relations_mass_hedge",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "biological_inference": "singlet_antigen_experienced_cluster_bystander",
        "biological_relations": {
          "clonal_expansion_richness_in_singlets_vs_tumor_cluster": "greater_than",
          "exhaustion_program_in_singlets_vs_tumor_cluster": "greater_than",
          "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": "greater_than",
          "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": "greater_than",
          "multichain_tcr_rate_in_singlets_vs_tumor_cluster": "greater_than",
          "naive_memory_program_in_singlets_vs_tumor_cluster": "less_than",
          "proliferation_program_in_singlets_vs_tumor_cluster": "greater_than"
        },
        "cd8_states_enriched_in_singlet_fraction": [
          {
            "functional_category": "stem_like_exhausted",
            "state_label": "TCF7_stem_like_Tex"
          },
          {
            "functional_category": "terminal_exhausted",
            "state_label": "TOXhigh_Tex"
          },
          {
            "functional_category": "proliferating",
            "state_label": "MKI67high_Tex_Tprol"
          }
        ],
        "cd8_states_enriched_in_tumor_cluster_fraction": [
          {
            "functional_category": "naive_memory",
            "state_label": "Tn"
          },
          {
            "functional_category": "naive_memory",
            "state_label": "Tn_Tmem"
          },
          {
            "functional_category": "early_effector_memory",
            "state_label": "early_Tem_GZMK"
          }
        ],
        "clonotype_expansion_finding": "singlet_modestly_more_expanded_than_cluster"
      },
      "nTurns": 36,
      "checkpointSignals": {
        "clustering": true,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": false,
        "nerveJointAnchor": false,
        "multiomeAssembly": false,
        "depthNormalizedAtac": false,
        "regulonInference": false,
        "cd8SubsetIntegration": true,
        "perPatientEnrichment": true,
        "tcrRichnessMetrics": true,
        "orthologProjection": false,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": true,
        "mousePseudobulkDesign": false,
        "cancerEpithelialDenominator": false,
        "geneLevelDe": false,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "trajectoryFile": "scbench-long/trajectories/10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1__claude-opus-4-8__claude-code__r1.json",
      "runId": "claude-opus-4-8__claude-code__10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1__r1",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-opus-4-8-claude-code",
      "modelName": "claude-opus-4-8",
      "harness": "claude-code",
      "task": "10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 2,
      "passed": false,
      "score": 14,
      "durationS": 2776.04,
      "cost": 7.624609500000001,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_singlet_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "TOXhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "LAG3high_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "GZMKhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "intermediate_exhausted_gzmk"
                    ],
                    "canonical": "intermediate_exhausted_gzmk"
                  }
                }
              },
              {
                "key": "TCF7_stem_like_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "stem_like_exhausted"
                    ],
                    "canonical": "stem_like_exhausted"
                  }
                }
              },
              {
                "key": "MKI67_pos_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              },
              {
                "key": "MKI67high_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_tumor_cluster_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "Tn",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "Tn_Tmem",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "early_Tem_GZMK",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              },
              {
                "key": "Tem_GZMH",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              }
            ]
          },
          {
            "name": "clonotype_expansion_direction_data_true",
            "role": "gate",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_substantially_more_expanded_than_cluster",
              "singlet_modestly_more_expanded_than_cluster"
            ]
          },
          {
            "name": "biological_inference_data_true",
            "role": "gate",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_antigen_experienced_cluster_bystander"
            ]
          },
          {
            "name": "singlet_top3_contains_naive_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Tn",
              "Tn_Tmem",
              "early_Tem_GZMK",
              "Tem_GZMH"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "TOXhigh_Tex",
              "LAG3high_Tex",
              "GZMKhigh_Tex",
              "TCF7_stem_like_Tex",
              "MKI67_pos_Tex_Tprol",
              "MKI67high_Tex_Tprol"
            ]
          },
          {
            "name": "singlet_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "TOXhigh_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "TCF7_stem_like_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "MKI67_state_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "Tn_or_Tn_Tmem_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_stemlike_precursor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_reported_canonical_expansion_direction",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_substantially_more_expanded_than_singlet",
              "cluster_modestly_more_expanded_than_singlet"
            ]
          },
          {
            "name": "cf_expansion_lazy_defer",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "expansion_pattern_not_detectable"
            ]
          },
          {
            "name": "cf_reported_canonical_expected_biology",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_antigen_experienced_singlet_bystander"
            ]
          },
          {
            "name": "cf_biological_inference_null_artifact_or_lineage",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "differences_reflect_dissociation_or_sorting_artifact",
              "cluster_and_singlet_capture_same_biology",
              "cluster_and_singlet_capture_distinct_cd8_lineages"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "biological_relations",
            "kind": "dict",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_greater_than",
                  "greater_than"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "less_than",
                  "much_less_than"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than",
                  "approximately_equal"
                ]
              },
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal"
                ]
              },
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal",
                  "greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_exhaustion_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_naive_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_proliferation_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_expansion_richness_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_multichain_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_intermediate_gzmk_overgeneralized",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_melanocytic_conjugate_overcall",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "values": [
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_relations_mass_hedge",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              }
            }
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: FAIL\n  + list_match: list_match (answer_field='cd8_states_enriched_in_singlet_fraction'): PASS  (score=3.0)\n  + list_match: list_match (answer_field='cd8_states_enriched_in_tumor_cluster_fraction'): PASS  (score=3.0)\n  + clonotype_expansion_direction_data_true: passed=True, score=1.0\n  + biological_inference_data_true: passed=True, score=1.0\n  x dict_match: dict_match (answer_field='biological_relations'): FAIL  (score=6.0)\n  + hard_fail 'singlet_top3_contains_naive_state_prior_pushing': not triggered\n  + hard_fail 'tumor_cluster_top3_contains_exhausted_state_prior_pushing': not triggered\n  + hard_fail 'singlet_top3_contains_decoy_token': not triggered\n  + hard_fail 'tumor_cluster_top3_contains_decoy_token': not triggered\n  + hard_fail 'TOXhigh_Tex_wrong_functional_category': not triggered\n  + hard_fail 'TCF7_stem_like_Tex_wrong_functional_category': not triggered\n  + hard_fail 'MKI67_state_wrong_functional_category': not triggered\n  + hard_fail 'Tn_or_Tn_Tmem_wrong_functional_category': not triggered\n  + hard_fail 'missing_exhausted_or_proliferating_anchor_in_singlet_top3': not triggered\n  + hard_fail 'missing_stemlike_precursor_in_singlet_top3': not triggered\n  + hard_fail 'missing_naive_or_early_memory_anchor_in_tumor_cluster_top3': not triggered\n  + hard_fail 'cf_reported_canonical_expansion_direction': not triggered\n  + hard_fail 'cf_expansion_lazy_defer': not triggered\n  + hard_fail 'cf_reported_canonical_expected_biology': not triggered\n  + hard_fail 'cf_biological_inference_null_artifact_or_lineage': not triggered\n  + hard_fail 'cf_exhaustion_inverted_prior_push': not triggered\n  + hard_fail 'cf_naive_inverted_prior_push': not triggered\n  + hard_fail 'cf_proliferation_inverted_prior_push': not triggered\n  + hard_fail 'cf_expansion_richness_inverted_prior_push': not triggered\n  + hard_fail 'cf_multichain_inverted_prior_push': not triggered\n  + hard_fail 'cf_intermediate_gzmk_overgeneralized': not triggered\n  + hard_fail 'cf_melanocytic_conjugate_overcall': not triggered\n  + hard_fail 'cf_relations_mass_hedge': not triggered",
      "metrics": {
        "hard_fail_triggered": [],
        "pass_rule": "all",
        "scoring_count": 5,
        "scoring_passed": 4,
        "scoring_total_score": 14
      },
      "fieldScores": {
        "biological_inference_data_true": 1,
        "clonotype_expansion_direction_data_true": 1,
        "dict_match": 6,
        "list_match": 3
      },
      "graderChecks": [
        {
          "name": "cd8_states_enriched_in_singlet_fraction",
          "kind": "list_match",
          "passed": true,
          "score": 3
        },
        {
          "name": "cd8_states_enriched_in_tumor_cluster_fraction",
          "kind": "list_match",
          "passed": true,
          "score": 3
        },
        {
          "name": "clonotype_expansion_direction_data_true",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "biological_inference_data_true",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "biological_relations",
          "kind": "list_match",
          "passed": false,
          "score": 6
        },
        {
          "name": "singlet_top3_contains_naive_state_prior_pushing",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "singlet_top3_contains_decoy_token",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "tumor_cluster_top3_contains_decoy_token",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "TOXhigh_Tex_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "TCF7_stem_like_Tex_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "MKI67_state_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "Tn_or_Tn_Tmem_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_stemlike_precursor_in_singlet_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_reported_canonical_expansion_direction",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_expansion_lazy_defer",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_reported_canonical_expected_biology",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_biological_inference_null_artifact_or_lineage",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_exhaustion_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_naive_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_proliferation_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_expansion_richness_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_multichain_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_intermediate_gzmk_overgeneralized",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_melanocytic_conjugate_overcall",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_relations_mass_hedge",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "biological_inference": "singlet_antigen_experienced_cluster_bystander",
        "biological_relations": {
          "clonal_expansion_richness_in_singlets_vs_tumor_cluster": "greater_than",
          "exhaustion_program_in_singlets_vs_tumor_cluster": "approximately_equal",
          "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": "approximately_equal",
          "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": "greater_than",
          "multichain_tcr_rate_in_singlets_vs_tumor_cluster": "greater_than",
          "naive_memory_program_in_singlets_vs_tumor_cluster": "less_than",
          "proliferation_program_in_singlets_vs_tumor_cluster": "greater_than"
        },
        "cd8_states_enriched_in_singlet_fraction": [
          {
            "functional_category": "stem_like_exhausted",
            "state_label": "TCF7_stem_like_Tex"
          },
          {
            "functional_category": "terminal_exhausted",
            "state_label": "TOXhigh_Tex"
          },
          {
            "functional_category": "proliferating",
            "state_label": "MKI67high_Tex_Tprol"
          }
        ],
        "cd8_states_enriched_in_tumor_cluster_fraction": [
          {
            "functional_category": "naive_memory",
            "state_label": "Tn"
          },
          {
            "functional_category": "early_effector_memory",
            "state_label": "early_Tem_GZMK"
          },
          {
            "functional_category": "early_effector_memory",
            "state_label": "Tem_GZMH"
          }
        ],
        "clonotype_expansion_finding": "singlet_modestly_more_expanded_than_cluster"
      },
      "nTurns": 48,
      "checkpointSignals": {
        "clustering": true,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": false,
        "nerveJointAnchor": false,
        "multiomeAssembly": false,
        "depthNormalizedAtac": true,
        "regulonInference": false,
        "cd8SubsetIntegration": true,
        "perPatientEnrichment": true,
        "tcrRichnessMetrics": true,
        "orthologProjection": false,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": true,
        "mousePseudobulkDesign": true,
        "cancerEpithelialDenominator": false,
        "geneLevelDe": true,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "trajectoryFile": "scbench-long/trajectories/10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1__claude-opus-4-8__claude-code__r2.json",
      "runId": "claude-opus-4-8__claude-code__10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1__r2",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-opus-4-8-claude-code",
      "modelName": "claude-opus-4-8",
      "harness": "claude-code",
      "task": "10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": false,
      "score": 14,
      "durationS": 2445.16,
      "cost": 6.10710425,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_singlet_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "TOXhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "LAG3high_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "GZMKhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "intermediate_exhausted_gzmk"
                    ],
                    "canonical": "intermediate_exhausted_gzmk"
                  }
                }
              },
              {
                "key": "TCF7_stem_like_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "stem_like_exhausted"
                    ],
                    "canonical": "stem_like_exhausted"
                  }
                }
              },
              {
                "key": "MKI67_pos_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              },
              {
                "key": "MKI67high_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_tumor_cluster_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "Tn",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "Tn_Tmem",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "early_Tem_GZMK",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              },
              {
                "key": "Tem_GZMH",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              }
            ]
          },
          {
            "name": "clonotype_expansion_direction_data_true",
            "role": "gate",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_substantially_more_expanded_than_cluster",
              "singlet_modestly_more_expanded_than_cluster"
            ]
          },
          {
            "name": "biological_inference_data_true",
            "role": "gate",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_antigen_experienced_cluster_bystander"
            ]
          },
          {
            "name": "singlet_top3_contains_naive_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Tn",
              "Tn_Tmem",
              "early_Tem_GZMK",
              "Tem_GZMH"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "TOXhigh_Tex",
              "LAG3high_Tex",
              "GZMKhigh_Tex",
              "TCF7_stem_like_Tex",
              "MKI67_pos_Tex_Tprol",
              "MKI67high_Tex_Tprol"
            ]
          },
          {
            "name": "singlet_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "TOXhigh_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "TCF7_stem_like_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "MKI67_state_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "Tn_or_Tn_Tmem_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_stemlike_precursor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_reported_canonical_expansion_direction",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_substantially_more_expanded_than_singlet",
              "cluster_modestly_more_expanded_than_singlet"
            ]
          },
          {
            "name": "cf_expansion_lazy_defer",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "expansion_pattern_not_detectable"
            ]
          },
          {
            "name": "cf_reported_canonical_expected_biology",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_antigen_experienced_singlet_bystander"
            ]
          },
          {
            "name": "cf_biological_inference_null_artifact_or_lineage",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "differences_reflect_dissociation_or_sorting_artifact",
              "cluster_and_singlet_capture_same_biology",
              "cluster_and_singlet_capture_distinct_cd8_lineages"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "biological_relations",
            "kind": "dict",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_greater_than",
                  "greater_than"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "less_than",
                  "much_less_than"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than",
                  "approximately_equal"
                ]
              },
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal"
                ]
              },
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal",
                  "greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_exhaustion_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_naive_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_proliferation_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_expansion_richness_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_multichain_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_intermediate_gzmk_overgeneralized",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_melanocytic_conjugate_overcall",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "values": [
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_relations_mass_hedge",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              }
            }
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: FAIL\n  + list_match: list_match (answer_field='cd8_states_enriched_in_singlet_fraction'): PASS  (score=3.0)\n  + list_match: list_match (answer_field='cd8_states_enriched_in_tumor_cluster_fraction'): PASS  (score=3.0)\n  + clonotype_expansion_direction_data_true: passed=True, score=1.0\n  + biological_inference_data_true: passed=True, score=1.0\n  x dict_match: dict_match (answer_field='biological_relations'): FAIL  (score=6.0)\n  + hard_fail 'singlet_top3_contains_naive_state_prior_pushing': not triggered\n  + hard_fail 'tumor_cluster_top3_contains_exhausted_state_prior_pushing': not triggered\n  + hard_fail 'singlet_top3_contains_decoy_token': not triggered\n  + hard_fail 'tumor_cluster_top3_contains_decoy_token': not triggered\n  + hard_fail 'TOXhigh_Tex_wrong_functional_category': not triggered\n  + hard_fail 'TCF7_stem_like_Tex_wrong_functional_category': not triggered\n  + hard_fail 'MKI67_state_wrong_functional_category': not triggered\n  + hard_fail 'Tn_or_Tn_Tmem_wrong_functional_category': not triggered\n  + hard_fail 'missing_exhausted_or_proliferating_anchor_in_singlet_top3': not triggered\n  + hard_fail 'missing_stemlike_precursor_in_singlet_top3': not triggered\n  + hard_fail 'missing_naive_or_early_memory_anchor_in_tumor_cluster_top3': not triggered\n  + hard_fail 'cf_reported_canonical_expansion_direction': not triggered\n  + hard_fail 'cf_expansion_lazy_defer': not triggered\n  + hard_fail 'cf_reported_canonical_expected_biology': not triggered\n  + hard_fail 'cf_biological_inference_null_artifact_or_lineage': not triggered\n  + hard_fail 'cf_exhaustion_inverted_prior_push': not triggered\n  + hard_fail 'cf_naive_inverted_prior_push': not triggered\n  + hard_fail 'cf_proliferation_inverted_prior_push': not triggered\n  + hard_fail 'cf_expansion_richness_inverted_prior_push': not triggered\n  + hard_fail 'cf_multichain_inverted_prior_push': not triggered\n  + hard_fail 'cf_intermediate_gzmk_overgeneralized': not triggered\n  + hard_fail 'cf_melanocytic_conjugate_overcall': not triggered\n  + hard_fail 'cf_relations_mass_hedge': not triggered",
      "metrics": {
        "hard_fail_triggered": [],
        "pass_rule": "all",
        "scoring_count": 5,
        "scoring_passed": 4,
        "scoring_total_score": 14
      },
      "fieldScores": {
        "biological_inference_data_true": 1,
        "clonotype_expansion_direction_data_true": 1,
        "dict_match": 6,
        "list_match": 3
      },
      "graderChecks": [
        {
          "name": "cd8_states_enriched_in_singlet_fraction",
          "kind": "list_match",
          "passed": true,
          "score": 3
        },
        {
          "name": "cd8_states_enriched_in_tumor_cluster_fraction",
          "kind": "list_match",
          "passed": true,
          "score": 3
        },
        {
          "name": "clonotype_expansion_direction_data_true",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "biological_inference_data_true",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "biological_relations",
          "kind": "list_match",
          "passed": false,
          "score": 6
        },
        {
          "name": "singlet_top3_contains_naive_state_prior_pushing",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "singlet_top3_contains_decoy_token",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "tumor_cluster_top3_contains_decoy_token",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "TOXhigh_Tex_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "TCF7_stem_like_Tex_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "MKI67_state_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "Tn_or_Tn_Tmem_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_stemlike_precursor_in_singlet_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_reported_canonical_expansion_direction",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_expansion_lazy_defer",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_reported_canonical_expected_biology",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_biological_inference_null_artifact_or_lineage",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_exhaustion_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_naive_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_proliferation_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_expansion_richness_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_multichain_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_intermediate_gzmk_overgeneralized",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_melanocytic_conjugate_overcall",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_relations_mass_hedge",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "biological_inference": "singlet_antigen_experienced_cluster_bystander",
        "biological_relations": {
          "clonal_expansion_richness_in_singlets_vs_tumor_cluster": "greater_than",
          "exhaustion_program_in_singlets_vs_tumor_cluster": "greater_than",
          "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": "approximately_equal",
          "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": "approximately_equal",
          "multichain_tcr_rate_in_singlets_vs_tumor_cluster": "approximately_equal",
          "naive_memory_program_in_singlets_vs_tumor_cluster": "less_than",
          "proliferation_program_in_singlets_vs_tumor_cluster": "approximately_equal"
        },
        "cd8_states_enriched_in_singlet_fraction": [
          {
            "functional_category": "stem_like_exhausted",
            "state_label": "TCF7_stem_like_Tex"
          },
          {
            "functional_category": "intermediate_exhausted_gzmk",
            "state_label": "GZMKhigh_Tex"
          },
          {
            "functional_category": "terminal_exhausted",
            "state_label": "TOXhigh_Tex"
          }
        ],
        "cd8_states_enriched_in_tumor_cluster_fraction": [
          {
            "functional_category": "naive_memory",
            "state_label": "Tn"
          },
          {
            "functional_category": "early_effector_memory",
            "state_label": "early_Tem_GZMK"
          },
          {
            "functional_category": "early_effector_memory",
            "state_label": "Tem_GZMH"
          }
        ],
        "clonotype_expansion_finding": "singlet_modestly_more_expanded_than_cluster"
      },
      "nTurns": 39,
      "checkpointSignals": {
        "clustering": true,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": false,
        "nerveJointAnchor": false,
        "multiomeAssembly": true,
        "depthNormalizedAtac": false,
        "regulonInference": true,
        "cd8SubsetIntegration": true,
        "perPatientEnrichment": true,
        "tcrRichnessMetrics": true,
        "orthologProjection": false,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": true,
        "mousePseudobulkDesign": false,
        "cancerEpithelialDenominator": false,
        "geneLevelDe": false,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "trajectoryFile": "scbench-long/trajectories/10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1__claude-opus-4-8__claude-code__r3.json",
      "runId": "claude-opus-4-8__claude-code__10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1__r3",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-opus-4-8-pi",
      "modelName": "claude-opus-4-8",
      "harness": "pi",
      "task": "10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 1,
      "passed": false,
      "score": 14,
      "durationS": 2894.01,
      "cost": 6.70895925,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_singlet_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "TOXhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "LAG3high_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "GZMKhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "intermediate_exhausted_gzmk"
                    ],
                    "canonical": "intermediate_exhausted_gzmk"
                  }
                }
              },
              {
                "key": "TCF7_stem_like_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "stem_like_exhausted"
                    ],
                    "canonical": "stem_like_exhausted"
                  }
                }
              },
              {
                "key": "MKI67_pos_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              },
              {
                "key": "MKI67high_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_tumor_cluster_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "Tn",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "Tn_Tmem",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "early_Tem_GZMK",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              },
              {
                "key": "Tem_GZMH",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              }
            ]
          },
          {
            "name": "clonotype_expansion_direction_data_true",
            "role": "gate",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_substantially_more_expanded_than_cluster",
              "singlet_modestly_more_expanded_than_cluster"
            ]
          },
          {
            "name": "biological_inference_data_true",
            "role": "gate",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_antigen_experienced_cluster_bystander"
            ]
          },
          {
            "name": "singlet_top3_contains_naive_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Tn",
              "Tn_Tmem",
              "early_Tem_GZMK",
              "Tem_GZMH"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "TOXhigh_Tex",
              "LAG3high_Tex",
              "GZMKhigh_Tex",
              "TCF7_stem_like_Tex",
              "MKI67_pos_Tex_Tprol",
              "MKI67high_Tex_Tprol"
            ]
          },
          {
            "name": "singlet_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "TOXhigh_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "TCF7_stem_like_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "MKI67_state_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "Tn_or_Tn_Tmem_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_stemlike_precursor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_reported_canonical_expansion_direction",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_substantially_more_expanded_than_singlet",
              "cluster_modestly_more_expanded_than_singlet"
            ]
          },
          {
            "name": "cf_expansion_lazy_defer",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "expansion_pattern_not_detectable"
            ]
          },
          {
            "name": "cf_reported_canonical_expected_biology",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_antigen_experienced_singlet_bystander"
            ]
          },
          {
            "name": "cf_biological_inference_null_artifact_or_lineage",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "differences_reflect_dissociation_or_sorting_artifact",
              "cluster_and_singlet_capture_same_biology",
              "cluster_and_singlet_capture_distinct_cd8_lineages"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "biological_relations",
            "kind": "dict",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_greater_than",
                  "greater_than"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "less_than",
                  "much_less_than"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than",
                  "approximately_equal"
                ]
              },
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal"
                ]
              },
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal",
                  "greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_exhaustion_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_naive_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_proliferation_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_expansion_richness_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_multichain_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_intermediate_gzmk_overgeneralized",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_melanocytic_conjugate_overcall",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "values": [
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_relations_mass_hedge",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              }
            }
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: FAIL\n  + list_match: list_match (answer_field='cd8_states_enriched_in_singlet_fraction'): PASS  (score=3.0)\n  + list_match: list_match (answer_field='cd8_states_enriched_in_tumor_cluster_fraction'): PASS  (score=3.0)\n  + clonotype_expansion_direction_data_true: passed=True, score=1.0\n  + biological_inference_data_true: passed=True, score=1.0\n  x dict_match: dict_match (answer_field='biological_relations'): FAIL  (score=6.0)\n  + hard_fail 'singlet_top3_contains_naive_state_prior_pushing': not triggered\n  + hard_fail 'tumor_cluster_top3_contains_exhausted_state_prior_pushing': not triggered\n  + hard_fail 'singlet_top3_contains_decoy_token': not triggered\n  + hard_fail 'tumor_cluster_top3_contains_decoy_token': not triggered\n  + hard_fail 'TOXhigh_Tex_wrong_functional_category': not triggered\n  + hard_fail 'TCF7_stem_like_Tex_wrong_functional_category': not triggered\n  + hard_fail 'MKI67_state_wrong_functional_category': not triggered\n  + hard_fail 'Tn_or_Tn_Tmem_wrong_functional_category': not triggered\n  + hard_fail 'missing_exhausted_or_proliferating_anchor_in_singlet_top3': not triggered\n  + hard_fail 'missing_stemlike_precursor_in_singlet_top3': not triggered\n  + hard_fail 'missing_naive_or_early_memory_anchor_in_tumor_cluster_top3': not triggered\n  + hard_fail 'cf_reported_canonical_expansion_direction': not triggered\n  + hard_fail 'cf_expansion_lazy_defer': not triggered\n  + hard_fail 'cf_reported_canonical_expected_biology': not triggered\n  + hard_fail 'cf_biological_inference_null_artifact_or_lineage': not triggered\n  + hard_fail 'cf_exhaustion_inverted_prior_push': not triggered\n  + hard_fail 'cf_naive_inverted_prior_push': not triggered\n  + hard_fail 'cf_proliferation_inverted_prior_push': not triggered\n  + hard_fail 'cf_expansion_richness_inverted_prior_push': not triggered\n  + hard_fail 'cf_multichain_inverted_prior_push': not triggered\n  x hard_fail 'cf_intermediate_gzmk_overgeneralized': TRIGGERED\n  + hard_fail 'cf_melanocytic_conjugate_overcall': not triggered\n  + hard_fail 'cf_relations_mass_hedge': not triggered",
      "metrics": {
        "hard_fail_triggered": [
          "cf_intermediate_gzmk_overgeneralized"
        ],
        "pass_rule": "all",
        "scoring_count": 5,
        "scoring_passed": 4,
        "scoring_total_score": 14
      },
      "fieldScores": {
        "biological_inference_data_true": 1,
        "clonotype_expansion_direction_data_true": 1,
        "dict_match": 6,
        "list_match": 3
      },
      "graderChecks": [
        {
          "name": "cd8_states_enriched_in_singlet_fraction",
          "kind": "list_match",
          "passed": true,
          "score": 3
        },
        {
          "name": "cd8_states_enriched_in_tumor_cluster_fraction",
          "kind": "list_match",
          "passed": true,
          "score": 3
        },
        {
          "name": "clonotype_expansion_direction_data_true",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "biological_inference_data_true",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "biological_relations",
          "kind": "list_match",
          "passed": false,
          "score": 6
        },
        {
          "name": "singlet_top3_contains_naive_state_prior_pushing",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "singlet_top3_contains_decoy_token",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "tumor_cluster_top3_contains_decoy_token",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "TOXhigh_Tex_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "TCF7_stem_like_Tex_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "MKI67_state_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "Tn_or_Tn_Tmem_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_stemlike_precursor_in_singlet_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_reported_canonical_expansion_direction",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_expansion_lazy_defer",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_reported_canonical_expected_biology",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_biological_inference_null_artifact_or_lineage",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_exhaustion_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_naive_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_proliferation_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_expansion_richness_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_multichain_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_intermediate_gzmk_overgeneralized",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "cf_melanocytic_conjugate_overcall",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_relations_mass_hedge",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "biological_inference": "singlet_antigen_experienced_cluster_bystander",
        "biological_relations": {
          "clonal_expansion_richness_in_singlets_vs_tumor_cluster": "greater_than",
          "exhaustion_program_in_singlets_vs_tumor_cluster": "greater_than",
          "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": "greater_than",
          "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": "approximately_equal",
          "multichain_tcr_rate_in_singlets_vs_tumor_cluster": "greater_than",
          "naive_memory_program_in_singlets_vs_tumor_cluster": "less_than",
          "proliferation_program_in_singlets_vs_tumor_cluster": "greater_than"
        },
        "cd8_states_enriched_in_singlet_fraction": [
          {
            "functional_category": "stem_like_exhausted",
            "state_label": "TCF7_stem_like_Tex"
          },
          {
            "functional_category": "terminal_exhausted",
            "state_label": "TOXhigh_Tex"
          },
          {
            "functional_category": "proliferating",
            "state_label": "MKI67high_Tex_Tprol"
          }
        ],
        "cd8_states_enriched_in_tumor_cluster_fraction": [
          {
            "functional_category": "naive_memory",
            "state_label": "Tn_Tmem"
          },
          {
            "functional_category": "early_effector_memory",
            "state_label": "early_Tem_GZMK"
          },
          {
            "functional_category": "early_effector_memory",
            "state_label": "Tem_GZMH"
          }
        ],
        "clonotype_expansion_finding": "singlet_modestly_more_expanded_than_cluster"
      },
      "nTurns": 37,
      "checkpointSignals": {
        "clustering": true,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": false,
        "nerveJointAnchor": false,
        "multiomeAssembly": false,
        "depthNormalizedAtac": false,
        "regulonInference": false,
        "cd8SubsetIntegration": true,
        "perPatientEnrichment": true,
        "tcrRichnessMetrics": true,
        "orthologProjection": false,
        "epiSubsetSelection": true,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": true,
        "mousePseudobulkDesign": true,
        "cancerEpithelialDenominator": true,
        "geneLevelDe": true,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "trajectoryFile": "scbench-long/trajectories/10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1__claude-opus-4-8__pi__r1.json",
      "runId": "claude-opus-4-8__pi__10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1__r1",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-opus-4-8-pi",
      "modelName": "claude-opus-4-8",
      "harness": "pi",
      "task": "10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 2,
      "passed": false,
      "score": 14,
      "durationS": 2042.13,
      "cost": 4.64880825,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_singlet_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "TOXhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "LAG3high_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "GZMKhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "intermediate_exhausted_gzmk"
                    ],
                    "canonical": "intermediate_exhausted_gzmk"
                  }
                }
              },
              {
                "key": "TCF7_stem_like_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "stem_like_exhausted"
                    ],
                    "canonical": "stem_like_exhausted"
                  }
                }
              },
              {
                "key": "MKI67_pos_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              },
              {
                "key": "MKI67high_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_tumor_cluster_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "Tn",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "Tn_Tmem",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "early_Tem_GZMK",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              },
              {
                "key": "Tem_GZMH",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              }
            ]
          },
          {
            "name": "clonotype_expansion_direction_data_true",
            "role": "gate",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_substantially_more_expanded_than_cluster",
              "singlet_modestly_more_expanded_than_cluster"
            ]
          },
          {
            "name": "biological_inference_data_true",
            "role": "gate",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_antigen_experienced_cluster_bystander"
            ]
          },
          {
            "name": "singlet_top3_contains_naive_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Tn",
              "Tn_Tmem",
              "early_Tem_GZMK",
              "Tem_GZMH"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "TOXhigh_Tex",
              "LAG3high_Tex",
              "GZMKhigh_Tex",
              "TCF7_stem_like_Tex",
              "MKI67_pos_Tex_Tprol",
              "MKI67high_Tex_Tprol"
            ]
          },
          {
            "name": "singlet_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "TOXhigh_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "TCF7_stem_like_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "MKI67_state_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "Tn_or_Tn_Tmem_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_stemlike_precursor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_reported_canonical_expansion_direction",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_substantially_more_expanded_than_singlet",
              "cluster_modestly_more_expanded_than_singlet"
            ]
          },
          {
            "name": "cf_expansion_lazy_defer",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "expansion_pattern_not_detectable"
            ]
          },
          {
            "name": "cf_reported_canonical_expected_biology",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_antigen_experienced_singlet_bystander"
            ]
          },
          {
            "name": "cf_biological_inference_null_artifact_or_lineage",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "differences_reflect_dissociation_or_sorting_artifact",
              "cluster_and_singlet_capture_same_biology",
              "cluster_and_singlet_capture_distinct_cd8_lineages"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "biological_relations",
            "kind": "dict",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_greater_than",
                  "greater_than"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "less_than",
                  "much_less_than"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than",
                  "approximately_equal"
                ]
              },
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal"
                ]
              },
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal",
                  "greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_exhaustion_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_naive_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_proliferation_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_expansion_richness_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_multichain_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_intermediate_gzmk_overgeneralized",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_melanocytic_conjugate_overcall",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "values": [
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_relations_mass_hedge",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              }
            }
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: FAIL\n  + list_match: list_match (answer_field='cd8_states_enriched_in_singlet_fraction'): PASS  (score=3.0)\n  + list_match: list_match (answer_field='cd8_states_enriched_in_tumor_cluster_fraction'): PASS  (score=3.0)\n  + clonotype_expansion_direction_data_true: passed=True, score=1.0\n  + biological_inference_data_true: passed=True, score=1.0\n  x dict_match: dict_match (answer_field='biological_relations'): FAIL  (score=6.0)\n  + hard_fail 'singlet_top3_contains_naive_state_prior_pushing': not triggered\n  + hard_fail 'tumor_cluster_top3_contains_exhausted_state_prior_pushing': not triggered\n  + hard_fail 'singlet_top3_contains_decoy_token': not triggered\n  + hard_fail 'tumor_cluster_top3_contains_decoy_token': not triggered\n  + hard_fail 'TOXhigh_Tex_wrong_functional_category': not triggered\n  + hard_fail 'TCF7_stem_like_Tex_wrong_functional_category': not triggered\n  + hard_fail 'MKI67_state_wrong_functional_category': not triggered\n  + hard_fail 'Tn_or_Tn_Tmem_wrong_functional_category': not triggered\n  + hard_fail 'missing_exhausted_or_proliferating_anchor_in_singlet_top3': not triggered\n  + hard_fail 'missing_stemlike_precursor_in_singlet_top3': not triggered\n  + hard_fail 'missing_naive_or_early_memory_anchor_in_tumor_cluster_top3': not triggered\n  + hard_fail 'cf_reported_canonical_expansion_direction': not triggered\n  + hard_fail 'cf_expansion_lazy_defer': not triggered\n  + hard_fail 'cf_reported_canonical_expected_biology': not triggered\n  + hard_fail 'cf_biological_inference_null_artifact_or_lineage': not triggered\n  + hard_fail 'cf_exhaustion_inverted_prior_push': not triggered\n  + hard_fail 'cf_naive_inverted_prior_push': not triggered\n  + hard_fail 'cf_proliferation_inverted_prior_push': not triggered\n  + hard_fail 'cf_expansion_richness_inverted_prior_push': not triggered\n  + hard_fail 'cf_multichain_inverted_prior_push': not triggered\n  x hard_fail 'cf_intermediate_gzmk_overgeneralized': TRIGGERED\n  + hard_fail 'cf_melanocytic_conjugate_overcall': not triggered\n  + hard_fail 'cf_relations_mass_hedge': not triggered",
      "metrics": {
        "hard_fail_triggered": [
          "cf_intermediate_gzmk_overgeneralized"
        ],
        "pass_rule": "all",
        "scoring_count": 5,
        "scoring_passed": 4,
        "scoring_total_score": 14
      },
      "fieldScores": {
        "biological_inference_data_true": 1,
        "clonotype_expansion_direction_data_true": 1,
        "dict_match": 6,
        "list_match": 3
      },
      "graderChecks": [
        {
          "name": "cd8_states_enriched_in_singlet_fraction",
          "kind": "list_match",
          "passed": true,
          "score": 3
        },
        {
          "name": "cd8_states_enriched_in_tumor_cluster_fraction",
          "kind": "list_match",
          "passed": true,
          "score": 3
        },
        {
          "name": "clonotype_expansion_direction_data_true",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "biological_inference_data_true",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "biological_relations",
          "kind": "list_match",
          "passed": false,
          "score": 6
        },
        {
          "name": "singlet_top3_contains_naive_state_prior_pushing",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "singlet_top3_contains_decoy_token",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "tumor_cluster_top3_contains_decoy_token",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "TOXhigh_Tex_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "TCF7_stem_like_Tex_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "MKI67_state_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "Tn_or_Tn_Tmem_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_stemlike_precursor_in_singlet_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_reported_canonical_expansion_direction",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_expansion_lazy_defer",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_reported_canonical_expected_biology",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_biological_inference_null_artifact_or_lineage",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_exhaustion_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_naive_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_proliferation_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_expansion_richness_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_multichain_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_intermediate_gzmk_overgeneralized",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "cf_melanocytic_conjugate_overcall",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_relations_mass_hedge",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "biological_inference": "singlet_antigen_experienced_cluster_bystander",
        "biological_relations": {
          "clonal_expansion_richness_in_singlets_vs_tumor_cluster": "greater_than",
          "exhaustion_program_in_singlets_vs_tumor_cluster": "greater_than",
          "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": "greater_than",
          "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": "approximately_equal",
          "multichain_tcr_rate_in_singlets_vs_tumor_cluster": "greater_than",
          "naive_memory_program_in_singlets_vs_tumor_cluster": "less_than",
          "proliferation_program_in_singlets_vs_tumor_cluster": "greater_than"
        },
        "cd8_states_enriched_in_singlet_fraction": [
          {
            "functional_category": "stem_like_exhausted",
            "state_label": "TCF7_stem_like_Tex"
          },
          {
            "functional_category": "intermediate_exhausted_gzmk",
            "state_label": "GZMKhigh_Tex"
          },
          {
            "functional_category": "terminal_exhausted",
            "state_label": "TOXhigh_Tex"
          }
        ],
        "cd8_states_enriched_in_tumor_cluster_fraction": [
          {
            "functional_category": "naive_memory",
            "state_label": "Tn_Tmem"
          },
          {
            "functional_category": "early_effector_memory",
            "state_label": "early_Tem_GZMK"
          },
          {
            "functional_category": "early_effector_memory",
            "state_label": "Tem_GZMH"
          }
        ],
        "clonotype_expansion_finding": "singlet_modestly_more_expanded_than_cluster"
      },
      "nTurns": 30,
      "checkpointSignals": {
        "clustering": true,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": true,
        "nerveJointAnchor": false,
        "multiomeAssembly": false,
        "depthNormalizedAtac": false,
        "regulonInference": false,
        "cd8SubsetIntegration": true,
        "perPatientEnrichment": true,
        "tcrRichnessMetrics": true,
        "orthologProjection": false,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": true,
        "mousePseudobulkDesign": false,
        "cancerEpithelialDenominator": false,
        "geneLevelDe": true,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "trajectoryFile": "scbench-long/trajectories/10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1__claude-opus-4-8__pi__r2.json",
      "runId": "claude-opus-4-8__pi__10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1__r2",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-opus-4-8-pi",
      "modelName": "claude-opus-4-8",
      "harness": "pi",
      "task": "10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": false,
      "score": 14,
      "durationS": 2366.14,
      "cost": 6.13086225,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_singlet_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "TOXhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "LAG3high_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "GZMKhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "intermediate_exhausted_gzmk"
                    ],
                    "canonical": "intermediate_exhausted_gzmk"
                  }
                }
              },
              {
                "key": "TCF7_stem_like_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "stem_like_exhausted"
                    ],
                    "canonical": "stem_like_exhausted"
                  }
                }
              },
              {
                "key": "MKI67_pos_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              },
              {
                "key": "MKI67high_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_tumor_cluster_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "Tn",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "Tn_Tmem",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "early_Tem_GZMK",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              },
              {
                "key": "Tem_GZMH",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              }
            ]
          },
          {
            "name": "clonotype_expansion_direction_data_true",
            "role": "gate",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_substantially_more_expanded_than_cluster",
              "singlet_modestly_more_expanded_than_cluster"
            ]
          },
          {
            "name": "biological_inference_data_true",
            "role": "gate",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_antigen_experienced_cluster_bystander"
            ]
          },
          {
            "name": "singlet_top3_contains_naive_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Tn",
              "Tn_Tmem",
              "early_Tem_GZMK",
              "Tem_GZMH"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "TOXhigh_Tex",
              "LAG3high_Tex",
              "GZMKhigh_Tex",
              "TCF7_stem_like_Tex",
              "MKI67_pos_Tex_Tprol",
              "MKI67high_Tex_Tprol"
            ]
          },
          {
            "name": "singlet_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "TOXhigh_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "TCF7_stem_like_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "MKI67_state_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "Tn_or_Tn_Tmem_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_stemlike_precursor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_reported_canonical_expansion_direction",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_substantially_more_expanded_than_singlet",
              "cluster_modestly_more_expanded_than_singlet"
            ]
          },
          {
            "name": "cf_expansion_lazy_defer",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "expansion_pattern_not_detectable"
            ]
          },
          {
            "name": "cf_reported_canonical_expected_biology",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_antigen_experienced_singlet_bystander"
            ]
          },
          {
            "name": "cf_biological_inference_null_artifact_or_lineage",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "differences_reflect_dissociation_or_sorting_artifact",
              "cluster_and_singlet_capture_same_biology",
              "cluster_and_singlet_capture_distinct_cd8_lineages"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "biological_relations",
            "kind": "dict",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_greater_than",
                  "greater_than"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "less_than",
                  "much_less_than"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than",
                  "approximately_equal"
                ]
              },
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal"
                ]
              },
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal",
                  "greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_exhaustion_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_naive_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_proliferation_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_expansion_richness_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_multichain_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_intermediate_gzmk_overgeneralized",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_melanocytic_conjugate_overcall",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "values": [
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_relations_mass_hedge",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              }
            }
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: FAIL\n  + list_match: list_match (answer_field='cd8_states_enriched_in_singlet_fraction'): PASS  (score=3.0)\n  + list_match: list_match (answer_field='cd8_states_enriched_in_tumor_cluster_fraction'): PASS  (score=3.0)\n  + clonotype_expansion_direction_data_true: passed=True, score=1.0\n  + biological_inference_data_true: passed=True, score=1.0\n  x dict_match: dict_match (answer_field='biological_relations'): FAIL  (score=6.0)\n  + hard_fail 'singlet_top3_contains_naive_state_prior_pushing': not triggered\n  + hard_fail 'tumor_cluster_top3_contains_exhausted_state_prior_pushing': not triggered\n  + hard_fail 'singlet_top3_contains_decoy_token': not triggered\n  + hard_fail 'tumor_cluster_top3_contains_decoy_token': not triggered\n  + hard_fail 'TOXhigh_Tex_wrong_functional_category': not triggered\n  + hard_fail 'TCF7_stem_like_Tex_wrong_functional_category': not triggered\n  + hard_fail 'MKI67_state_wrong_functional_category': not triggered\n  + hard_fail 'Tn_or_Tn_Tmem_wrong_functional_category': not triggered\n  + hard_fail 'missing_exhausted_or_proliferating_anchor_in_singlet_top3': not triggered\n  + hard_fail 'missing_stemlike_precursor_in_singlet_top3': not triggered\n  + hard_fail 'missing_naive_or_early_memory_anchor_in_tumor_cluster_top3': not triggered\n  + hard_fail 'cf_reported_canonical_expansion_direction': not triggered\n  + hard_fail 'cf_expansion_lazy_defer': not triggered\n  + hard_fail 'cf_reported_canonical_expected_biology': not triggered\n  + hard_fail 'cf_biological_inference_null_artifact_or_lineage': not triggered\n  + hard_fail 'cf_exhaustion_inverted_prior_push': not triggered\n  + hard_fail 'cf_naive_inverted_prior_push': not triggered\n  + hard_fail 'cf_proliferation_inverted_prior_push': not triggered\n  + hard_fail 'cf_expansion_richness_inverted_prior_push': not triggered\n  + hard_fail 'cf_multichain_inverted_prior_push': not triggered\n  x hard_fail 'cf_intermediate_gzmk_overgeneralized': TRIGGERED\n  + hard_fail 'cf_melanocytic_conjugate_overcall': not triggered\n  + hard_fail 'cf_relations_mass_hedge': not triggered",
      "metrics": {
        "hard_fail_triggered": [
          "cf_intermediate_gzmk_overgeneralized"
        ],
        "pass_rule": "all",
        "scoring_count": 5,
        "scoring_passed": 4,
        "scoring_total_score": 14
      },
      "fieldScores": {
        "biological_inference_data_true": 1,
        "clonotype_expansion_direction_data_true": 1,
        "dict_match": 6,
        "list_match": 3
      },
      "graderChecks": [
        {
          "name": "cd8_states_enriched_in_singlet_fraction",
          "kind": "list_match",
          "passed": true,
          "score": 3
        },
        {
          "name": "cd8_states_enriched_in_tumor_cluster_fraction",
          "kind": "list_match",
          "passed": true,
          "score": 3
        },
        {
          "name": "clonotype_expansion_direction_data_true",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "biological_inference_data_true",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "biological_relations",
          "kind": "list_match",
          "passed": false,
          "score": 6
        },
        {
          "name": "singlet_top3_contains_naive_state_prior_pushing",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "singlet_top3_contains_decoy_token",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "tumor_cluster_top3_contains_decoy_token",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "TOXhigh_Tex_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "TCF7_stem_like_Tex_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "MKI67_state_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "Tn_or_Tn_Tmem_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_stemlike_precursor_in_singlet_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_reported_canonical_expansion_direction",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_expansion_lazy_defer",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_reported_canonical_expected_biology",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_biological_inference_null_artifact_or_lineage",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_exhaustion_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_naive_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_proliferation_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_expansion_richness_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_multichain_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_intermediate_gzmk_overgeneralized",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "cf_melanocytic_conjugate_overcall",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_relations_mass_hedge",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "biological_inference": "singlet_antigen_experienced_cluster_bystander",
        "biological_relations": {
          "clonal_expansion_richness_in_singlets_vs_tumor_cluster": "greater_than",
          "exhaustion_program_in_singlets_vs_tumor_cluster": "greater_than",
          "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": "greater_than",
          "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": "approximately_equal",
          "multichain_tcr_rate_in_singlets_vs_tumor_cluster": "greater_than",
          "naive_memory_program_in_singlets_vs_tumor_cluster": "much_less_than",
          "proliferation_program_in_singlets_vs_tumor_cluster": "greater_than"
        },
        "cd8_states_enriched_in_singlet_fraction": [
          {
            "functional_category": "stem_like_exhausted",
            "state_label": "TCF7_stem_like_Tex"
          },
          {
            "functional_category": "terminal_exhausted",
            "state_label": "TOXhigh_Tex"
          },
          {
            "functional_category": "proliferating",
            "state_label": "MKI67high_Tex_Tprol"
          }
        ],
        "cd8_states_enriched_in_tumor_cluster_fraction": [
          {
            "functional_category": "naive_memory",
            "state_label": "Tn"
          },
          {
            "functional_category": "naive_memory",
            "state_label": "Tn_Tmem"
          },
          {
            "functional_category": "early_effector_memory",
            "state_label": "early_Tem_GZMK"
          }
        ],
        "clonotype_expansion_finding": "singlet_modestly_more_expanded_than_cluster"
      },
      "nTurns": 39,
      "checkpointSignals": {
        "clustering": true,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": false,
        "nerveJointAnchor": false,
        "multiomeAssembly": false,
        "depthNormalizedAtac": false,
        "regulonInference": false,
        "cd8SubsetIntegration": true,
        "perPatientEnrichment": true,
        "tcrRichnessMetrics": true,
        "orthologProjection": false,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": true,
        "mousePseudobulkDesign": false,
        "cancerEpithelialDenominator": true,
        "geneLevelDe": false,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "trajectoryFile": "scbench-long/trajectories/10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1__claude-opus-4-8__pi__r3.json",
      "runId": "claude-opus-4-8__pi__10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1__r3",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-sonnet-4-6-pi",
      "modelName": "claude-sonnet-4-6",
      "harness": "pi",
      "task": "10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 1,
      "passed": false,
      "score": 13,
      "durationS": 1674.82,
      "cost": 1.71881835,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_singlet_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "TOXhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "LAG3high_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "GZMKhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "intermediate_exhausted_gzmk"
                    ],
                    "canonical": "intermediate_exhausted_gzmk"
                  }
                }
              },
              {
                "key": "TCF7_stem_like_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "stem_like_exhausted"
                    ],
                    "canonical": "stem_like_exhausted"
                  }
                }
              },
              {
                "key": "MKI67_pos_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              },
              {
                "key": "MKI67high_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_tumor_cluster_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "Tn",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "Tn_Tmem",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "early_Tem_GZMK",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              },
              {
                "key": "Tem_GZMH",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              }
            ]
          },
          {
            "name": "clonotype_expansion_direction_data_true",
            "role": "gate",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_substantially_more_expanded_than_cluster",
              "singlet_modestly_more_expanded_than_cluster"
            ]
          },
          {
            "name": "biological_inference_data_true",
            "role": "gate",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_antigen_experienced_cluster_bystander"
            ]
          },
          {
            "name": "singlet_top3_contains_naive_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Tn",
              "Tn_Tmem",
              "early_Tem_GZMK",
              "Tem_GZMH"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "TOXhigh_Tex",
              "LAG3high_Tex",
              "GZMKhigh_Tex",
              "TCF7_stem_like_Tex",
              "MKI67_pos_Tex_Tprol",
              "MKI67high_Tex_Tprol"
            ]
          },
          {
            "name": "singlet_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "TOXhigh_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "TCF7_stem_like_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "MKI67_state_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "Tn_or_Tn_Tmem_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_stemlike_precursor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_reported_canonical_expansion_direction",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_substantially_more_expanded_than_singlet",
              "cluster_modestly_more_expanded_than_singlet"
            ]
          },
          {
            "name": "cf_expansion_lazy_defer",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "expansion_pattern_not_detectable"
            ]
          },
          {
            "name": "cf_reported_canonical_expected_biology",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_antigen_experienced_singlet_bystander"
            ]
          },
          {
            "name": "cf_biological_inference_null_artifact_or_lineage",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "differences_reflect_dissociation_or_sorting_artifact",
              "cluster_and_singlet_capture_same_biology",
              "cluster_and_singlet_capture_distinct_cd8_lineages"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "biological_relations",
            "kind": "dict",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_greater_than",
                  "greater_than"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "less_than",
                  "much_less_than"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than",
                  "approximately_equal"
                ]
              },
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal"
                ]
              },
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal",
                  "greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_exhaustion_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_naive_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_proliferation_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_expansion_richness_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_multichain_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_intermediate_gzmk_overgeneralized",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_melanocytic_conjugate_overcall",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "values": [
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_relations_mass_hedge",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              }
            }
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: FAIL\n  + list_match: list_match (answer_field='cd8_states_enriched_in_singlet_fraction'): PASS  (score=3.0)\n  + list_match: list_match (answer_field='cd8_states_enriched_in_tumor_cluster_fraction'): PASS  (score=3.0)\n  + clonotype_expansion_direction_data_true: passed=True, score=1.0\n  + biological_inference_data_true: passed=True, score=1.0\n  x dict_match: dict_match (answer_field='biological_relations'): FAIL  (score=5.0)\n  + hard_fail 'singlet_top3_contains_naive_state_prior_pushing': not triggered\n  + hard_fail 'tumor_cluster_top3_contains_exhausted_state_prior_pushing': not triggered\n  + hard_fail 'singlet_top3_contains_decoy_token': not triggered\n  + hard_fail 'tumor_cluster_top3_contains_decoy_token': not triggered\n  + hard_fail 'TOXhigh_Tex_wrong_functional_category': not triggered\n  + hard_fail 'TCF7_stem_like_Tex_wrong_functional_category': not triggered\n  + hard_fail 'MKI67_state_wrong_functional_category': not triggered\n  + hard_fail 'Tn_or_Tn_Tmem_wrong_functional_category': not triggered\n  + hard_fail 'missing_exhausted_or_proliferating_anchor_in_singlet_top3': not triggered\n  + hard_fail 'missing_stemlike_precursor_in_singlet_top3': not triggered\n  + hard_fail 'missing_naive_or_early_memory_anchor_in_tumor_cluster_top3': not triggered\n  + hard_fail 'cf_reported_canonical_expansion_direction': not triggered\n  + hard_fail 'cf_expansion_lazy_defer': not triggered\n  + hard_fail 'cf_reported_canonical_expected_biology': not triggered\n  + hard_fail 'cf_biological_inference_null_artifact_or_lineage': not triggered\n  + hard_fail 'cf_exhaustion_inverted_prior_push': not triggered\n  + hard_fail 'cf_naive_inverted_prior_push': not triggered\n  + hard_fail 'cf_proliferation_inverted_prior_push': not triggered\n  + hard_fail 'cf_expansion_richness_inverted_prior_push': not triggered\n  + hard_fail 'cf_multichain_inverted_prior_push': not triggered\n  x hard_fail 'cf_intermediate_gzmk_overgeneralized': TRIGGERED\n  + hard_fail 'cf_melanocytic_conjugate_overcall': not triggered\n  + hard_fail 'cf_relations_mass_hedge': not triggered",
      "metrics": {
        "hard_fail_triggered": [
          "cf_intermediate_gzmk_overgeneralized"
        ],
        "pass_rule": "all",
        "scoring_count": 5,
        "scoring_passed": 4,
        "scoring_total_score": 13
      },
      "fieldScores": {
        "biological_inference_data_true": 1,
        "clonotype_expansion_direction_data_true": 1,
        "dict_match": 5,
        "list_match": 3
      },
      "graderChecks": [
        {
          "name": "cd8_states_enriched_in_singlet_fraction",
          "kind": "list_match",
          "passed": true,
          "score": 3
        },
        {
          "name": "cd8_states_enriched_in_tumor_cluster_fraction",
          "kind": "list_match",
          "passed": true,
          "score": 3
        },
        {
          "name": "clonotype_expansion_direction_data_true",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "biological_inference_data_true",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "biological_relations",
          "kind": "list_match",
          "passed": false,
          "score": 5
        },
        {
          "name": "singlet_top3_contains_naive_state_prior_pushing",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "singlet_top3_contains_decoy_token",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "tumor_cluster_top3_contains_decoy_token",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "TOXhigh_Tex_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "TCF7_stem_like_Tex_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "MKI67_state_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "Tn_or_Tn_Tmem_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_stemlike_precursor_in_singlet_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_reported_canonical_expansion_direction",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_expansion_lazy_defer",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_reported_canonical_expected_biology",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_biological_inference_null_artifact_or_lineage",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_exhaustion_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_naive_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_proliferation_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_expansion_richness_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_multichain_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_intermediate_gzmk_overgeneralized",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "cf_melanocytic_conjugate_overcall",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_relations_mass_hedge",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "biological_inference": "singlet_antigen_experienced_cluster_bystander",
        "biological_relations": {
          "clonal_expansion_richness_in_singlets_vs_tumor_cluster": "greater_than",
          "exhaustion_program_in_singlets_vs_tumor_cluster": "greater_than",
          "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": "greater_than",
          "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": "greater_than",
          "multichain_tcr_rate_in_singlets_vs_tumor_cluster": "greater_than",
          "naive_memory_program_in_singlets_vs_tumor_cluster": "much_less_than",
          "proliferation_program_in_singlets_vs_tumor_cluster": "approximately_equal"
        },
        "cd8_states_enriched_in_singlet_fraction": [
          {
            "functional_category": "stem_like_exhausted",
            "state_label": "TCF7_stem_like_Tex"
          },
          {
            "functional_category": "intermediate_exhausted_gzmk",
            "state_label": "GZMKhigh_Tex"
          },
          {
            "functional_category": "terminal_exhausted",
            "state_label": "LAG3high_Tex"
          }
        ],
        "cd8_states_enriched_in_tumor_cluster_fraction": [
          {
            "functional_category": "early_effector_memory",
            "state_label": "early_Tem_GZMK"
          },
          {
            "functional_category": "naive_memory",
            "state_label": "Tn_Tmem"
          },
          {
            "functional_category": "naive_memory",
            "state_label": "Tn"
          }
        ],
        "clonotype_expansion_finding": "singlet_modestly_more_expanded_than_cluster"
      },
      "nTurns": 25,
      "checkpointSignals": {
        "clustering": true,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": false,
        "nerveJointAnchor": false,
        "multiomeAssembly": false,
        "depthNormalizedAtac": false,
        "regulonInference": false,
        "cd8SubsetIntegration": true,
        "perPatientEnrichment": true,
        "tcrRichnessMetrics": true,
        "orthologProjection": false,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": true,
        "mousePseudobulkDesign": false,
        "cancerEpithelialDenominator": false,
        "geneLevelDe": false,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "trajectoryFile": "scbench-long/trajectories/10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1__claude-sonnet-4-6__pi__r1.json",
      "runId": "claude-sonnet-4-6__pi__10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1__r1",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-sonnet-4-6-pi",
      "modelName": "claude-sonnet-4-6",
      "harness": "pi",
      "task": "10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 2,
      "passed": true,
      "score": 14,
      "durationS": 2783.73,
      "cost": 3.3729869999999997,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_singlet_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "TOXhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "LAG3high_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "GZMKhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "intermediate_exhausted_gzmk"
                    ],
                    "canonical": "intermediate_exhausted_gzmk"
                  }
                }
              },
              {
                "key": "TCF7_stem_like_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "stem_like_exhausted"
                    ],
                    "canonical": "stem_like_exhausted"
                  }
                }
              },
              {
                "key": "MKI67_pos_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              },
              {
                "key": "MKI67high_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_tumor_cluster_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "Tn",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "Tn_Tmem",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "early_Tem_GZMK",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              },
              {
                "key": "Tem_GZMH",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              }
            ]
          },
          {
            "name": "clonotype_expansion_direction_data_true",
            "role": "gate",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_substantially_more_expanded_than_cluster",
              "singlet_modestly_more_expanded_than_cluster"
            ]
          },
          {
            "name": "biological_inference_data_true",
            "role": "gate",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_antigen_experienced_cluster_bystander"
            ]
          },
          {
            "name": "singlet_top3_contains_naive_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Tn",
              "Tn_Tmem",
              "early_Tem_GZMK",
              "Tem_GZMH"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "TOXhigh_Tex",
              "LAG3high_Tex",
              "GZMKhigh_Tex",
              "TCF7_stem_like_Tex",
              "MKI67_pos_Tex_Tprol",
              "MKI67high_Tex_Tprol"
            ]
          },
          {
            "name": "singlet_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "TOXhigh_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "TCF7_stem_like_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "MKI67_state_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "Tn_or_Tn_Tmem_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_stemlike_precursor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_reported_canonical_expansion_direction",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_substantially_more_expanded_than_singlet",
              "cluster_modestly_more_expanded_than_singlet"
            ]
          },
          {
            "name": "cf_expansion_lazy_defer",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "expansion_pattern_not_detectable"
            ]
          },
          {
            "name": "cf_reported_canonical_expected_biology",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_antigen_experienced_singlet_bystander"
            ]
          },
          {
            "name": "cf_biological_inference_null_artifact_or_lineage",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "differences_reflect_dissociation_or_sorting_artifact",
              "cluster_and_singlet_capture_same_biology",
              "cluster_and_singlet_capture_distinct_cd8_lineages"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "biological_relations",
            "kind": "dict",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_greater_than",
                  "greater_than"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "less_than",
                  "much_less_than"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than",
                  "approximately_equal"
                ]
              },
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal"
                ]
              },
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal",
                  "greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_exhaustion_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_naive_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_proliferation_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_expansion_richness_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_multichain_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_intermediate_gzmk_overgeneralized",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_melanocytic_conjugate_overcall",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "values": [
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_relations_mass_hedge",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              }
            }
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: PASS\n  + list_match: list_match (answer_field='cd8_states_enriched_in_singlet_fraction'): PASS  (score=3.0)\n  + list_match: list_match (answer_field='cd8_states_enriched_in_tumor_cluster_fraction'): PASS  (score=2.0)\n  + clonotype_expansion_direction_data_true: passed=True, score=1.0\n  + biological_inference_data_true: passed=True, score=1.0\n  + dict_match: dict_match (answer_field='biological_relations'): PASS  (score=7.0)\n  + hard_fail 'singlet_top3_contains_naive_state_prior_pushing': not triggered\n  + hard_fail 'tumor_cluster_top3_contains_exhausted_state_prior_pushing': not triggered\n  + hard_fail 'singlet_top3_contains_decoy_token': not triggered\n  + hard_fail 'tumor_cluster_top3_contains_decoy_token': not triggered\n  + hard_fail 'TOXhigh_Tex_wrong_functional_category': not triggered\n  + hard_fail 'TCF7_stem_like_Tex_wrong_functional_category': not triggered\n  + hard_fail 'MKI67_state_wrong_functional_category': not triggered\n  + hard_fail 'Tn_or_Tn_Tmem_wrong_functional_category': not triggered\n  + hard_fail 'missing_exhausted_or_proliferating_anchor_in_singlet_top3': not triggered\n  + hard_fail 'missing_stemlike_precursor_in_singlet_top3': not triggered\n  + hard_fail 'missing_naive_or_early_memory_anchor_in_tumor_cluster_top3': not triggered\n  + hard_fail 'cf_reported_canonical_expansion_direction': not triggered\n  + hard_fail 'cf_expansion_lazy_defer': not triggered\n  + hard_fail 'cf_reported_canonical_expected_biology': not triggered\n  + hard_fail 'cf_biological_inference_null_artifact_or_lineage': not triggered\n  + hard_fail 'cf_exhaustion_inverted_prior_push': not triggered\n  + hard_fail 'cf_naive_inverted_prior_push': not triggered\n  + hard_fail 'cf_proliferation_inverted_prior_push': not triggered\n  + hard_fail 'cf_expansion_richness_inverted_prior_push': not triggered\n  + hard_fail 'cf_multichain_inverted_prior_push': not triggered\n  + hard_fail 'cf_intermediate_gzmk_overgeneralized': not triggered\n  + hard_fail 'cf_melanocytic_conjugate_overcall': not triggered\n  + hard_fail 'cf_relations_mass_hedge': not triggered",
      "metrics": {
        "hard_fail_triggered": [],
        "pass_rule": "all",
        "scoring_count": 5,
        "scoring_passed": 5,
        "scoring_total_score": 14
      },
      "fieldScores": {
        "biological_inference_data_true": 1,
        "clonotype_expansion_direction_data_true": 1,
        "dict_match": 7,
        "list_match": 2
      },
      "graderChecks": [
        {
          "name": "cd8_states_enriched_in_singlet_fraction",
          "kind": "list_match",
          "passed": true,
          "score": 3
        },
        {
          "name": "cd8_states_enriched_in_tumor_cluster_fraction",
          "kind": "list_match",
          "passed": true,
          "score": 2
        },
        {
          "name": "clonotype_expansion_direction_data_true",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "biological_inference_data_true",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "biological_relations",
          "kind": "list_match",
          "passed": true,
          "score": 7
        },
        {
          "name": "singlet_top3_contains_naive_state_prior_pushing",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "singlet_top3_contains_decoy_token",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "tumor_cluster_top3_contains_decoy_token",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "TOXhigh_Tex_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "TCF7_stem_like_Tex_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "MKI67_state_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "Tn_or_Tn_Tmem_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_stemlike_precursor_in_singlet_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_reported_canonical_expansion_direction",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_expansion_lazy_defer",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_reported_canonical_expected_biology",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_biological_inference_null_artifact_or_lineage",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_exhaustion_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_naive_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_proliferation_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_expansion_richness_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_multichain_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_intermediate_gzmk_overgeneralized",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_melanocytic_conjugate_overcall",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_relations_mass_hedge",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "biological_inference": "singlet_antigen_experienced_cluster_bystander",
        "biological_relations": {
          "clonal_expansion_richness_in_singlets_vs_tumor_cluster": "greater_than",
          "exhaustion_program_in_singlets_vs_tumor_cluster": "greater_than",
          "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": "approximately_equal",
          "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": "approximately_equal",
          "multichain_tcr_rate_in_singlets_vs_tumor_cluster": "greater_than",
          "naive_memory_program_in_singlets_vs_tumor_cluster": "much_less_than",
          "proliferation_program_in_singlets_vs_tumor_cluster": "greater_than"
        },
        "cd8_states_enriched_in_singlet_fraction": [
          {
            "functional_category": "stem_like_exhausted",
            "state_label": "TCF7_stem_like_Tex"
          },
          {
            "functional_category": "intermediate_exhausted_gzmk",
            "state_label": "GZMKhigh_Tex"
          },
          {
            "functional_category": "terminal_exhausted",
            "state_label": "LAG3high_Tex"
          }
        ],
        "cd8_states_enriched_in_tumor_cluster_fraction": [
          {
            "functional_category": "naive_memory",
            "state_label": "Tn"
          },
          {
            "functional_category": "early_effector_memory",
            "state_label": "early_Tem_GZMK"
          },
          {
            "functional_category": "isg_response",
            "state_label": "ISG_pos"
          }
        ],
        "clonotype_expansion_finding": "singlet_modestly_more_expanded_than_cluster"
      },
      "nTurns": 42,
      "checkpointSignals": {
        "clustering": true,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": false,
        "nerveJointAnchor": false,
        "multiomeAssembly": true,
        "depthNormalizedAtac": false,
        "regulonInference": true,
        "cd8SubsetIntegration": true,
        "perPatientEnrichment": true,
        "tcrRichnessMetrics": true,
        "orthologProjection": false,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": true,
        "mousePseudobulkDesign": false,
        "cancerEpithelialDenominator": false,
        "geneLevelDe": true,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "trajectoryFile": "scbench-long/trajectories/10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1__claude-sonnet-4-6__pi__r2.json",
      "runId": "claude-sonnet-4-6__pi__10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1__r2",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-sonnet-4-6-pi",
      "modelName": "claude-sonnet-4-6",
      "harness": "pi",
      "task": "10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": true,
      "score": 15,
      "durationS": 2495.63,
      "cost": 2.9629857000000004,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_singlet_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "TOXhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "LAG3high_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "GZMKhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "intermediate_exhausted_gzmk"
                    ],
                    "canonical": "intermediate_exhausted_gzmk"
                  }
                }
              },
              {
                "key": "TCF7_stem_like_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "stem_like_exhausted"
                    ],
                    "canonical": "stem_like_exhausted"
                  }
                }
              },
              {
                "key": "MKI67_pos_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              },
              {
                "key": "MKI67high_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_tumor_cluster_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "Tn",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "Tn_Tmem",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "early_Tem_GZMK",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              },
              {
                "key": "Tem_GZMH",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              }
            ]
          },
          {
            "name": "clonotype_expansion_direction_data_true",
            "role": "gate",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_substantially_more_expanded_than_cluster",
              "singlet_modestly_more_expanded_than_cluster"
            ]
          },
          {
            "name": "biological_inference_data_true",
            "role": "gate",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_antigen_experienced_cluster_bystander"
            ]
          },
          {
            "name": "singlet_top3_contains_naive_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Tn",
              "Tn_Tmem",
              "early_Tem_GZMK",
              "Tem_GZMH"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "TOXhigh_Tex",
              "LAG3high_Tex",
              "GZMKhigh_Tex",
              "TCF7_stem_like_Tex",
              "MKI67_pos_Tex_Tprol",
              "MKI67high_Tex_Tprol"
            ]
          },
          {
            "name": "singlet_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "TOXhigh_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "TCF7_stem_like_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "MKI67_state_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "Tn_or_Tn_Tmem_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_stemlike_precursor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_reported_canonical_expansion_direction",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_substantially_more_expanded_than_singlet",
              "cluster_modestly_more_expanded_than_singlet"
            ]
          },
          {
            "name": "cf_expansion_lazy_defer",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "expansion_pattern_not_detectable"
            ]
          },
          {
            "name": "cf_reported_canonical_expected_biology",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_antigen_experienced_singlet_bystander"
            ]
          },
          {
            "name": "cf_biological_inference_null_artifact_or_lineage",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "differences_reflect_dissociation_or_sorting_artifact",
              "cluster_and_singlet_capture_same_biology",
              "cluster_and_singlet_capture_distinct_cd8_lineages"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "biological_relations",
            "kind": "dict",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_greater_than",
                  "greater_than"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "less_than",
                  "much_less_than"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than",
                  "approximately_equal"
                ]
              },
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal"
                ]
              },
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal",
                  "greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_exhaustion_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_naive_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_proliferation_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_expansion_richness_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_multichain_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_intermediate_gzmk_overgeneralized",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_melanocytic_conjugate_overcall",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "values": [
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_relations_mass_hedge",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              }
            }
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: PASS\n  + list_match: list_match (answer_field='cd8_states_enriched_in_singlet_fraction'): PASS  (score=3.0)\n  + list_match: list_match (answer_field='cd8_states_enriched_in_tumor_cluster_fraction'): PASS  (score=3.0)\n  + clonotype_expansion_direction_data_true: passed=True, score=1.0\n  + biological_inference_data_true: passed=True, score=1.0\n  + dict_match: dict_match (answer_field='biological_relations'): PASS  (score=7.0)\n  + hard_fail 'singlet_top3_contains_naive_state_prior_pushing': not triggered\n  + hard_fail 'tumor_cluster_top3_contains_exhausted_state_prior_pushing': not triggered\n  + hard_fail 'singlet_top3_contains_decoy_token': not triggered\n  + hard_fail 'tumor_cluster_top3_contains_decoy_token': not triggered\n  + hard_fail 'TOXhigh_Tex_wrong_functional_category': not triggered\n  + hard_fail 'TCF7_stem_like_Tex_wrong_functional_category': not triggered\n  + hard_fail 'MKI67_state_wrong_functional_category': not triggered\n  + hard_fail 'Tn_or_Tn_Tmem_wrong_functional_category': not triggered\n  + hard_fail 'missing_exhausted_or_proliferating_anchor_in_singlet_top3': not triggered\n  + hard_fail 'missing_stemlike_precursor_in_singlet_top3': not triggered\n  + hard_fail 'missing_naive_or_early_memory_anchor_in_tumor_cluster_top3': not triggered\n  + hard_fail 'cf_reported_canonical_expansion_direction': not triggered\n  + hard_fail 'cf_expansion_lazy_defer': not triggered\n  + hard_fail 'cf_reported_canonical_expected_biology': not triggered\n  + hard_fail 'cf_biological_inference_null_artifact_or_lineage': not triggered\n  + hard_fail 'cf_exhaustion_inverted_prior_push': not triggered\n  + hard_fail 'cf_naive_inverted_prior_push': not triggered\n  + hard_fail 'cf_proliferation_inverted_prior_push': not triggered\n  + hard_fail 'cf_expansion_richness_inverted_prior_push': not triggered\n  + hard_fail 'cf_multichain_inverted_prior_push': not triggered\n  + hard_fail 'cf_intermediate_gzmk_overgeneralized': not triggered\n  + hard_fail 'cf_melanocytic_conjugate_overcall': not triggered\n  + hard_fail 'cf_relations_mass_hedge': not triggered",
      "metrics": {
        "hard_fail_triggered": [],
        "pass_rule": "all",
        "scoring_count": 5,
        "scoring_passed": 5,
        "scoring_total_score": 15
      },
      "fieldScores": {
        "biological_inference_data_true": 1,
        "clonotype_expansion_direction_data_true": 1,
        "dict_match": 7,
        "list_match": 3
      },
      "graderChecks": [
        {
          "name": "cd8_states_enriched_in_singlet_fraction",
          "kind": "list_match",
          "passed": true,
          "score": 3
        },
        {
          "name": "cd8_states_enriched_in_tumor_cluster_fraction",
          "kind": "list_match",
          "passed": true,
          "score": 3
        },
        {
          "name": "clonotype_expansion_direction_data_true",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "biological_inference_data_true",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "biological_relations",
          "kind": "list_match",
          "passed": true,
          "score": 7
        },
        {
          "name": "singlet_top3_contains_naive_state_prior_pushing",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "singlet_top3_contains_decoy_token",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "tumor_cluster_top3_contains_decoy_token",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "TOXhigh_Tex_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "TCF7_stem_like_Tex_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "MKI67_state_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "Tn_or_Tn_Tmem_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_stemlike_precursor_in_singlet_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_reported_canonical_expansion_direction",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_expansion_lazy_defer",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_reported_canonical_expected_biology",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_biological_inference_null_artifact_or_lineage",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_exhaustion_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_naive_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_proliferation_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_expansion_richness_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_multichain_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_intermediate_gzmk_overgeneralized",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_melanocytic_conjugate_overcall",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_relations_mass_hedge",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "biological_inference": "singlet_antigen_experienced_cluster_bystander",
        "biological_relations": {
          "clonal_expansion_richness_in_singlets_vs_tumor_cluster": "greater_than",
          "exhaustion_program_in_singlets_vs_tumor_cluster": "greater_than",
          "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": "approximately_equal",
          "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": "greater_than",
          "multichain_tcr_rate_in_singlets_vs_tumor_cluster": "greater_than",
          "naive_memory_program_in_singlets_vs_tumor_cluster": "much_less_than",
          "proliferation_program_in_singlets_vs_tumor_cluster": "greater_than"
        },
        "cd8_states_enriched_in_singlet_fraction": [
          {
            "functional_category": "stem_like_exhausted",
            "state_label": "TCF7_stem_like_Tex"
          },
          {
            "functional_category": "intermediate_exhausted_gzmk",
            "state_label": "GZMKhigh_Tex"
          },
          {
            "functional_category": "terminal_exhausted",
            "state_label": "LAG3high_Tex"
          }
        ],
        "cd8_states_enriched_in_tumor_cluster_fraction": [
          {
            "functional_category": "early_effector_memory",
            "state_label": "Tem_GZMH"
          },
          {
            "functional_category": "early_effector_memory",
            "state_label": "early_Tem_GZMK"
          },
          {
            "functional_category": "naive_memory",
            "state_label": "Tn_Tmem"
          }
        ],
        "clonotype_expansion_finding": "singlet_modestly_more_expanded_than_cluster"
      },
      "nTurns": 33,
      "checkpointSignals": {
        "clustering": true,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": false,
        "nerveJointAnchor": false,
        "multiomeAssembly": false,
        "depthNormalizedAtac": false,
        "regulonInference": false,
        "cd8SubsetIntegration": true,
        "perPatientEnrichment": true,
        "tcrRichnessMetrics": true,
        "orthologProjection": false,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": true,
        "mousePseudobulkDesign": false,
        "cancerEpithelialDenominator": false,
        "geneLevelDe": false,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "trajectoryFile": "scbench-long/trajectories/10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1__claude-sonnet-4-6__pi__r3.json",
      "runId": "claude-sonnet-4-6__pi__10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1__r3",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-gemini-3-1-pro-preview-pi",
      "modelName": "gemini-3.1-pro-preview",
      "harness": "pi",
      "task": "10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 1,
      "passed": false,
      "score": 14,
      "durationS": 850.48,
      "cost": 0.6879344000000002,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_singlet_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "TOXhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "LAG3high_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "GZMKhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "intermediate_exhausted_gzmk"
                    ],
                    "canonical": "intermediate_exhausted_gzmk"
                  }
                }
              },
              {
                "key": "TCF7_stem_like_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "stem_like_exhausted"
                    ],
                    "canonical": "stem_like_exhausted"
                  }
                }
              },
              {
                "key": "MKI67_pos_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              },
              {
                "key": "MKI67high_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_tumor_cluster_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "Tn",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "Tn_Tmem",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "early_Tem_GZMK",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              },
              {
                "key": "Tem_GZMH",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              }
            ]
          },
          {
            "name": "clonotype_expansion_direction_data_true",
            "role": "gate",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_substantially_more_expanded_than_cluster",
              "singlet_modestly_more_expanded_than_cluster"
            ]
          },
          {
            "name": "biological_inference_data_true",
            "role": "gate",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_antigen_experienced_cluster_bystander"
            ]
          },
          {
            "name": "singlet_top3_contains_naive_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Tn",
              "Tn_Tmem",
              "early_Tem_GZMK",
              "Tem_GZMH"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "TOXhigh_Tex",
              "LAG3high_Tex",
              "GZMKhigh_Tex",
              "TCF7_stem_like_Tex",
              "MKI67_pos_Tex_Tprol",
              "MKI67high_Tex_Tprol"
            ]
          },
          {
            "name": "singlet_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "TOXhigh_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "TCF7_stem_like_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "MKI67_state_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "Tn_or_Tn_Tmem_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_stemlike_precursor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_reported_canonical_expansion_direction",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_substantially_more_expanded_than_singlet",
              "cluster_modestly_more_expanded_than_singlet"
            ]
          },
          {
            "name": "cf_expansion_lazy_defer",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "expansion_pattern_not_detectable"
            ]
          },
          {
            "name": "cf_reported_canonical_expected_biology",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_antigen_experienced_singlet_bystander"
            ]
          },
          {
            "name": "cf_biological_inference_null_artifact_or_lineage",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "differences_reflect_dissociation_or_sorting_artifact",
              "cluster_and_singlet_capture_same_biology",
              "cluster_and_singlet_capture_distinct_cd8_lineages"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "biological_relations",
            "kind": "dict",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_greater_than",
                  "greater_than"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "less_than",
                  "much_less_than"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than",
                  "approximately_equal"
                ]
              },
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal"
                ]
              },
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal",
                  "greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_exhaustion_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_naive_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_proliferation_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_expansion_richness_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_multichain_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_intermediate_gzmk_overgeneralized",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_melanocytic_conjugate_overcall",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "values": [
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_relations_mass_hedge",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              }
            }
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: FAIL\n  + list_match: list_match (answer_field='cd8_states_enriched_in_singlet_fraction'): PASS  (score=3.0)\n  + list_match: list_match (answer_field='cd8_states_enriched_in_tumor_cluster_fraction'): PASS  (score=3.0)\n  + clonotype_expansion_direction_data_true: passed=True, score=1.0\n  + biological_inference_data_true: passed=True, score=1.0\n  x dict_match: dict_match (answer_field='biological_relations'): FAIL  (score=6.0)\n  + hard_fail 'singlet_top3_contains_naive_state_prior_pushing': not triggered\n  + hard_fail 'tumor_cluster_top3_contains_exhausted_state_prior_pushing': not triggered\n  + hard_fail 'singlet_top3_contains_decoy_token': not triggered\n  + hard_fail 'tumor_cluster_top3_contains_decoy_token': not triggered\n  + hard_fail 'TOXhigh_Tex_wrong_functional_category': not triggered\n  + hard_fail 'TCF7_stem_like_Tex_wrong_functional_category': not triggered\n  + hard_fail 'MKI67_state_wrong_functional_category': not triggered\n  + hard_fail 'Tn_or_Tn_Tmem_wrong_functional_category': not triggered\n  + hard_fail 'missing_exhausted_or_proliferating_anchor_in_singlet_top3': not triggered\n  + hard_fail 'missing_stemlike_precursor_in_singlet_top3': not triggered\n  + hard_fail 'missing_naive_or_early_memory_anchor_in_tumor_cluster_top3': not triggered\n  + hard_fail 'cf_reported_canonical_expansion_direction': not triggered\n  + hard_fail 'cf_expansion_lazy_defer': not triggered\n  + hard_fail 'cf_reported_canonical_expected_biology': not triggered\n  + hard_fail 'cf_biological_inference_null_artifact_or_lineage': not triggered\n  + hard_fail 'cf_exhaustion_inverted_prior_push': not triggered\n  + hard_fail 'cf_naive_inverted_prior_push': not triggered\n  + hard_fail 'cf_proliferation_inverted_prior_push': not triggered\n  + hard_fail 'cf_expansion_richness_inverted_prior_push': not triggered\n  + hard_fail 'cf_multichain_inverted_prior_push': not triggered\n  x hard_fail 'cf_intermediate_gzmk_overgeneralized': TRIGGERED\n  + hard_fail 'cf_melanocytic_conjugate_overcall': not triggered\n  + hard_fail 'cf_relations_mass_hedge': not triggered",
      "metrics": {
        "hard_fail_triggered": [
          "cf_intermediate_gzmk_overgeneralized"
        ],
        "pass_rule": "all",
        "scoring_count": 5,
        "scoring_passed": 4,
        "scoring_total_score": 14
      },
      "fieldScores": {
        "biological_inference_data_true": 1,
        "clonotype_expansion_direction_data_true": 1,
        "dict_match": 6,
        "list_match": 3
      },
      "graderChecks": [
        {
          "name": "cd8_states_enriched_in_singlet_fraction",
          "kind": "list_match",
          "passed": true,
          "score": 3
        },
        {
          "name": "cd8_states_enriched_in_tumor_cluster_fraction",
          "kind": "list_match",
          "passed": true,
          "score": 3
        },
        {
          "name": "clonotype_expansion_direction_data_true",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "biological_inference_data_true",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "biological_relations",
          "kind": "list_match",
          "passed": false,
          "score": 6
        },
        {
          "name": "singlet_top3_contains_naive_state_prior_pushing",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "singlet_top3_contains_decoy_token",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "tumor_cluster_top3_contains_decoy_token",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "TOXhigh_Tex_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "TCF7_stem_like_Tex_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "MKI67_state_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "Tn_or_Tn_Tmem_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_stemlike_precursor_in_singlet_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_reported_canonical_expansion_direction",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_expansion_lazy_defer",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_reported_canonical_expected_biology",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_biological_inference_null_artifact_or_lineage",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_exhaustion_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_naive_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_proliferation_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_expansion_richness_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_multichain_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_intermediate_gzmk_overgeneralized",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "cf_melanocytic_conjugate_overcall",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_relations_mass_hedge",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "biological_inference": "singlet_antigen_experienced_cluster_bystander",
        "biological_relations": {
          "clonal_expansion_richness_in_singlets_vs_tumor_cluster": "greater_than",
          "exhaustion_program_in_singlets_vs_tumor_cluster": "much_greater_than",
          "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": "greater_than",
          "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": "greater_than",
          "multichain_tcr_rate_in_singlets_vs_tumor_cluster": "greater_than",
          "naive_memory_program_in_singlets_vs_tumor_cluster": "much_less_than",
          "proliferation_program_in_singlets_vs_tumor_cluster": "greater_than"
        },
        "cd8_states_enriched_in_singlet_fraction": [
          {
            "functional_category": "stem_like_exhausted",
            "state_label": "TCF7_stem_like_Tex"
          },
          {
            "functional_category": "intermediate_exhausted_gzmk",
            "state_label": "GZMKhigh_Tex"
          },
          {
            "functional_category": "terminal_exhausted",
            "state_label": "TOXhigh_Tex"
          }
        ],
        "cd8_states_enriched_in_tumor_cluster_fraction": [
          {
            "functional_category": "naive_memory",
            "state_label": "Tn"
          },
          {
            "functional_category": "naive_memory",
            "state_label": "Tn_Tmem"
          },
          {
            "functional_category": "early_effector_memory",
            "state_label": "early_Tem_GZMK"
          }
        ],
        "clonotype_expansion_finding": "singlet_modestly_more_expanded_than_cluster"
      },
      "nTurns": 35,
      "checkpointSignals": {
        "clustering": true,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": false,
        "nerveJointAnchor": false,
        "multiomeAssembly": false,
        "depthNormalizedAtac": false,
        "regulonInference": false,
        "cd8SubsetIntegration": true,
        "perPatientEnrichment": true,
        "tcrRichnessMetrics": true,
        "orthologProjection": false,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": true,
        "mousePseudobulkDesign": false,
        "cancerEpithelialDenominator": false,
        "geneLevelDe": false,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "trajectoryFile": "scbench-long/trajectories/10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1__gemini-3.1-pro-preview__pi__r1.json",
      "runId": "gemini-3.1-pro-preview__pi__10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1__r1",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-gemini-3-1-pro-preview-pi",
      "modelName": "gemini-3.1-pro-preview",
      "harness": "pi",
      "task": "10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 2,
      "passed": true,
      "score": 15,
      "durationS": 963.33,
      "cost": 0.6311628,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_singlet_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "TOXhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "LAG3high_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "GZMKhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "intermediate_exhausted_gzmk"
                    ],
                    "canonical": "intermediate_exhausted_gzmk"
                  }
                }
              },
              {
                "key": "TCF7_stem_like_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "stem_like_exhausted"
                    ],
                    "canonical": "stem_like_exhausted"
                  }
                }
              },
              {
                "key": "MKI67_pos_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              },
              {
                "key": "MKI67high_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_tumor_cluster_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "Tn",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "Tn_Tmem",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "early_Tem_GZMK",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              },
              {
                "key": "Tem_GZMH",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              }
            ]
          },
          {
            "name": "clonotype_expansion_direction_data_true",
            "role": "gate",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_substantially_more_expanded_than_cluster",
              "singlet_modestly_more_expanded_than_cluster"
            ]
          },
          {
            "name": "biological_inference_data_true",
            "role": "gate",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_antigen_experienced_cluster_bystander"
            ]
          },
          {
            "name": "singlet_top3_contains_naive_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Tn",
              "Tn_Tmem",
              "early_Tem_GZMK",
              "Tem_GZMH"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "TOXhigh_Tex",
              "LAG3high_Tex",
              "GZMKhigh_Tex",
              "TCF7_stem_like_Tex",
              "MKI67_pos_Tex_Tprol",
              "MKI67high_Tex_Tprol"
            ]
          },
          {
            "name": "singlet_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "TOXhigh_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "TCF7_stem_like_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "MKI67_state_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "Tn_or_Tn_Tmem_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_stemlike_precursor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_reported_canonical_expansion_direction",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_substantially_more_expanded_than_singlet",
              "cluster_modestly_more_expanded_than_singlet"
            ]
          },
          {
            "name": "cf_expansion_lazy_defer",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "expansion_pattern_not_detectable"
            ]
          },
          {
            "name": "cf_reported_canonical_expected_biology",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_antigen_experienced_singlet_bystander"
            ]
          },
          {
            "name": "cf_biological_inference_null_artifact_or_lineage",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "differences_reflect_dissociation_or_sorting_artifact",
              "cluster_and_singlet_capture_same_biology",
              "cluster_and_singlet_capture_distinct_cd8_lineages"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "biological_relations",
            "kind": "dict",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_greater_than",
                  "greater_than"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "less_than",
                  "much_less_than"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than",
                  "approximately_equal"
                ]
              },
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal"
                ]
              },
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal",
                  "greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_exhaustion_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_naive_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_proliferation_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_expansion_richness_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_multichain_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_intermediate_gzmk_overgeneralized",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_melanocytic_conjugate_overcall",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "values": [
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_relations_mass_hedge",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              }
            }
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: PASS\n  + list_match: list_match (answer_field='cd8_states_enriched_in_singlet_fraction'): PASS  (score=3.0)\n  + list_match: list_match (answer_field='cd8_states_enriched_in_tumor_cluster_fraction'): PASS  (score=3.0)\n  + clonotype_expansion_direction_data_true: passed=True, score=1.0\n  + biological_inference_data_true: passed=True, score=1.0\n  + dict_match: dict_match (answer_field='biological_relations'): PASS  (score=7.0)\n  + hard_fail 'singlet_top3_contains_naive_state_prior_pushing': not triggered\n  + hard_fail 'tumor_cluster_top3_contains_exhausted_state_prior_pushing': not triggered\n  + hard_fail 'singlet_top3_contains_decoy_token': not triggered\n  + hard_fail 'tumor_cluster_top3_contains_decoy_token': not triggered\n  + hard_fail 'TOXhigh_Tex_wrong_functional_category': not triggered\n  + hard_fail 'TCF7_stem_like_Tex_wrong_functional_category': not triggered\n  + hard_fail 'MKI67_state_wrong_functional_category': not triggered\n  + hard_fail 'Tn_or_Tn_Tmem_wrong_functional_category': not triggered\n  + hard_fail 'missing_exhausted_or_proliferating_anchor_in_singlet_top3': not triggered\n  + hard_fail 'missing_stemlike_precursor_in_singlet_top3': not triggered\n  + hard_fail 'missing_naive_or_early_memory_anchor_in_tumor_cluster_top3': not triggered\n  + hard_fail 'cf_reported_canonical_expansion_direction': not triggered\n  + hard_fail 'cf_expansion_lazy_defer': not triggered\n  + hard_fail 'cf_reported_canonical_expected_biology': not triggered\n  + hard_fail 'cf_biological_inference_null_artifact_or_lineage': not triggered\n  + hard_fail 'cf_exhaustion_inverted_prior_push': not triggered\n  + hard_fail 'cf_naive_inverted_prior_push': not triggered\n  + hard_fail 'cf_proliferation_inverted_prior_push': not triggered\n  + hard_fail 'cf_expansion_richness_inverted_prior_push': not triggered\n  + hard_fail 'cf_multichain_inverted_prior_push': not triggered\n  + hard_fail 'cf_intermediate_gzmk_overgeneralized': not triggered\n  + hard_fail 'cf_melanocytic_conjugate_overcall': not triggered\n  + hard_fail 'cf_relations_mass_hedge': not triggered",
      "metrics": {
        "hard_fail_triggered": [],
        "pass_rule": "all",
        "scoring_count": 5,
        "scoring_passed": 5,
        "scoring_total_score": 15
      },
      "fieldScores": {
        "biological_inference_data_true": 1,
        "clonotype_expansion_direction_data_true": 1,
        "dict_match": 7,
        "list_match": 3
      },
      "graderChecks": [
        {
          "name": "cd8_states_enriched_in_singlet_fraction",
          "kind": "list_match",
          "passed": true,
          "score": 3
        },
        {
          "name": "cd8_states_enriched_in_tumor_cluster_fraction",
          "kind": "list_match",
          "passed": true,
          "score": 3
        },
        {
          "name": "clonotype_expansion_direction_data_true",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "biological_inference_data_true",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "biological_relations",
          "kind": "list_match",
          "passed": true,
          "score": 7
        },
        {
          "name": "singlet_top3_contains_naive_state_prior_pushing",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "singlet_top3_contains_decoy_token",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "tumor_cluster_top3_contains_decoy_token",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "TOXhigh_Tex_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "TCF7_stem_like_Tex_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "MKI67_state_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "Tn_or_Tn_Tmem_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_stemlike_precursor_in_singlet_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_reported_canonical_expansion_direction",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_expansion_lazy_defer",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_reported_canonical_expected_biology",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_biological_inference_null_artifact_or_lineage",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_exhaustion_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_naive_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_proliferation_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_expansion_richness_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_multichain_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_intermediate_gzmk_overgeneralized",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_melanocytic_conjugate_overcall",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_relations_mass_hedge",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "biological_inference": "singlet_antigen_experienced_cluster_bystander",
        "biological_relations": {
          "clonal_expansion_richness_in_singlets_vs_tumor_cluster": "greater_than",
          "exhaustion_program_in_singlets_vs_tumor_cluster": "much_greater_than",
          "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": "approximately_equal",
          "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": "greater_than",
          "multichain_tcr_rate_in_singlets_vs_tumor_cluster": "greater_than",
          "naive_memory_program_in_singlets_vs_tumor_cluster": "much_less_than",
          "proliferation_program_in_singlets_vs_tumor_cluster": "greater_than"
        },
        "cd8_states_enriched_in_singlet_fraction": [
          {
            "functional_category": "stem_like_exhausted",
            "state_label": "TCF7_stem_like_Tex"
          },
          {
            "functional_category": "intermediate_exhausted_gzmk",
            "state_label": "GZMKhigh_Tex"
          },
          {
            "functional_category": "terminal_exhausted",
            "state_label": "TOXhigh_Tex"
          }
        ],
        "cd8_states_enriched_in_tumor_cluster_fraction": [
          {
            "functional_category": "naive_memory",
            "state_label": "Tn"
          },
          {
            "functional_category": "naive_memory",
            "state_label": "Tn_Tmem"
          },
          {
            "functional_category": "early_effector_memory",
            "state_label": "early_Tem_GZMK"
          }
        ],
        "clonotype_expansion_finding": "singlet_modestly_more_expanded_than_cluster"
      },
      "nTurns": 42,
      "checkpointSignals": {
        "clustering": true,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": false,
        "nerveJointAnchor": false,
        "multiomeAssembly": false,
        "depthNormalizedAtac": false,
        "regulonInference": false,
        "cd8SubsetIntegration": true,
        "perPatientEnrichment": true,
        "tcrRichnessMetrics": true,
        "orthologProjection": false,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": true,
        "mousePseudobulkDesign": false,
        "cancerEpithelialDenominator": true,
        "geneLevelDe": false,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "trajectoryFile": "scbench-long/trajectories/10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1__gemini-3.1-pro-preview__pi__r2.json",
      "runId": "gemini-3.1-pro-preview__pi__10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1__r2",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-gemini-3-1-pro-preview-pi",
      "modelName": "gemini-3.1-pro-preview",
      "harness": "pi",
      "task": "10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": false,
      "score": 13,
      "durationS": 862.73,
      "cost": 0.8741769999999999,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_singlet_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "TOXhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "LAG3high_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "GZMKhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "intermediate_exhausted_gzmk"
                    ],
                    "canonical": "intermediate_exhausted_gzmk"
                  }
                }
              },
              {
                "key": "TCF7_stem_like_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "stem_like_exhausted"
                    ],
                    "canonical": "stem_like_exhausted"
                  }
                }
              },
              {
                "key": "MKI67_pos_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              },
              {
                "key": "MKI67high_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_tumor_cluster_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "Tn",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "Tn_Tmem",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "early_Tem_GZMK",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              },
              {
                "key": "Tem_GZMH",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              }
            ]
          },
          {
            "name": "clonotype_expansion_direction_data_true",
            "role": "gate",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_substantially_more_expanded_than_cluster",
              "singlet_modestly_more_expanded_than_cluster"
            ]
          },
          {
            "name": "biological_inference_data_true",
            "role": "gate",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_antigen_experienced_cluster_bystander"
            ]
          },
          {
            "name": "singlet_top3_contains_naive_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Tn",
              "Tn_Tmem",
              "early_Tem_GZMK",
              "Tem_GZMH"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "TOXhigh_Tex",
              "LAG3high_Tex",
              "GZMKhigh_Tex",
              "TCF7_stem_like_Tex",
              "MKI67_pos_Tex_Tprol",
              "MKI67high_Tex_Tprol"
            ]
          },
          {
            "name": "singlet_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "TOXhigh_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "TCF7_stem_like_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "MKI67_state_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "Tn_or_Tn_Tmem_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_stemlike_precursor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_reported_canonical_expansion_direction",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_substantially_more_expanded_than_singlet",
              "cluster_modestly_more_expanded_than_singlet"
            ]
          },
          {
            "name": "cf_expansion_lazy_defer",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "expansion_pattern_not_detectable"
            ]
          },
          {
            "name": "cf_reported_canonical_expected_biology",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_antigen_experienced_singlet_bystander"
            ]
          },
          {
            "name": "cf_biological_inference_null_artifact_or_lineage",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "differences_reflect_dissociation_or_sorting_artifact",
              "cluster_and_singlet_capture_same_biology",
              "cluster_and_singlet_capture_distinct_cd8_lineages"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "biological_relations",
            "kind": "dict",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_greater_than",
                  "greater_than"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "less_than",
                  "much_less_than"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than",
                  "approximately_equal"
                ]
              },
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal"
                ]
              },
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal",
                  "greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_exhaustion_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_naive_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_proliferation_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_expansion_richness_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_multichain_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_intermediate_gzmk_overgeneralized",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_melanocytic_conjugate_overcall",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "values": [
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_relations_mass_hedge",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              }
            }
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: FAIL\n  + list_match: list_match (answer_field='cd8_states_enriched_in_singlet_fraction'): PASS  (score=3.0)\n  + list_match: list_match (answer_field='cd8_states_enriched_in_tumor_cluster_fraction'): PASS  (score=3.0)\n  + clonotype_expansion_direction_data_true: passed=True, score=1.0\n  + biological_inference_data_true: passed=True, score=1.0\n  x dict_match: dict_match (answer_field='biological_relations'): FAIL  (score=5.0)\n  + hard_fail 'singlet_top3_contains_naive_state_prior_pushing': not triggered\n  + hard_fail 'tumor_cluster_top3_contains_exhausted_state_prior_pushing': not triggered\n  + hard_fail 'singlet_top3_contains_decoy_token': not triggered\n  + hard_fail 'tumor_cluster_top3_contains_decoy_token': not triggered\n  + hard_fail 'TOXhigh_Tex_wrong_functional_category': not triggered\n  + hard_fail 'TCF7_stem_like_Tex_wrong_functional_category': not triggered\n  + hard_fail 'MKI67_state_wrong_functional_category': not triggered\n  + hard_fail 'Tn_or_Tn_Tmem_wrong_functional_category': not triggered\n  + hard_fail 'missing_exhausted_or_proliferating_anchor_in_singlet_top3': not triggered\n  + hard_fail 'missing_stemlike_precursor_in_singlet_top3': not triggered\n  + hard_fail 'missing_naive_or_early_memory_anchor_in_tumor_cluster_top3': not triggered\n  + hard_fail 'cf_reported_canonical_expansion_direction': not triggered\n  + hard_fail 'cf_expansion_lazy_defer': not triggered\n  + hard_fail 'cf_reported_canonical_expected_biology': not triggered\n  + hard_fail 'cf_biological_inference_null_artifact_or_lineage': not triggered\n  + hard_fail 'cf_exhaustion_inverted_prior_push': not triggered\n  + hard_fail 'cf_naive_inverted_prior_push': not triggered\n  + hard_fail 'cf_proliferation_inverted_prior_push': not triggered\n  + hard_fail 'cf_expansion_richness_inverted_prior_push': not triggered\n  + hard_fail 'cf_multichain_inverted_prior_push': not triggered\n  x hard_fail 'cf_intermediate_gzmk_overgeneralized': TRIGGERED\n  + hard_fail 'cf_melanocytic_conjugate_overcall': not triggered\n  + hard_fail 'cf_relations_mass_hedge': not triggered",
      "metrics": {
        "hard_fail_triggered": [
          "cf_intermediate_gzmk_overgeneralized"
        ],
        "pass_rule": "all",
        "scoring_count": 5,
        "scoring_passed": 4,
        "scoring_total_score": 13
      },
      "fieldScores": {
        "biological_inference_data_true": 1,
        "clonotype_expansion_direction_data_true": 1,
        "dict_match": 5,
        "list_match": 3
      },
      "graderChecks": [
        {
          "name": "cd8_states_enriched_in_singlet_fraction",
          "kind": "list_match",
          "passed": true,
          "score": 3
        },
        {
          "name": "cd8_states_enriched_in_tumor_cluster_fraction",
          "kind": "list_match",
          "passed": true,
          "score": 3
        },
        {
          "name": "clonotype_expansion_direction_data_true",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "biological_inference_data_true",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "biological_relations",
          "kind": "list_match",
          "passed": false,
          "score": 5
        },
        {
          "name": "singlet_top3_contains_naive_state_prior_pushing",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "singlet_top3_contains_decoy_token",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "tumor_cluster_top3_contains_decoy_token",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "TOXhigh_Tex_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "TCF7_stem_like_Tex_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "MKI67_state_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "Tn_or_Tn_Tmem_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_stemlike_precursor_in_singlet_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_reported_canonical_expansion_direction",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_expansion_lazy_defer",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_reported_canonical_expected_biology",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_biological_inference_null_artifact_or_lineage",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_exhaustion_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_naive_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_proliferation_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_expansion_richness_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_multichain_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_intermediate_gzmk_overgeneralized",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "cf_melanocytic_conjugate_overcall",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_relations_mass_hedge",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "biological_inference": "singlet_antigen_experienced_cluster_bystander",
        "biological_relations": {
          "clonal_expansion_richness_in_singlets_vs_tumor_cluster": "greater_than",
          "exhaustion_program_in_singlets_vs_tumor_cluster": "greater_than",
          "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": "greater_than",
          "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": "approximately_equal",
          "multichain_tcr_rate_in_singlets_vs_tumor_cluster": "greater_than",
          "naive_memory_program_in_singlets_vs_tumor_cluster": "less_than",
          "proliferation_program_in_singlets_vs_tumor_cluster": "approximately_equal"
        },
        "cd8_states_enriched_in_singlet_fraction": [
          {
            "functional_category": "stem_like_exhausted",
            "state_label": "TCF7_stem_like_Tex"
          },
          {
            "functional_category": "intermediate_exhausted_gzmk",
            "state_label": "GZMKhigh_Tex"
          },
          {
            "functional_category": "terminal_exhausted",
            "state_label": "TOXhigh_Tex"
          }
        ],
        "cd8_states_enriched_in_tumor_cluster_fraction": [
          {
            "functional_category": "naive_memory",
            "state_label": "Tn"
          },
          {
            "functional_category": "naive_memory",
            "state_label": "Tn_Tmem"
          },
          {
            "functional_category": "early_effector_memory",
            "state_label": "early_Tem_GZMK"
          }
        ],
        "clonotype_expansion_finding": "singlet_modestly_more_expanded_than_cluster"
      },
      "nTurns": 43,
      "checkpointSignals": {
        "clustering": true,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": false,
        "nerveJointAnchor": false,
        "multiomeAssembly": false,
        "depthNormalizedAtac": false,
        "regulonInference": false,
        "cd8SubsetIntegration": true,
        "perPatientEnrichment": true,
        "tcrRichnessMetrics": true,
        "orthologProjection": false,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": true,
        "mousePseudobulkDesign": false,
        "cancerEpithelialDenominator": false,
        "geneLevelDe": false,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "trajectoryFile": "scbench-long/trajectories/10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1__gemini-3.1-pro-preview__pi__r3.json",
      "runId": "gemini-3.1-pro-preview__pi__10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1__r3",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-gemini-3-5-flash-pi",
      "modelName": "gemini-3.5-flash",
      "harness": "pi",
      "task": "10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 1,
      "passed": true,
      "score": 15,
      "durationS": 890.56,
      "cost": 1.44600735,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_singlet_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "TOXhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "LAG3high_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "GZMKhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "intermediate_exhausted_gzmk"
                    ],
                    "canonical": "intermediate_exhausted_gzmk"
                  }
                }
              },
              {
                "key": "TCF7_stem_like_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "stem_like_exhausted"
                    ],
                    "canonical": "stem_like_exhausted"
                  }
                }
              },
              {
                "key": "MKI67_pos_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              },
              {
                "key": "MKI67high_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_tumor_cluster_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "Tn",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "Tn_Tmem",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "early_Tem_GZMK",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              },
              {
                "key": "Tem_GZMH",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              }
            ]
          },
          {
            "name": "clonotype_expansion_direction_data_true",
            "role": "gate",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_substantially_more_expanded_than_cluster",
              "singlet_modestly_more_expanded_than_cluster"
            ]
          },
          {
            "name": "biological_inference_data_true",
            "role": "gate",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_antigen_experienced_cluster_bystander"
            ]
          },
          {
            "name": "singlet_top3_contains_naive_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Tn",
              "Tn_Tmem",
              "early_Tem_GZMK",
              "Tem_GZMH"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "TOXhigh_Tex",
              "LAG3high_Tex",
              "GZMKhigh_Tex",
              "TCF7_stem_like_Tex",
              "MKI67_pos_Tex_Tprol",
              "MKI67high_Tex_Tprol"
            ]
          },
          {
            "name": "singlet_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "TOXhigh_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "TCF7_stem_like_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "MKI67_state_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "Tn_or_Tn_Tmem_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_stemlike_precursor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_reported_canonical_expansion_direction",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_substantially_more_expanded_than_singlet",
              "cluster_modestly_more_expanded_than_singlet"
            ]
          },
          {
            "name": "cf_expansion_lazy_defer",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "expansion_pattern_not_detectable"
            ]
          },
          {
            "name": "cf_reported_canonical_expected_biology",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_antigen_experienced_singlet_bystander"
            ]
          },
          {
            "name": "cf_biological_inference_null_artifact_or_lineage",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "differences_reflect_dissociation_or_sorting_artifact",
              "cluster_and_singlet_capture_same_biology",
              "cluster_and_singlet_capture_distinct_cd8_lineages"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "biological_relations",
            "kind": "dict",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_greater_than",
                  "greater_than"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "less_than",
                  "much_less_than"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than",
                  "approximately_equal"
                ]
              },
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal"
                ]
              },
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal",
                  "greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_exhaustion_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_naive_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_proliferation_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_expansion_richness_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_multichain_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_intermediate_gzmk_overgeneralized",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_melanocytic_conjugate_overcall",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "values": [
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_relations_mass_hedge",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              }
            }
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: PASS\n  + list_match: list_match (answer_field='cd8_states_enriched_in_singlet_fraction'): PASS  (score=3.0)\n  + list_match: list_match (answer_field='cd8_states_enriched_in_tumor_cluster_fraction'): PASS  (score=3.0)\n  + clonotype_expansion_direction_data_true: passed=True, score=1.0\n  + biological_inference_data_true: passed=True, score=1.0\n  + dict_match: dict_match (answer_field='biological_relations'): PASS  (score=7.0)\n  + hard_fail 'singlet_top3_contains_naive_state_prior_pushing': not triggered\n  + hard_fail 'tumor_cluster_top3_contains_exhausted_state_prior_pushing': not triggered\n  + hard_fail 'singlet_top3_contains_decoy_token': not triggered\n  + hard_fail 'tumor_cluster_top3_contains_decoy_token': not triggered\n  + hard_fail 'TOXhigh_Tex_wrong_functional_category': not triggered\n  + hard_fail 'TCF7_stem_like_Tex_wrong_functional_category': not triggered\n  + hard_fail 'MKI67_state_wrong_functional_category': not triggered\n  + hard_fail 'Tn_or_Tn_Tmem_wrong_functional_category': not triggered\n  + hard_fail 'missing_exhausted_or_proliferating_anchor_in_singlet_top3': not triggered\n  + hard_fail 'missing_stemlike_precursor_in_singlet_top3': not triggered\n  + hard_fail 'missing_naive_or_early_memory_anchor_in_tumor_cluster_top3': not triggered\n  + hard_fail 'cf_reported_canonical_expansion_direction': not triggered\n  + hard_fail 'cf_expansion_lazy_defer': not triggered\n  + hard_fail 'cf_reported_canonical_expected_biology': not triggered\n  + hard_fail 'cf_biological_inference_null_artifact_or_lineage': not triggered\n  + hard_fail 'cf_exhaustion_inverted_prior_push': not triggered\n  + hard_fail 'cf_naive_inverted_prior_push': not triggered\n  + hard_fail 'cf_proliferation_inverted_prior_push': not triggered\n  + hard_fail 'cf_expansion_richness_inverted_prior_push': not triggered\n  + hard_fail 'cf_multichain_inverted_prior_push': not triggered\n  + hard_fail 'cf_intermediate_gzmk_overgeneralized': not triggered\n  + hard_fail 'cf_melanocytic_conjugate_overcall': not triggered\n  + hard_fail 'cf_relations_mass_hedge': not triggered",
      "metrics": {
        "hard_fail_triggered": [],
        "pass_rule": "all",
        "scoring_count": 5,
        "scoring_passed": 5,
        "scoring_total_score": 15
      },
      "fieldScores": {
        "biological_inference_data_true": 1,
        "clonotype_expansion_direction_data_true": 1,
        "dict_match": 7,
        "list_match": 3
      },
      "graderChecks": [
        {
          "name": "cd8_states_enriched_in_singlet_fraction",
          "kind": "list_match",
          "passed": true,
          "score": 3
        },
        {
          "name": "cd8_states_enriched_in_tumor_cluster_fraction",
          "kind": "list_match",
          "passed": true,
          "score": 3
        },
        {
          "name": "clonotype_expansion_direction_data_true",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "biological_inference_data_true",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "biological_relations",
          "kind": "list_match",
          "passed": true,
          "score": 7
        },
        {
          "name": "singlet_top3_contains_naive_state_prior_pushing",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "singlet_top3_contains_decoy_token",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "tumor_cluster_top3_contains_decoy_token",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "TOXhigh_Tex_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "TCF7_stem_like_Tex_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "MKI67_state_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "Tn_or_Tn_Tmem_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_stemlike_precursor_in_singlet_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_reported_canonical_expansion_direction",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_expansion_lazy_defer",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_reported_canonical_expected_biology",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_biological_inference_null_artifact_or_lineage",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_exhaustion_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_naive_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_proliferation_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_expansion_richness_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_multichain_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_intermediate_gzmk_overgeneralized",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_melanocytic_conjugate_overcall",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_relations_mass_hedge",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "biological_inference": "singlet_antigen_experienced_cluster_bystander",
        "biological_relations": {
          "clonal_expansion_richness_in_singlets_vs_tumor_cluster": "greater_than",
          "exhaustion_program_in_singlets_vs_tumor_cluster": "greater_than",
          "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": "approximately_equal",
          "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": "approximately_equal",
          "multichain_tcr_rate_in_singlets_vs_tumor_cluster": "greater_than",
          "naive_memory_program_in_singlets_vs_tumor_cluster": "less_than",
          "proliferation_program_in_singlets_vs_tumor_cluster": "greater_than"
        },
        "cd8_states_enriched_in_singlet_fraction": [
          {
            "functional_category": "stem_like_exhausted",
            "state_label": "TCF7_stem_like_Tex"
          },
          {
            "functional_category": "intermediate_exhausted_gzmk",
            "state_label": "GZMKhigh_Tex"
          },
          {
            "functional_category": "terminal_exhausted",
            "state_label": "TOXhigh_Tex"
          }
        ],
        "cd8_states_enriched_in_tumor_cluster_fraction": [
          {
            "functional_category": "naive_memory",
            "state_label": "Tn"
          },
          {
            "functional_category": "naive_memory",
            "state_label": "Tn_Tmem"
          },
          {
            "functional_category": "early_effector_memory",
            "state_label": "early_Tem_GZMK"
          }
        ],
        "clonotype_expansion_finding": "singlet_substantially_more_expanded_than_cluster"
      },
      "nTurns": 51,
      "checkpointSignals": {
        "clustering": true,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": false,
        "nerveJointAnchor": false,
        "multiomeAssembly": false,
        "depthNormalizedAtac": false,
        "regulonInference": false,
        "cd8SubsetIntegration": true,
        "perPatientEnrichment": true,
        "tcrRichnessMetrics": true,
        "orthologProjection": false,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": true,
        "mousePseudobulkDesign": false,
        "cancerEpithelialDenominator": false,
        "geneLevelDe": false,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "trajectoryFile": "scbench-long/trajectories/10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1__gemini-3.5-flash__pi__r1.json",
      "runId": "gemini-3.5-flash__pi__10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1__r1",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-gemini-3-5-flash-pi",
      "modelName": "gemini-3.5-flash",
      "harness": "pi",
      "task": "10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 2,
      "passed": false,
      "score": 14,
      "durationS": 958.44,
      "cost": 1.4022025500000002,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_singlet_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "TOXhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "LAG3high_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "GZMKhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "intermediate_exhausted_gzmk"
                    ],
                    "canonical": "intermediate_exhausted_gzmk"
                  }
                }
              },
              {
                "key": "TCF7_stem_like_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "stem_like_exhausted"
                    ],
                    "canonical": "stem_like_exhausted"
                  }
                }
              },
              {
                "key": "MKI67_pos_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              },
              {
                "key": "MKI67high_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_tumor_cluster_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "Tn",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "Tn_Tmem",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "early_Tem_GZMK",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              },
              {
                "key": "Tem_GZMH",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              }
            ]
          },
          {
            "name": "clonotype_expansion_direction_data_true",
            "role": "gate",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_substantially_more_expanded_than_cluster",
              "singlet_modestly_more_expanded_than_cluster"
            ]
          },
          {
            "name": "biological_inference_data_true",
            "role": "gate",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_antigen_experienced_cluster_bystander"
            ]
          },
          {
            "name": "singlet_top3_contains_naive_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Tn",
              "Tn_Tmem",
              "early_Tem_GZMK",
              "Tem_GZMH"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "TOXhigh_Tex",
              "LAG3high_Tex",
              "GZMKhigh_Tex",
              "TCF7_stem_like_Tex",
              "MKI67_pos_Tex_Tprol",
              "MKI67high_Tex_Tprol"
            ]
          },
          {
            "name": "singlet_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "TOXhigh_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "TCF7_stem_like_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "MKI67_state_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "Tn_or_Tn_Tmem_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_stemlike_precursor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_reported_canonical_expansion_direction",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_substantially_more_expanded_than_singlet",
              "cluster_modestly_more_expanded_than_singlet"
            ]
          },
          {
            "name": "cf_expansion_lazy_defer",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "expansion_pattern_not_detectable"
            ]
          },
          {
            "name": "cf_reported_canonical_expected_biology",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_antigen_experienced_singlet_bystander"
            ]
          },
          {
            "name": "cf_biological_inference_null_artifact_or_lineage",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "differences_reflect_dissociation_or_sorting_artifact",
              "cluster_and_singlet_capture_same_biology",
              "cluster_and_singlet_capture_distinct_cd8_lineages"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "biological_relations",
            "kind": "dict",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_greater_than",
                  "greater_than"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "less_than",
                  "much_less_than"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than",
                  "approximately_equal"
                ]
              },
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal"
                ]
              },
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal",
                  "greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_exhaustion_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_naive_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_proliferation_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_expansion_richness_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_multichain_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_intermediate_gzmk_overgeneralized",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_melanocytic_conjugate_overcall",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "values": [
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_relations_mass_hedge",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              }
            }
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: FAIL\n  + list_match: list_match (answer_field='cd8_states_enriched_in_singlet_fraction'): PASS  (score=3.0)\n  + list_match: list_match (answer_field='cd8_states_enriched_in_tumor_cluster_fraction'): PASS  (score=3.0)\n  + clonotype_expansion_direction_data_true: passed=True, score=1.0\n  + biological_inference_data_true: passed=True, score=1.0\n  x dict_match: dict_match (answer_field='biological_relations'): FAIL  (score=6.0)\n  + hard_fail 'singlet_top3_contains_naive_state_prior_pushing': not triggered\n  + hard_fail 'tumor_cluster_top3_contains_exhausted_state_prior_pushing': not triggered\n  + hard_fail 'singlet_top3_contains_decoy_token': not triggered\n  + hard_fail 'tumor_cluster_top3_contains_decoy_token': not triggered\n  + hard_fail 'TOXhigh_Tex_wrong_functional_category': not triggered\n  + hard_fail 'TCF7_stem_like_Tex_wrong_functional_category': not triggered\n  + hard_fail 'MKI67_state_wrong_functional_category': not triggered\n  + hard_fail 'Tn_or_Tn_Tmem_wrong_functional_category': not triggered\n  + hard_fail 'missing_exhausted_or_proliferating_anchor_in_singlet_top3': not triggered\n  + hard_fail 'missing_stemlike_precursor_in_singlet_top3': not triggered\n  + hard_fail 'missing_naive_or_early_memory_anchor_in_tumor_cluster_top3': not triggered\n  + hard_fail 'cf_reported_canonical_expansion_direction': not triggered\n  + hard_fail 'cf_expansion_lazy_defer': not triggered\n  + hard_fail 'cf_reported_canonical_expected_biology': not triggered\n  + hard_fail 'cf_biological_inference_null_artifact_or_lineage': not triggered\n  + hard_fail 'cf_exhaustion_inverted_prior_push': not triggered\n  + hard_fail 'cf_naive_inverted_prior_push': not triggered\n  + hard_fail 'cf_proliferation_inverted_prior_push': not triggered\n  + hard_fail 'cf_expansion_richness_inverted_prior_push': not triggered\n  + hard_fail 'cf_multichain_inverted_prior_push': not triggered\n  x hard_fail 'cf_intermediate_gzmk_overgeneralized': TRIGGERED\n  + hard_fail 'cf_melanocytic_conjugate_overcall': not triggered\n  + hard_fail 'cf_relations_mass_hedge': not triggered",
      "metrics": {
        "hard_fail_triggered": [
          "cf_intermediate_gzmk_overgeneralized"
        ],
        "pass_rule": "all",
        "scoring_count": 5,
        "scoring_passed": 4,
        "scoring_total_score": 14
      },
      "fieldScores": {
        "biological_inference_data_true": 1,
        "clonotype_expansion_direction_data_true": 1,
        "dict_match": 6,
        "list_match": 3
      },
      "graderChecks": [
        {
          "name": "cd8_states_enriched_in_singlet_fraction",
          "kind": "list_match",
          "passed": true,
          "score": 3
        },
        {
          "name": "cd8_states_enriched_in_tumor_cluster_fraction",
          "kind": "list_match",
          "passed": true,
          "score": 3
        },
        {
          "name": "clonotype_expansion_direction_data_true",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "biological_inference_data_true",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "biological_relations",
          "kind": "list_match",
          "passed": false,
          "score": 6
        },
        {
          "name": "singlet_top3_contains_naive_state_prior_pushing",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "singlet_top3_contains_decoy_token",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "tumor_cluster_top3_contains_decoy_token",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "TOXhigh_Tex_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "TCF7_stem_like_Tex_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "MKI67_state_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "Tn_or_Tn_Tmem_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_stemlike_precursor_in_singlet_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_reported_canonical_expansion_direction",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_expansion_lazy_defer",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_reported_canonical_expected_biology",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_biological_inference_null_artifact_or_lineage",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_exhaustion_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_naive_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_proliferation_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_expansion_richness_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_multichain_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_intermediate_gzmk_overgeneralized",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "cf_melanocytic_conjugate_overcall",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_relations_mass_hedge",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "biological_inference": "singlet_antigen_experienced_cluster_bystander",
        "biological_relations": {
          "clonal_expansion_richness_in_singlets_vs_tumor_cluster": "much_greater_than",
          "exhaustion_program_in_singlets_vs_tumor_cluster": "much_greater_than",
          "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": "greater_than",
          "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": "greater_than",
          "multichain_tcr_rate_in_singlets_vs_tumor_cluster": "greater_than",
          "naive_memory_program_in_singlets_vs_tumor_cluster": "much_less_than",
          "proliferation_program_in_singlets_vs_tumor_cluster": "greater_than"
        },
        "cd8_states_enriched_in_singlet_fraction": [
          {
            "functional_category": "stem_like_exhausted",
            "state_label": "TCF7_stem_like_Tex"
          },
          {
            "functional_category": "intermediate_exhausted_gzmk",
            "state_label": "GZMKhigh_Tex"
          },
          {
            "functional_category": "terminal_exhausted",
            "state_label": "TOXhigh_Tex"
          }
        ],
        "cd8_states_enriched_in_tumor_cluster_fraction": [
          {
            "functional_category": "naive_memory",
            "state_label": "Tn"
          },
          {
            "functional_category": "early_effector_memory",
            "state_label": "early_Tem_GZMK"
          },
          {
            "functional_category": "early_effector_memory",
            "state_label": "Tem_GZMH"
          }
        ],
        "clonotype_expansion_finding": "singlet_substantially_more_expanded_than_cluster"
      },
      "nTurns": 46,
      "checkpointSignals": {
        "clustering": true,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": false,
        "nerveJointAnchor": false,
        "multiomeAssembly": false,
        "depthNormalizedAtac": false,
        "regulonInference": false,
        "cd8SubsetIntegration": true,
        "perPatientEnrichment": true,
        "tcrRichnessMetrics": true,
        "orthologProjection": false,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": true,
        "mousePseudobulkDesign": false,
        "cancerEpithelialDenominator": false,
        "geneLevelDe": false,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "trajectoryFile": "scbench-long/trajectories/10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1__gemini-3.5-flash__pi__r2.json",
      "runId": "gemini-3.5-flash__pi__10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1__r2",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-gemini-3-5-flash-pi",
      "modelName": "gemini-3.5-flash",
      "harness": "pi",
      "task": "10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": false,
      "score": 14,
      "durationS": 2016.96,
      "cost": 2.0434642500000004,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_singlet_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "TOXhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "LAG3high_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "GZMKhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "intermediate_exhausted_gzmk"
                    ],
                    "canonical": "intermediate_exhausted_gzmk"
                  }
                }
              },
              {
                "key": "TCF7_stem_like_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "stem_like_exhausted"
                    ],
                    "canonical": "stem_like_exhausted"
                  }
                }
              },
              {
                "key": "MKI67_pos_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              },
              {
                "key": "MKI67high_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_tumor_cluster_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "Tn",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "Tn_Tmem",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "early_Tem_GZMK",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              },
              {
                "key": "Tem_GZMH",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              }
            ]
          },
          {
            "name": "clonotype_expansion_direction_data_true",
            "role": "gate",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_substantially_more_expanded_than_cluster",
              "singlet_modestly_more_expanded_than_cluster"
            ]
          },
          {
            "name": "biological_inference_data_true",
            "role": "gate",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_antigen_experienced_cluster_bystander"
            ]
          },
          {
            "name": "singlet_top3_contains_naive_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Tn",
              "Tn_Tmem",
              "early_Tem_GZMK",
              "Tem_GZMH"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "TOXhigh_Tex",
              "LAG3high_Tex",
              "GZMKhigh_Tex",
              "TCF7_stem_like_Tex",
              "MKI67_pos_Tex_Tprol",
              "MKI67high_Tex_Tprol"
            ]
          },
          {
            "name": "singlet_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "TOXhigh_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "TCF7_stem_like_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "MKI67_state_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "Tn_or_Tn_Tmem_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_stemlike_precursor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_reported_canonical_expansion_direction",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_substantially_more_expanded_than_singlet",
              "cluster_modestly_more_expanded_than_singlet"
            ]
          },
          {
            "name": "cf_expansion_lazy_defer",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "expansion_pattern_not_detectable"
            ]
          },
          {
            "name": "cf_reported_canonical_expected_biology",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_antigen_experienced_singlet_bystander"
            ]
          },
          {
            "name": "cf_biological_inference_null_artifact_or_lineage",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "differences_reflect_dissociation_or_sorting_artifact",
              "cluster_and_singlet_capture_same_biology",
              "cluster_and_singlet_capture_distinct_cd8_lineages"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "biological_relations",
            "kind": "dict",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_greater_than",
                  "greater_than"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "less_than",
                  "much_less_than"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than",
                  "approximately_equal"
                ]
              },
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal"
                ]
              },
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal",
                  "greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_exhaustion_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_naive_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_proliferation_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_expansion_richness_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_multichain_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_intermediate_gzmk_overgeneralized",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_melanocytic_conjugate_overcall",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "values": [
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_relations_mass_hedge",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              }
            }
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: FAIL\n  + list_match: list_match (answer_field='cd8_states_enriched_in_singlet_fraction'): PASS  (score=3.0)\n  + list_match: list_match (answer_field='cd8_states_enriched_in_tumor_cluster_fraction'): PASS  (score=3.0)\n  + clonotype_expansion_direction_data_true: passed=True, score=1.0\n  + biological_inference_data_true: passed=True, score=1.0\n  x dict_match: dict_match (answer_field='biological_relations'): FAIL  (score=6.0)\n  + hard_fail 'singlet_top3_contains_naive_state_prior_pushing': not triggered\n  + hard_fail 'tumor_cluster_top3_contains_exhausted_state_prior_pushing': not triggered\n  + hard_fail 'singlet_top3_contains_decoy_token': not triggered\n  + hard_fail 'tumor_cluster_top3_contains_decoy_token': not triggered\n  + hard_fail 'TOXhigh_Tex_wrong_functional_category': not triggered\n  + hard_fail 'TCF7_stem_like_Tex_wrong_functional_category': not triggered\n  + hard_fail 'MKI67_state_wrong_functional_category': not triggered\n  + hard_fail 'Tn_or_Tn_Tmem_wrong_functional_category': not triggered\n  + hard_fail 'missing_exhausted_or_proliferating_anchor_in_singlet_top3': not triggered\n  + hard_fail 'missing_stemlike_precursor_in_singlet_top3': not triggered\n  + hard_fail 'missing_naive_or_early_memory_anchor_in_tumor_cluster_top3': not triggered\n  + hard_fail 'cf_reported_canonical_expansion_direction': not triggered\n  + hard_fail 'cf_expansion_lazy_defer': not triggered\n  + hard_fail 'cf_reported_canonical_expected_biology': not triggered\n  + hard_fail 'cf_biological_inference_null_artifact_or_lineage': not triggered\n  + hard_fail 'cf_exhaustion_inverted_prior_push': not triggered\n  + hard_fail 'cf_naive_inverted_prior_push': not triggered\n  + hard_fail 'cf_proliferation_inverted_prior_push': not triggered\n  + hard_fail 'cf_expansion_richness_inverted_prior_push': not triggered\n  + hard_fail 'cf_multichain_inverted_prior_push': not triggered\n  x hard_fail 'cf_intermediate_gzmk_overgeneralized': TRIGGERED\n  + hard_fail 'cf_melanocytic_conjugate_overcall': not triggered\n  + hard_fail 'cf_relations_mass_hedge': not triggered",
      "metrics": {
        "hard_fail_triggered": [
          "cf_intermediate_gzmk_overgeneralized"
        ],
        "pass_rule": "all",
        "scoring_count": 5,
        "scoring_passed": 4,
        "scoring_total_score": 14
      },
      "fieldScores": {
        "biological_inference_data_true": 1,
        "clonotype_expansion_direction_data_true": 1,
        "dict_match": 6,
        "list_match": 3
      },
      "graderChecks": [
        {
          "name": "cd8_states_enriched_in_singlet_fraction",
          "kind": "list_match",
          "passed": true,
          "score": 3
        },
        {
          "name": "cd8_states_enriched_in_tumor_cluster_fraction",
          "kind": "list_match",
          "passed": true,
          "score": 3
        },
        {
          "name": "clonotype_expansion_direction_data_true",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "biological_inference_data_true",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "biological_relations",
          "kind": "list_match",
          "passed": false,
          "score": 6
        },
        {
          "name": "singlet_top3_contains_naive_state_prior_pushing",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "singlet_top3_contains_decoy_token",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "tumor_cluster_top3_contains_decoy_token",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "TOXhigh_Tex_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "TCF7_stem_like_Tex_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "MKI67_state_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "Tn_or_Tn_Tmem_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_stemlike_precursor_in_singlet_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_reported_canonical_expansion_direction",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_expansion_lazy_defer",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_reported_canonical_expected_biology",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_biological_inference_null_artifact_or_lineage",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_exhaustion_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_naive_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_proliferation_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_expansion_richness_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_multichain_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_intermediate_gzmk_overgeneralized",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "cf_melanocytic_conjugate_overcall",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_relations_mass_hedge",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "biological_inference": "singlet_antigen_experienced_cluster_bystander",
        "biological_relations": {
          "clonal_expansion_richness_in_singlets_vs_tumor_cluster": "greater_than",
          "exhaustion_program_in_singlets_vs_tumor_cluster": "greater_than",
          "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": "greater_than",
          "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": "greater_than",
          "multichain_tcr_rate_in_singlets_vs_tumor_cluster": "greater_than",
          "naive_memory_program_in_singlets_vs_tumor_cluster": "less_than",
          "proliferation_program_in_singlets_vs_tumor_cluster": "greater_than"
        },
        "cd8_states_enriched_in_singlet_fraction": [
          {
            "functional_category": "stem_like_exhausted",
            "state_label": "TCF7_stem_like_Tex"
          },
          {
            "functional_category": "intermediate_exhausted_gzmk",
            "state_label": "GZMKhigh_Tex"
          },
          {
            "functional_category": "terminal_exhausted",
            "state_label": "TOXhigh_Tex"
          }
        ],
        "cd8_states_enriched_in_tumor_cluster_fraction": [
          {
            "functional_category": "naive_memory",
            "state_label": "Tn"
          },
          {
            "functional_category": "early_effector_memory",
            "state_label": "early_Tem_GZMK"
          },
          {
            "functional_category": "early_effector_memory",
            "state_label": "Tem_GZMH"
          }
        ],
        "clonotype_expansion_finding": "singlet_substantially_more_expanded_than_cluster"
      },
      "nTurns": 76,
      "checkpointSignals": {
        "clustering": true,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": false,
        "nerveJointAnchor": false,
        "multiomeAssembly": false,
        "depthNormalizedAtac": false,
        "regulonInference": true,
        "cd8SubsetIntegration": true,
        "perPatientEnrichment": true,
        "tcrRichnessMetrics": true,
        "orthologProjection": false,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": true,
        "mousePseudobulkDesign": false,
        "cancerEpithelialDenominator": true,
        "geneLevelDe": false,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "trajectoryFile": "scbench-long/trajectories/10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1__gemini-3.5-flash__pi__r3.json",
      "runId": "gemini-3.5-flash__pi__10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1__r3",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-glm-5p2-pi",
      "modelName": "glm-5p2",
      "harness": "pi",
      "task": "10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 1,
      "passed": true,
      "score": 14,
      "durationS": 2161.72,
      "cost": 0.5964632799999999,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_singlet_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "TOXhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "LAG3high_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "GZMKhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "intermediate_exhausted_gzmk"
                    ],
                    "canonical": "intermediate_exhausted_gzmk"
                  }
                }
              },
              {
                "key": "TCF7_stem_like_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "stem_like_exhausted"
                    ],
                    "canonical": "stem_like_exhausted"
                  }
                }
              },
              {
                "key": "MKI67_pos_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              },
              {
                "key": "MKI67high_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_tumor_cluster_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "Tn",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "Tn_Tmem",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "early_Tem_GZMK",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              },
              {
                "key": "Tem_GZMH",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              }
            ]
          },
          {
            "name": "clonotype_expansion_direction_data_true",
            "role": "gate",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_substantially_more_expanded_than_cluster",
              "singlet_modestly_more_expanded_than_cluster"
            ]
          },
          {
            "name": "biological_inference_data_true",
            "role": "gate",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_antigen_experienced_cluster_bystander"
            ]
          },
          {
            "name": "singlet_top3_contains_naive_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Tn",
              "Tn_Tmem",
              "early_Tem_GZMK",
              "Tem_GZMH"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "TOXhigh_Tex",
              "LAG3high_Tex",
              "GZMKhigh_Tex",
              "TCF7_stem_like_Tex",
              "MKI67_pos_Tex_Tprol",
              "MKI67high_Tex_Tprol"
            ]
          },
          {
            "name": "singlet_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "TOXhigh_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "TCF7_stem_like_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "MKI67_state_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "Tn_or_Tn_Tmem_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_stemlike_precursor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_reported_canonical_expansion_direction",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_substantially_more_expanded_than_singlet",
              "cluster_modestly_more_expanded_than_singlet"
            ]
          },
          {
            "name": "cf_expansion_lazy_defer",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "expansion_pattern_not_detectable"
            ]
          },
          {
            "name": "cf_reported_canonical_expected_biology",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_antigen_experienced_singlet_bystander"
            ]
          },
          {
            "name": "cf_biological_inference_null_artifact_or_lineage",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "differences_reflect_dissociation_or_sorting_artifact",
              "cluster_and_singlet_capture_same_biology",
              "cluster_and_singlet_capture_distinct_cd8_lineages"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "biological_relations",
            "kind": "dict",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_greater_than",
                  "greater_than"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "less_than",
                  "much_less_than"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than",
                  "approximately_equal"
                ]
              },
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal"
                ]
              },
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal",
                  "greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_exhaustion_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_naive_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_proliferation_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_expansion_richness_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_multichain_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_intermediate_gzmk_overgeneralized",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_melanocytic_conjugate_overcall",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "values": [
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_relations_mass_hedge",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              }
            }
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: PASS\n  + list_match: list_match (answer_field='cd8_states_enriched_in_singlet_fraction'): PASS  (score=3.0)\n  + list_match: list_match (answer_field='cd8_states_enriched_in_tumor_cluster_fraction'): PASS  (score=2.0)\n  + clonotype_expansion_direction_data_true: passed=True, score=1.0\n  + biological_inference_data_true: passed=True, score=1.0\n  + dict_match: dict_match (answer_field='biological_relations'): PASS  (score=7.0)\n  + hard_fail 'singlet_top3_contains_naive_state_prior_pushing': not triggered\n  + hard_fail 'tumor_cluster_top3_contains_exhausted_state_prior_pushing': not triggered\n  + hard_fail 'singlet_top3_contains_decoy_token': not triggered\n  + hard_fail 'tumor_cluster_top3_contains_decoy_token': not triggered\n  + hard_fail 'TOXhigh_Tex_wrong_functional_category': not triggered\n  + hard_fail 'TCF7_stem_like_Tex_wrong_functional_category': not triggered\n  + hard_fail 'MKI67_state_wrong_functional_category': not triggered\n  + hard_fail 'Tn_or_Tn_Tmem_wrong_functional_category': not triggered\n  + hard_fail 'missing_exhausted_or_proliferating_anchor_in_singlet_top3': not triggered\n  + hard_fail 'missing_stemlike_precursor_in_singlet_top3': not triggered\n  + hard_fail 'missing_naive_or_early_memory_anchor_in_tumor_cluster_top3': not triggered\n  + hard_fail 'cf_reported_canonical_expansion_direction': not triggered\n  + hard_fail 'cf_expansion_lazy_defer': not triggered\n  + hard_fail 'cf_reported_canonical_expected_biology': not triggered\n  + hard_fail 'cf_biological_inference_null_artifact_or_lineage': not triggered\n  + hard_fail 'cf_exhaustion_inverted_prior_push': not triggered\n  + hard_fail 'cf_naive_inverted_prior_push': not triggered\n  + hard_fail 'cf_proliferation_inverted_prior_push': not triggered\n  + hard_fail 'cf_expansion_richness_inverted_prior_push': not triggered\n  + hard_fail 'cf_multichain_inverted_prior_push': not triggered\n  + hard_fail 'cf_intermediate_gzmk_overgeneralized': not triggered\n  + hard_fail 'cf_melanocytic_conjugate_overcall': not triggered\n  + hard_fail 'cf_relations_mass_hedge': not triggered",
      "metrics": {
        "hard_fail_triggered": [],
        "pass_rule": "all",
        "scoring_count": 5,
        "scoring_passed": 5,
        "scoring_total_score": 14
      },
      "fieldScores": {
        "biological_inference_data_true": 1,
        "clonotype_expansion_direction_data_true": 1,
        "dict_match": 7,
        "list_match": 2
      },
      "graderChecks": [
        {
          "name": "cd8_states_enriched_in_singlet_fraction",
          "kind": "list_match",
          "passed": true,
          "score": 3
        },
        {
          "name": "cd8_states_enriched_in_tumor_cluster_fraction",
          "kind": "list_match",
          "passed": true,
          "score": 2
        },
        {
          "name": "clonotype_expansion_direction_data_true",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "biological_inference_data_true",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "biological_relations",
          "kind": "list_match",
          "passed": true,
          "score": 7
        },
        {
          "name": "singlet_top3_contains_naive_state_prior_pushing",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "singlet_top3_contains_decoy_token",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "tumor_cluster_top3_contains_decoy_token",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "TOXhigh_Tex_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "TCF7_stem_like_Tex_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "MKI67_state_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "Tn_or_Tn_Tmem_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_stemlike_precursor_in_singlet_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_reported_canonical_expansion_direction",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_expansion_lazy_defer",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_reported_canonical_expected_biology",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_biological_inference_null_artifact_or_lineage",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_exhaustion_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_naive_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_proliferation_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_expansion_richness_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_multichain_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_intermediate_gzmk_overgeneralized",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_melanocytic_conjugate_overcall",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_relations_mass_hedge",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "biological_inference": "singlet_antigen_experienced_cluster_bystander",
        "biological_relations": {
          "clonal_expansion_richness_in_singlets_vs_tumor_cluster": "greater_than",
          "exhaustion_program_in_singlets_vs_tumor_cluster": "greater_than",
          "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": "less_than",
          "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": "greater_than",
          "multichain_tcr_rate_in_singlets_vs_tumor_cluster": "greater_than",
          "naive_memory_program_in_singlets_vs_tumor_cluster": "less_than",
          "proliferation_program_in_singlets_vs_tumor_cluster": "greater_than"
        },
        "cd8_states_enriched_in_singlet_fraction": [
          {
            "functional_category": "stem_like_exhausted",
            "state_label": "TCF7_stem_like_Tex"
          },
          {
            "functional_category": "terminal_exhausted",
            "state_label": "TOXhigh_Tex"
          },
          {
            "functional_category": "proliferating",
            "state_label": "MKI67high_Tex_Tprol"
          }
        ],
        "cd8_states_enriched_in_tumor_cluster_fraction": [
          {
            "functional_category": "naive_memory",
            "state_label": "Tn"
          },
          {
            "functional_category": "early_effector_memory",
            "state_label": "early_Tem_GZMK"
          },
          {
            "functional_category": "isg_response",
            "state_label": "ISG_pos"
          }
        ],
        "clonotype_expansion_finding": "singlet_modestly_more_expanded_than_cluster"
      },
      "nTurns": 64,
      "checkpointSignals": {
        "clustering": true,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": true,
        "nerveJointAnchor": false,
        "multiomeAssembly": true,
        "depthNormalizedAtac": false,
        "regulonInference": true,
        "cd8SubsetIntegration": true,
        "perPatientEnrichment": true,
        "tcrRichnessMetrics": true,
        "orthologProjection": false,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": true,
        "mousePseudobulkDesign": false,
        "cancerEpithelialDenominator": false,
        "geneLevelDe": false,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "trajectoryFile": "scbench-long/trajectories/10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1__glm-5p2__pi__r1.json",
      "runId": "glm-5p2__pi__10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1__r1",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-glm-5p2-pi",
      "modelName": "glm-5p2",
      "harness": "pi",
      "task": "10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 2,
      "passed": true,
      "score": 15,
      "durationS": 621.27,
      "cost": 0.32935208000000005,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_singlet_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "TOXhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "LAG3high_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "GZMKhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "intermediate_exhausted_gzmk"
                    ],
                    "canonical": "intermediate_exhausted_gzmk"
                  }
                }
              },
              {
                "key": "TCF7_stem_like_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "stem_like_exhausted"
                    ],
                    "canonical": "stem_like_exhausted"
                  }
                }
              },
              {
                "key": "MKI67_pos_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              },
              {
                "key": "MKI67high_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_tumor_cluster_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "Tn",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "Tn_Tmem",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "early_Tem_GZMK",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              },
              {
                "key": "Tem_GZMH",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              }
            ]
          },
          {
            "name": "clonotype_expansion_direction_data_true",
            "role": "gate",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_substantially_more_expanded_than_cluster",
              "singlet_modestly_more_expanded_than_cluster"
            ]
          },
          {
            "name": "biological_inference_data_true",
            "role": "gate",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_antigen_experienced_cluster_bystander"
            ]
          },
          {
            "name": "singlet_top3_contains_naive_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Tn",
              "Tn_Tmem",
              "early_Tem_GZMK",
              "Tem_GZMH"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "TOXhigh_Tex",
              "LAG3high_Tex",
              "GZMKhigh_Tex",
              "TCF7_stem_like_Tex",
              "MKI67_pos_Tex_Tprol",
              "MKI67high_Tex_Tprol"
            ]
          },
          {
            "name": "singlet_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "TOXhigh_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "TCF7_stem_like_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "MKI67_state_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "Tn_or_Tn_Tmem_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_stemlike_precursor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_reported_canonical_expansion_direction",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_substantially_more_expanded_than_singlet",
              "cluster_modestly_more_expanded_than_singlet"
            ]
          },
          {
            "name": "cf_expansion_lazy_defer",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "expansion_pattern_not_detectable"
            ]
          },
          {
            "name": "cf_reported_canonical_expected_biology",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_antigen_experienced_singlet_bystander"
            ]
          },
          {
            "name": "cf_biological_inference_null_artifact_or_lineage",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "differences_reflect_dissociation_or_sorting_artifact",
              "cluster_and_singlet_capture_same_biology",
              "cluster_and_singlet_capture_distinct_cd8_lineages"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "biological_relations",
            "kind": "dict",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_greater_than",
                  "greater_than"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "less_than",
                  "much_less_than"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than",
                  "approximately_equal"
                ]
              },
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal"
                ]
              },
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal",
                  "greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_exhaustion_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_naive_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_proliferation_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_expansion_richness_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_multichain_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_intermediate_gzmk_overgeneralized",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_melanocytic_conjugate_overcall",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "values": [
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_relations_mass_hedge",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              }
            }
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: PASS\n  + list_match: list_match (answer_field='cd8_states_enriched_in_singlet_fraction'): PASS  (score=3.0)\n  + list_match: list_match (answer_field='cd8_states_enriched_in_tumor_cluster_fraction'): PASS  (score=3.0)\n  + clonotype_expansion_direction_data_true: passed=True, score=1.0\n  + biological_inference_data_true: passed=True, score=1.0\n  + dict_match: dict_match (answer_field='biological_relations'): PASS  (score=7.0)\n  + hard_fail 'singlet_top3_contains_naive_state_prior_pushing': not triggered\n  + hard_fail 'tumor_cluster_top3_contains_exhausted_state_prior_pushing': not triggered\n  + hard_fail 'singlet_top3_contains_decoy_token': not triggered\n  + hard_fail 'tumor_cluster_top3_contains_decoy_token': not triggered\n  + hard_fail 'TOXhigh_Tex_wrong_functional_category': not triggered\n  + hard_fail 'TCF7_stem_like_Tex_wrong_functional_category': not triggered\n  + hard_fail 'MKI67_state_wrong_functional_category': not triggered\n  + hard_fail 'Tn_or_Tn_Tmem_wrong_functional_category': not triggered\n  + hard_fail 'missing_exhausted_or_proliferating_anchor_in_singlet_top3': not triggered\n  + hard_fail 'missing_stemlike_precursor_in_singlet_top3': not triggered\n  + hard_fail 'missing_naive_or_early_memory_anchor_in_tumor_cluster_top3': not triggered\n  + hard_fail 'cf_reported_canonical_expansion_direction': not triggered\n  + hard_fail 'cf_expansion_lazy_defer': not triggered\n  + hard_fail 'cf_reported_canonical_expected_biology': not triggered\n  + hard_fail 'cf_biological_inference_null_artifact_or_lineage': not triggered\n  + hard_fail 'cf_exhaustion_inverted_prior_push': not triggered\n  + hard_fail 'cf_naive_inverted_prior_push': not triggered\n  + hard_fail 'cf_proliferation_inverted_prior_push': not triggered\n  + hard_fail 'cf_expansion_richness_inverted_prior_push': not triggered\n  + hard_fail 'cf_multichain_inverted_prior_push': not triggered\n  + hard_fail 'cf_intermediate_gzmk_overgeneralized': not triggered\n  + hard_fail 'cf_melanocytic_conjugate_overcall': not triggered\n  + hard_fail 'cf_relations_mass_hedge': not triggered",
      "metrics": {
        "hard_fail_triggered": [],
        "pass_rule": "all",
        "scoring_count": 5,
        "scoring_passed": 5,
        "scoring_total_score": 15
      },
      "fieldScores": {
        "biological_inference_data_true": 1,
        "clonotype_expansion_direction_data_true": 1,
        "dict_match": 7,
        "list_match": 3
      },
      "graderChecks": [
        {
          "name": "cd8_states_enriched_in_singlet_fraction",
          "kind": "list_match",
          "passed": true,
          "score": 3
        },
        {
          "name": "cd8_states_enriched_in_tumor_cluster_fraction",
          "kind": "list_match",
          "passed": true,
          "score": 3
        },
        {
          "name": "clonotype_expansion_direction_data_true",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "biological_inference_data_true",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "biological_relations",
          "kind": "list_match",
          "passed": true,
          "score": 7
        },
        {
          "name": "singlet_top3_contains_naive_state_prior_pushing",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "singlet_top3_contains_decoy_token",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "tumor_cluster_top3_contains_decoy_token",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "TOXhigh_Tex_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "TCF7_stem_like_Tex_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "MKI67_state_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "Tn_or_Tn_Tmem_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_stemlike_precursor_in_singlet_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_reported_canonical_expansion_direction",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_expansion_lazy_defer",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_reported_canonical_expected_biology",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_biological_inference_null_artifact_or_lineage",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_exhaustion_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_naive_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_proliferation_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_expansion_richness_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_multichain_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_intermediate_gzmk_overgeneralized",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_melanocytic_conjugate_overcall",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_relations_mass_hedge",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "biological_inference": "singlet_antigen_experienced_cluster_bystander",
        "biological_relations": {
          "clonal_expansion_richness_in_singlets_vs_tumor_cluster": "greater_than",
          "exhaustion_program_in_singlets_vs_tumor_cluster": "greater_than",
          "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": "approximately_equal",
          "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": "approximately_equal",
          "multichain_tcr_rate_in_singlets_vs_tumor_cluster": "greater_than",
          "naive_memory_program_in_singlets_vs_tumor_cluster": "less_than",
          "proliferation_program_in_singlets_vs_tumor_cluster": "greater_than"
        },
        "cd8_states_enriched_in_singlet_fraction": [
          {
            "functional_category": "stem_like_exhausted",
            "state_label": "TCF7_stem_like_Tex"
          },
          {
            "functional_category": "intermediate_exhausted_gzmk",
            "state_label": "GZMKhigh_Tex"
          },
          {
            "functional_category": "terminal_exhausted",
            "state_label": "TOXhigh_Tex"
          }
        ],
        "cd8_states_enriched_in_tumor_cluster_fraction": [
          {
            "functional_category": "naive_memory",
            "state_label": "Tn"
          },
          {
            "functional_category": "early_effector_memory",
            "state_label": "early_Tem_GZMK"
          },
          {
            "functional_category": "naive_memory",
            "state_label": "Tn_Tmem"
          }
        ],
        "clonotype_expansion_finding": "singlet_modestly_more_expanded_than_cluster"
      },
      "nTurns": 37,
      "checkpointSignals": {
        "clustering": false,
        "markerAnnotation": true,
        "scoreBasedAnnotation": true,
        "normalization": false,
        "nerveJointAnchor": false,
        "multiomeAssembly": true,
        "depthNormalizedAtac": false,
        "regulonInference": true,
        "cd8SubsetIntegration": true,
        "perPatientEnrichment": true,
        "tcrRichnessMetrics": true,
        "orthologProjection": false,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": true,
        "mousePseudobulkDesign": false,
        "cancerEpithelialDenominator": false,
        "geneLevelDe": false,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "trajectoryFile": "scbench-long/trajectories/10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1__glm-5p2__pi__r2.json",
      "runId": "glm-5p2__pi__10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1__r2",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-glm-5p2-pi",
      "modelName": "glm-5p2",
      "harness": "pi",
      "task": "10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": false,
      "score": 13,
      "durationS": 1337.22,
      "cost": 0.32228921,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_singlet_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "TOXhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "LAG3high_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "GZMKhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "intermediate_exhausted_gzmk"
                    ],
                    "canonical": "intermediate_exhausted_gzmk"
                  }
                }
              },
              {
                "key": "TCF7_stem_like_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "stem_like_exhausted"
                    ],
                    "canonical": "stem_like_exhausted"
                  }
                }
              },
              {
                "key": "MKI67_pos_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              },
              {
                "key": "MKI67high_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_tumor_cluster_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "Tn",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "Tn_Tmem",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "early_Tem_GZMK",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              },
              {
                "key": "Tem_GZMH",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              }
            ]
          },
          {
            "name": "clonotype_expansion_direction_data_true",
            "role": "gate",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_substantially_more_expanded_than_cluster",
              "singlet_modestly_more_expanded_than_cluster"
            ]
          },
          {
            "name": "biological_inference_data_true",
            "role": "gate",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_antigen_experienced_cluster_bystander"
            ]
          },
          {
            "name": "singlet_top3_contains_naive_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Tn",
              "Tn_Tmem",
              "early_Tem_GZMK",
              "Tem_GZMH"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "TOXhigh_Tex",
              "LAG3high_Tex",
              "GZMKhigh_Tex",
              "TCF7_stem_like_Tex",
              "MKI67_pos_Tex_Tprol",
              "MKI67high_Tex_Tprol"
            ]
          },
          {
            "name": "singlet_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "TOXhigh_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "TCF7_stem_like_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "MKI67_state_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "Tn_or_Tn_Tmem_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_stemlike_precursor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_reported_canonical_expansion_direction",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_substantially_more_expanded_than_singlet",
              "cluster_modestly_more_expanded_than_singlet"
            ]
          },
          {
            "name": "cf_expansion_lazy_defer",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "expansion_pattern_not_detectable"
            ]
          },
          {
            "name": "cf_reported_canonical_expected_biology",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_antigen_experienced_singlet_bystander"
            ]
          },
          {
            "name": "cf_biological_inference_null_artifact_or_lineage",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "differences_reflect_dissociation_or_sorting_artifact",
              "cluster_and_singlet_capture_same_biology",
              "cluster_and_singlet_capture_distinct_cd8_lineages"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "biological_relations",
            "kind": "dict",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_greater_than",
                  "greater_than"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "less_than",
                  "much_less_than"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than",
                  "approximately_equal"
                ]
              },
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal"
                ]
              },
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal",
                  "greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_exhaustion_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_naive_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_proliferation_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_expansion_richness_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_multichain_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_intermediate_gzmk_overgeneralized",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_melanocytic_conjugate_overcall",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "values": [
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_relations_mass_hedge",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              }
            }
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: FAIL\n  + list_match: list_match (answer_field='cd8_states_enriched_in_singlet_fraction'): PASS  (score=3.0)\n  + list_match: list_match (answer_field='cd8_states_enriched_in_tumor_cluster_fraction'): PASS  (score=2.0)\n  + clonotype_expansion_direction_data_true: passed=True, score=1.0\n  + biological_inference_data_true: passed=True, score=1.0\n  x dict_match: dict_match (answer_field='biological_relations'): FAIL  (score=6.0)\n  + hard_fail 'singlet_top3_contains_naive_state_prior_pushing': not triggered\n  x hard_fail 'tumor_cluster_top3_contains_exhausted_state_prior_pushing': TRIGGERED\n  + hard_fail 'singlet_top3_contains_decoy_token': not triggered\n  + hard_fail 'tumor_cluster_top3_contains_decoy_token': not triggered\n  + hard_fail 'TOXhigh_Tex_wrong_functional_category': not triggered\n  + hard_fail 'TCF7_stem_like_Tex_wrong_functional_category': not triggered\n  + hard_fail 'MKI67_state_wrong_functional_category': not triggered\n  + hard_fail 'Tn_or_Tn_Tmem_wrong_functional_category': not triggered\n  + hard_fail 'missing_exhausted_or_proliferating_anchor_in_singlet_top3': not triggered\n  + hard_fail 'missing_stemlike_precursor_in_singlet_top3': not triggered\n  + hard_fail 'missing_naive_or_early_memory_anchor_in_tumor_cluster_top3': not triggered\n  + hard_fail 'cf_reported_canonical_expansion_direction': not triggered\n  + hard_fail 'cf_expansion_lazy_defer': not triggered\n  + hard_fail 'cf_reported_canonical_expected_biology': not triggered\n  + hard_fail 'cf_biological_inference_null_artifact_or_lineage': not triggered\n  + hard_fail 'cf_exhaustion_inverted_prior_push': not triggered\n  + hard_fail 'cf_naive_inverted_prior_push': not triggered\n  + hard_fail 'cf_proliferation_inverted_prior_push': not triggered\n  + hard_fail 'cf_expansion_richness_inverted_prior_push': not triggered\n  + hard_fail 'cf_multichain_inverted_prior_push': not triggered\n  x hard_fail 'cf_intermediate_gzmk_overgeneralized': TRIGGERED\n  + hard_fail 'cf_melanocytic_conjugate_overcall': not triggered\n  + hard_fail 'cf_relations_mass_hedge': not triggered",
      "metrics": {
        "hard_fail_triggered": [
          "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
          "cf_intermediate_gzmk_overgeneralized"
        ],
        "pass_rule": "all",
        "scoring_count": 5,
        "scoring_passed": 4,
        "scoring_total_score": 13
      },
      "fieldScores": {
        "biological_inference_data_true": 1,
        "clonotype_expansion_direction_data_true": 1,
        "dict_match": 6,
        "list_match": 2
      },
      "graderChecks": [
        {
          "name": "cd8_states_enriched_in_singlet_fraction",
          "kind": "list_match",
          "passed": true,
          "score": 3
        },
        {
          "name": "cd8_states_enriched_in_tumor_cluster_fraction",
          "kind": "list_match",
          "passed": true,
          "score": 2
        },
        {
          "name": "clonotype_expansion_direction_data_true",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "biological_inference_data_true",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "biological_relations",
          "kind": "list_match",
          "passed": false,
          "score": 6
        },
        {
          "name": "singlet_top3_contains_naive_state_prior_pushing",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "singlet_top3_contains_decoy_token",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "tumor_cluster_top3_contains_decoy_token",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "TOXhigh_Tex_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "TCF7_stem_like_Tex_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "MKI67_state_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "Tn_or_Tn_Tmem_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_stemlike_precursor_in_singlet_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_reported_canonical_expansion_direction",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_expansion_lazy_defer",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_reported_canonical_expected_biology",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_biological_inference_null_artifact_or_lineage",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_exhaustion_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_naive_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_proliferation_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_expansion_richness_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_multichain_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_intermediate_gzmk_overgeneralized",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "cf_melanocytic_conjugate_overcall",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_relations_mass_hedge",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "biological_inference": "singlet_antigen_experienced_cluster_bystander",
        "biological_relations": {
          "clonal_expansion_richness_in_singlets_vs_tumor_cluster": "greater_than",
          "exhaustion_program_in_singlets_vs_tumor_cluster": "greater_than",
          "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": "greater_than",
          "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": "approximately_equal",
          "multichain_tcr_rate_in_singlets_vs_tumor_cluster": "greater_than",
          "naive_memory_program_in_singlets_vs_tumor_cluster": "less_than",
          "proliferation_program_in_singlets_vs_tumor_cluster": "greater_than"
        },
        "cd8_states_enriched_in_singlet_fraction": [
          {
            "functional_category": "stem_like_exhausted",
            "state_label": "TCF7_stem_like_Tex"
          },
          {
            "functional_category": "intermediate_exhausted_gzmk",
            "state_label": "GZMKhigh_Tex"
          },
          {
            "functional_category": "terminal_exhausted",
            "state_label": "TOXhigh_Tex"
          }
        ],
        "cd8_states_enriched_in_tumor_cluster_fraction": [
          {
            "functional_category": "naive_memory",
            "state_label": "Tn"
          },
          {
            "functional_category": "naive_memory",
            "state_label": "Tn_Tmem"
          },
          {
            "functional_category": "stem_like_exhausted",
            "state_label": "TCF7_stem_like_Tex"
          }
        ],
        "clonotype_expansion_finding": "singlet_modestly_more_expanded_than_cluster"
      },
      "nTurns": 42,
      "checkpointSignals": {
        "clustering": false,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": false,
        "nerveJointAnchor": false,
        "multiomeAssembly": true,
        "depthNormalizedAtac": true,
        "regulonInference": true,
        "cd8SubsetIntegration": true,
        "perPatientEnrichment": true,
        "tcrRichnessMetrics": true,
        "orthologProjection": false,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": true,
        "mousePseudobulkDesign": false,
        "cancerEpithelialDenominator": false,
        "geneLevelDe": false,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "trajectoryFile": "scbench-long/trajectories/10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1__glm-5p2__pi__r3.json",
      "runId": "glm-5p2__pi__10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1__r3",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-gpt-5-4-codex",
      "modelName": "gpt-5.4",
      "harness": "openai-codex",
      "task": "10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 1,
      "passed": false,
      "score": 14,
      "durationS": 952.62,
      "cost": 3.087758,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_singlet_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "TOXhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "LAG3high_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "GZMKhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "intermediate_exhausted_gzmk"
                    ],
                    "canonical": "intermediate_exhausted_gzmk"
                  }
                }
              },
              {
                "key": "TCF7_stem_like_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "stem_like_exhausted"
                    ],
                    "canonical": "stem_like_exhausted"
                  }
                }
              },
              {
                "key": "MKI67_pos_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              },
              {
                "key": "MKI67high_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_tumor_cluster_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "Tn",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "Tn_Tmem",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "early_Tem_GZMK",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              },
              {
                "key": "Tem_GZMH",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              }
            ]
          },
          {
            "name": "clonotype_expansion_direction_data_true",
            "role": "gate",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_substantially_more_expanded_than_cluster",
              "singlet_modestly_more_expanded_than_cluster"
            ]
          },
          {
            "name": "biological_inference_data_true",
            "role": "gate",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_antigen_experienced_cluster_bystander"
            ]
          },
          {
            "name": "singlet_top3_contains_naive_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Tn",
              "Tn_Tmem",
              "early_Tem_GZMK",
              "Tem_GZMH"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "TOXhigh_Tex",
              "LAG3high_Tex",
              "GZMKhigh_Tex",
              "TCF7_stem_like_Tex",
              "MKI67_pos_Tex_Tprol",
              "MKI67high_Tex_Tprol"
            ]
          },
          {
            "name": "singlet_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "TOXhigh_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "TCF7_stem_like_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "MKI67_state_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "Tn_or_Tn_Tmem_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_stemlike_precursor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_reported_canonical_expansion_direction",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_substantially_more_expanded_than_singlet",
              "cluster_modestly_more_expanded_than_singlet"
            ]
          },
          {
            "name": "cf_expansion_lazy_defer",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "expansion_pattern_not_detectable"
            ]
          },
          {
            "name": "cf_reported_canonical_expected_biology",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_antigen_experienced_singlet_bystander"
            ]
          },
          {
            "name": "cf_biological_inference_null_artifact_or_lineage",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "differences_reflect_dissociation_or_sorting_artifact",
              "cluster_and_singlet_capture_same_biology",
              "cluster_and_singlet_capture_distinct_cd8_lineages"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "biological_relations",
            "kind": "dict",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_greater_than",
                  "greater_than"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "less_than",
                  "much_less_than"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than",
                  "approximately_equal"
                ]
              },
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal"
                ]
              },
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal",
                  "greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_exhaustion_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_naive_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_proliferation_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_expansion_richness_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_multichain_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_intermediate_gzmk_overgeneralized",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_melanocytic_conjugate_overcall",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "values": [
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_relations_mass_hedge",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              }
            }
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: FAIL\n  + list_match: list_match (answer_field='cd8_states_enriched_in_singlet_fraction'): PASS  (score=3.0)\n  + list_match: list_match (answer_field='cd8_states_enriched_in_tumor_cluster_fraction'): PASS  (score=3.0)\n  + clonotype_expansion_direction_data_true: passed=True, score=1.0\n  + biological_inference_data_true: passed=True, score=1.0\n  x dict_match: dict_match (answer_field='biological_relations'): FAIL  (score=6.0)\n  + hard_fail 'singlet_top3_contains_naive_state_prior_pushing': not triggered\n  + hard_fail 'tumor_cluster_top3_contains_exhausted_state_prior_pushing': not triggered\n  + hard_fail 'singlet_top3_contains_decoy_token': not triggered\n  + hard_fail 'tumor_cluster_top3_contains_decoy_token': not triggered\n  + hard_fail 'TOXhigh_Tex_wrong_functional_category': not triggered\n  + hard_fail 'TCF7_stem_like_Tex_wrong_functional_category': not triggered\n  + hard_fail 'MKI67_state_wrong_functional_category': not triggered\n  + hard_fail 'Tn_or_Tn_Tmem_wrong_functional_category': not triggered\n  + hard_fail 'missing_exhausted_or_proliferating_anchor_in_singlet_top3': not triggered\n  + hard_fail 'missing_stemlike_precursor_in_singlet_top3': not triggered\n  + hard_fail 'missing_naive_or_early_memory_anchor_in_tumor_cluster_top3': not triggered\n  + hard_fail 'cf_reported_canonical_expansion_direction': not triggered\n  + hard_fail 'cf_expansion_lazy_defer': not triggered\n  + hard_fail 'cf_reported_canonical_expected_biology': not triggered\n  + hard_fail 'cf_biological_inference_null_artifact_or_lineage': not triggered\n  + hard_fail 'cf_exhaustion_inverted_prior_push': not triggered\n  + hard_fail 'cf_naive_inverted_prior_push': not triggered\n  + hard_fail 'cf_proliferation_inverted_prior_push': not triggered\n  + hard_fail 'cf_expansion_richness_inverted_prior_push': not triggered\n  + hard_fail 'cf_multichain_inverted_prior_push': not triggered\n  x hard_fail 'cf_intermediate_gzmk_overgeneralized': TRIGGERED\n  + hard_fail 'cf_melanocytic_conjugate_overcall': not triggered\n  + hard_fail 'cf_relations_mass_hedge': not triggered",
      "metrics": {
        "hard_fail_triggered": [
          "cf_intermediate_gzmk_overgeneralized"
        ],
        "pass_rule": "all",
        "scoring_count": 5,
        "scoring_passed": 4,
        "scoring_total_score": 14
      },
      "fieldScores": {
        "biological_inference_data_true": 1,
        "clonotype_expansion_direction_data_true": 1,
        "dict_match": 6,
        "list_match": 3
      },
      "graderChecks": [
        {
          "name": "cd8_states_enriched_in_singlet_fraction",
          "kind": "list_match",
          "passed": true,
          "score": 3
        },
        {
          "name": "cd8_states_enriched_in_tumor_cluster_fraction",
          "kind": "list_match",
          "passed": true,
          "score": 3
        },
        {
          "name": "clonotype_expansion_direction_data_true",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "biological_inference_data_true",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "biological_relations",
          "kind": "list_match",
          "passed": false,
          "score": 6
        },
        {
          "name": "singlet_top3_contains_naive_state_prior_pushing",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "singlet_top3_contains_decoy_token",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "tumor_cluster_top3_contains_decoy_token",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "TOXhigh_Tex_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "TCF7_stem_like_Tex_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "MKI67_state_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "Tn_or_Tn_Tmem_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_stemlike_precursor_in_singlet_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_reported_canonical_expansion_direction",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_expansion_lazy_defer",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_reported_canonical_expected_biology",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_biological_inference_null_artifact_or_lineage",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_exhaustion_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_naive_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_proliferation_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_expansion_richness_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_multichain_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_intermediate_gzmk_overgeneralized",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "cf_melanocytic_conjugate_overcall",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_relations_mass_hedge",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "biological_inference": "singlet_antigen_experienced_cluster_bystander",
        "biological_relations": {
          "clonal_expansion_richness_in_singlets_vs_tumor_cluster": "greater_than",
          "exhaustion_program_in_singlets_vs_tumor_cluster": "greater_than",
          "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": "greater_than",
          "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": "greater_than",
          "multichain_tcr_rate_in_singlets_vs_tumor_cluster": "greater_than",
          "naive_memory_program_in_singlets_vs_tumor_cluster": "less_than",
          "proliferation_program_in_singlets_vs_tumor_cluster": "greater_than"
        },
        "cd8_states_enriched_in_singlet_fraction": [
          {
            "functional_category": "stem_like_exhausted",
            "state_label": "TCF7_stem_like_Tex"
          },
          {
            "functional_category": "intermediate_exhausted_gzmk",
            "state_label": "GZMKhigh_Tex"
          },
          {
            "functional_category": "terminal_exhausted",
            "state_label": "LAG3high_Tex"
          }
        ],
        "cd8_states_enriched_in_tumor_cluster_fraction": [
          {
            "functional_category": "naive_memory",
            "state_label": "Tn_Tmem"
          },
          {
            "functional_category": "early_effector_memory",
            "state_label": "early_Tem_GZMK"
          },
          {
            "functional_category": "early_effector_memory",
            "state_label": "Tem_GZMH"
          }
        ],
        "clonotype_expansion_finding": "singlet_modestly_more_expanded_than_cluster"
      },
      "nTurns": 1,
      "checkpointSignals": {
        "clustering": true,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": false,
        "nerveJointAnchor": false,
        "multiomeAssembly": false,
        "depthNormalizedAtac": false,
        "regulonInference": true,
        "cd8SubsetIntegration": true,
        "perPatientEnrichment": true,
        "tcrRichnessMetrics": true,
        "orthologProjection": false,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": true,
        "mousePseudobulkDesign": false,
        "cancerEpithelialDenominator": false,
        "geneLevelDe": false,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "trajectoryFile": "scbench-long/trajectories/10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1__gpt-5.4__openai-codex__r1.json",
      "runId": "gpt-5.4__openai-codex__10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1__r1",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-gpt-5-4-codex",
      "modelName": "gpt-5.4",
      "harness": "openai-codex",
      "task": "10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 2,
      "passed": false,
      "score": 12,
      "durationS": 777.47,
      "cost": 2.050728,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_singlet_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "TOXhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "LAG3high_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "GZMKhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "intermediate_exhausted_gzmk"
                    ],
                    "canonical": "intermediate_exhausted_gzmk"
                  }
                }
              },
              {
                "key": "TCF7_stem_like_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "stem_like_exhausted"
                    ],
                    "canonical": "stem_like_exhausted"
                  }
                }
              },
              {
                "key": "MKI67_pos_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              },
              {
                "key": "MKI67high_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_tumor_cluster_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "Tn",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "Tn_Tmem",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "early_Tem_GZMK",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              },
              {
                "key": "Tem_GZMH",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              }
            ]
          },
          {
            "name": "clonotype_expansion_direction_data_true",
            "role": "gate",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_substantially_more_expanded_than_cluster",
              "singlet_modestly_more_expanded_than_cluster"
            ]
          },
          {
            "name": "biological_inference_data_true",
            "role": "gate",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_antigen_experienced_cluster_bystander"
            ]
          },
          {
            "name": "singlet_top3_contains_naive_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Tn",
              "Tn_Tmem",
              "early_Tem_GZMK",
              "Tem_GZMH"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "TOXhigh_Tex",
              "LAG3high_Tex",
              "GZMKhigh_Tex",
              "TCF7_stem_like_Tex",
              "MKI67_pos_Tex_Tprol",
              "MKI67high_Tex_Tprol"
            ]
          },
          {
            "name": "singlet_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "TOXhigh_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "TCF7_stem_like_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "MKI67_state_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "Tn_or_Tn_Tmem_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_stemlike_precursor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_reported_canonical_expansion_direction",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_substantially_more_expanded_than_singlet",
              "cluster_modestly_more_expanded_than_singlet"
            ]
          },
          {
            "name": "cf_expansion_lazy_defer",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "expansion_pattern_not_detectable"
            ]
          },
          {
            "name": "cf_reported_canonical_expected_biology",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_antigen_experienced_singlet_bystander"
            ]
          },
          {
            "name": "cf_biological_inference_null_artifact_or_lineage",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "differences_reflect_dissociation_or_sorting_artifact",
              "cluster_and_singlet_capture_same_biology",
              "cluster_and_singlet_capture_distinct_cd8_lineages"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "biological_relations",
            "kind": "dict",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_greater_than",
                  "greater_than"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "less_than",
                  "much_less_than"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than",
                  "approximately_equal"
                ]
              },
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal"
                ]
              },
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal",
                  "greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_exhaustion_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_naive_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_proliferation_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_expansion_richness_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_multichain_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_intermediate_gzmk_overgeneralized",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_melanocytic_conjugate_overcall",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "values": [
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_relations_mass_hedge",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              }
            }
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: FAIL\n  + list_match: list_match (answer_field='cd8_states_enriched_in_singlet_fraction'): PASS  (score=3.0)\n  + list_match: list_match (answer_field='cd8_states_enriched_in_tumor_cluster_fraction'): PASS  (score=2.0)\n  + clonotype_expansion_direction_data_true: passed=True, score=1.0\n  + biological_inference_data_true: passed=True, score=1.0\n  x dict_match: dict_match (answer_field='biological_relations'): FAIL  (score=5.0)\n  + hard_fail 'singlet_top3_contains_naive_state_prior_pushing': not triggered\n  x hard_fail 'tumor_cluster_top3_contains_exhausted_state_prior_pushing': TRIGGERED\n  + hard_fail 'singlet_top3_contains_decoy_token': not triggered\n  + hard_fail 'tumor_cluster_top3_contains_decoy_token': not triggered\n  + hard_fail 'TOXhigh_Tex_wrong_functional_category': not triggered\n  + hard_fail 'TCF7_stem_like_Tex_wrong_functional_category': not triggered\n  + hard_fail 'MKI67_state_wrong_functional_category': not triggered\n  + hard_fail 'Tn_or_Tn_Tmem_wrong_functional_category': not triggered\n  + hard_fail 'missing_exhausted_or_proliferating_anchor_in_singlet_top3': not triggered\n  + hard_fail 'missing_stemlike_precursor_in_singlet_top3': not triggered\n  + hard_fail 'missing_naive_or_early_memory_anchor_in_tumor_cluster_top3': not triggered\n  + hard_fail 'cf_reported_canonical_expansion_direction': not triggered\n  + hard_fail 'cf_expansion_lazy_defer': not triggered\n  + hard_fail 'cf_reported_canonical_expected_biology': not triggered\n  + hard_fail 'cf_biological_inference_null_artifact_or_lineage': not triggered\n  + hard_fail 'cf_exhaustion_inverted_prior_push': not triggered\n  + hard_fail 'cf_naive_inverted_prior_push': not triggered\n  + hard_fail 'cf_proliferation_inverted_prior_push': not triggered\n  x hard_fail 'cf_expansion_richness_inverted_prior_push': TRIGGERED\n  + hard_fail 'cf_multichain_inverted_prior_push': not triggered\n  x hard_fail 'cf_intermediate_gzmk_overgeneralized': TRIGGERED\n  + hard_fail 'cf_melanocytic_conjugate_overcall': not triggered\n  + hard_fail 'cf_relations_mass_hedge': not triggered",
      "metrics": {
        "hard_fail_triggered": [
          "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
          "cf_expansion_richness_inverted_prior_push",
          "cf_intermediate_gzmk_overgeneralized"
        ],
        "pass_rule": "all",
        "scoring_count": 5,
        "scoring_passed": 4,
        "scoring_total_score": 12
      },
      "fieldScores": {
        "biological_inference_data_true": 1,
        "clonotype_expansion_direction_data_true": 1,
        "dict_match": 5,
        "list_match": 2
      },
      "graderChecks": [
        {
          "name": "cd8_states_enriched_in_singlet_fraction",
          "kind": "list_match",
          "passed": true,
          "score": 3
        },
        {
          "name": "cd8_states_enriched_in_tumor_cluster_fraction",
          "kind": "list_match",
          "passed": true,
          "score": 2
        },
        {
          "name": "clonotype_expansion_direction_data_true",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "biological_inference_data_true",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "biological_relations",
          "kind": "list_match",
          "passed": false,
          "score": 5
        },
        {
          "name": "singlet_top3_contains_naive_state_prior_pushing",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "singlet_top3_contains_decoy_token",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "tumor_cluster_top3_contains_decoy_token",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "TOXhigh_Tex_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "TCF7_stem_like_Tex_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "MKI67_state_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "Tn_or_Tn_Tmem_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_stemlike_precursor_in_singlet_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_reported_canonical_expansion_direction",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_expansion_lazy_defer",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_reported_canonical_expected_biology",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_biological_inference_null_artifact_or_lineage",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_exhaustion_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_naive_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_proliferation_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_expansion_richness_inverted_prior_push",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "cf_multichain_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_intermediate_gzmk_overgeneralized",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "cf_melanocytic_conjugate_overcall",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_relations_mass_hedge",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "biological_inference": "singlet_antigen_experienced_cluster_bystander",
        "biological_relations": {
          "clonal_expansion_richness_in_singlets_vs_tumor_cluster": "less_than",
          "exhaustion_program_in_singlets_vs_tumor_cluster": "greater_than",
          "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": "greater_than",
          "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": "approximately_equal",
          "multichain_tcr_rate_in_singlets_vs_tumor_cluster": "approximately_equal",
          "naive_memory_program_in_singlets_vs_tumor_cluster": "less_than",
          "proliferation_program_in_singlets_vs_tumor_cluster": "greater_than"
        },
        "cd8_states_enriched_in_singlet_fraction": [
          {
            "functional_category": "stem_like_exhausted",
            "state_label": "TCF7_stem_like_Tex"
          },
          {
            "functional_category": "intermediate_exhausted_gzmk",
            "state_label": "GZMKhigh_Tex"
          },
          {
            "functional_category": "terminal_exhausted",
            "state_label": "LAG3high_Tex"
          }
        ],
        "cd8_states_enriched_in_tumor_cluster_fraction": [
          {
            "functional_category": "naive_memory",
            "state_label": "Tn_Tmem"
          },
          {
            "functional_category": "early_effector_memory",
            "state_label": "early_Tem_GZMK"
          },
          {
            "functional_category": "intermediate_exhausted_gzmk",
            "state_label": "GZMKhigh_Tex"
          }
        ],
        "clonotype_expansion_finding": "singlet_modestly_more_expanded_than_cluster"
      },
      "nTurns": 1,
      "checkpointSignals": {
        "clustering": true,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": false,
        "nerveJointAnchor": false,
        "multiomeAssembly": false,
        "depthNormalizedAtac": true,
        "regulonInference": false,
        "cd8SubsetIntegration": true,
        "perPatientEnrichment": true,
        "tcrRichnessMetrics": true,
        "orthologProjection": false,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": false,
        "mousePseudobulkDesign": true,
        "cancerEpithelialDenominator": true,
        "geneLevelDe": true,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "trajectoryFile": "scbench-long/trajectories/10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1__gpt-5.4__openai-codex__r2.json",
      "runId": "gpt-5.4__openai-codex__10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1__r2",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-gpt-5-4-codex",
      "modelName": "gpt-5.4",
      "harness": "openai-codex",
      "task": "10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": false,
      "score": 12,
      "durationS": 1175.58,
      "cost": 2.9427705,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_singlet_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "TOXhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "LAG3high_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "GZMKhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "intermediate_exhausted_gzmk"
                    ],
                    "canonical": "intermediate_exhausted_gzmk"
                  }
                }
              },
              {
                "key": "TCF7_stem_like_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "stem_like_exhausted"
                    ],
                    "canonical": "stem_like_exhausted"
                  }
                }
              },
              {
                "key": "MKI67_pos_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              },
              {
                "key": "MKI67high_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_tumor_cluster_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "Tn",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "Tn_Tmem",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "early_Tem_GZMK",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              },
              {
                "key": "Tem_GZMH",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              }
            ]
          },
          {
            "name": "clonotype_expansion_direction_data_true",
            "role": "gate",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_substantially_more_expanded_than_cluster",
              "singlet_modestly_more_expanded_than_cluster"
            ]
          },
          {
            "name": "biological_inference_data_true",
            "role": "gate",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_antigen_experienced_cluster_bystander"
            ]
          },
          {
            "name": "singlet_top3_contains_naive_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Tn",
              "Tn_Tmem",
              "early_Tem_GZMK",
              "Tem_GZMH"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "TOXhigh_Tex",
              "LAG3high_Tex",
              "GZMKhigh_Tex",
              "TCF7_stem_like_Tex",
              "MKI67_pos_Tex_Tprol",
              "MKI67high_Tex_Tprol"
            ]
          },
          {
            "name": "singlet_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "TOXhigh_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "TCF7_stem_like_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "MKI67_state_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "Tn_or_Tn_Tmem_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_stemlike_precursor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_reported_canonical_expansion_direction",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_substantially_more_expanded_than_singlet",
              "cluster_modestly_more_expanded_than_singlet"
            ]
          },
          {
            "name": "cf_expansion_lazy_defer",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "expansion_pattern_not_detectable"
            ]
          },
          {
            "name": "cf_reported_canonical_expected_biology",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_antigen_experienced_singlet_bystander"
            ]
          },
          {
            "name": "cf_biological_inference_null_artifact_or_lineage",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "differences_reflect_dissociation_or_sorting_artifact",
              "cluster_and_singlet_capture_same_biology",
              "cluster_and_singlet_capture_distinct_cd8_lineages"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "biological_relations",
            "kind": "dict",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_greater_than",
                  "greater_than"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "less_than",
                  "much_less_than"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than",
                  "approximately_equal"
                ]
              },
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal"
                ]
              },
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal",
                  "greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_exhaustion_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_naive_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_proliferation_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_expansion_richness_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_multichain_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_intermediate_gzmk_overgeneralized",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_melanocytic_conjugate_overcall",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "values": [
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_relations_mass_hedge",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              }
            }
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: FAIL\n  + list_match: list_match (answer_field='cd8_states_enriched_in_singlet_fraction'): PASS  (score=3.0)\n  + list_match: list_match (answer_field='cd8_states_enriched_in_tumor_cluster_fraction'): PASS  (score=2.0)\n  + clonotype_expansion_direction_data_true: passed=True, score=1.0\n  x biological_inference_data_true: passed=False, score=0.0\n  x dict_match: dict_match (answer_field='biological_relations'): FAIL  (score=6.0)\n  + hard_fail 'singlet_top3_contains_naive_state_prior_pushing': not triggered\n  x hard_fail 'tumor_cluster_top3_contains_exhausted_state_prior_pushing': TRIGGERED\n  + hard_fail 'singlet_top3_contains_decoy_token': not triggered\n  + hard_fail 'tumor_cluster_top3_contains_decoy_token': not triggered\n  + hard_fail 'TOXhigh_Tex_wrong_functional_category': not triggered\n  + hard_fail 'TCF7_stem_like_Tex_wrong_functional_category': not triggered\n  + hard_fail 'MKI67_state_wrong_functional_category': not triggered\n  + hard_fail 'Tn_or_Tn_Tmem_wrong_functional_category': not triggered\n  + hard_fail 'missing_exhausted_or_proliferating_anchor_in_singlet_top3': not triggered\n  + hard_fail 'missing_stemlike_precursor_in_singlet_top3': not triggered\n  + hard_fail 'missing_naive_or_early_memory_anchor_in_tumor_cluster_top3': not triggered\n  + hard_fail 'cf_reported_canonical_expansion_direction': not triggered\n  + hard_fail 'cf_expansion_lazy_defer': not triggered\n  + hard_fail 'cf_reported_canonical_expected_biology': not triggered\n  + hard_fail 'cf_biological_inference_null_artifact_or_lineage': not triggered\n  + hard_fail 'cf_exhaustion_inverted_prior_push': not triggered\n  + hard_fail 'cf_naive_inverted_prior_push': not triggered\n  + hard_fail 'cf_proliferation_inverted_prior_push': not triggered\n  + hard_fail 'cf_expansion_richness_inverted_prior_push': not triggered\n  + hard_fail 'cf_multichain_inverted_prior_push': not triggered\n  + hard_fail 'cf_intermediate_gzmk_overgeneralized': not triggered\n  + hard_fail 'cf_melanocytic_conjugate_overcall': not triggered\n  + hard_fail 'cf_relations_mass_hedge': not triggered",
      "metrics": {
        "hard_fail_triggered": [
          "tumor_cluster_top3_contains_exhausted_state_prior_pushing"
        ],
        "pass_rule": "all",
        "scoring_count": 5,
        "scoring_passed": 3,
        "scoring_total_score": 12
      },
      "fieldScores": {
        "biological_inference_data_true": 0,
        "clonotype_expansion_direction_data_true": 1,
        "dict_match": 6,
        "list_match": 2
      },
      "graderChecks": [
        {
          "name": "cd8_states_enriched_in_singlet_fraction",
          "kind": "list_match",
          "passed": true,
          "score": 3
        },
        {
          "name": "cd8_states_enriched_in_tumor_cluster_fraction",
          "kind": "list_match",
          "passed": true,
          "score": 2
        },
        {
          "name": "clonotype_expansion_direction_data_true",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "biological_inference_data_true",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "biological_relations",
          "kind": "list_match",
          "passed": false,
          "score": 6
        },
        {
          "name": "singlet_top3_contains_naive_state_prior_pushing",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "singlet_top3_contains_decoy_token",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "tumor_cluster_top3_contains_decoy_token",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "TOXhigh_Tex_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "TCF7_stem_like_Tex_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "MKI67_state_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "Tn_or_Tn_Tmem_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_stemlike_precursor_in_singlet_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_reported_canonical_expansion_direction",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_expansion_lazy_defer",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_reported_canonical_expected_biology",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_biological_inference_null_artifact_or_lineage",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_exhaustion_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_naive_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_proliferation_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_expansion_richness_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_multichain_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_intermediate_gzmk_overgeneralized",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_melanocytic_conjugate_overcall",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_relations_mass_hedge",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "biological_inference": "singlet_activated_cluster_resting",
        "biological_relations": {
          "clonal_expansion_richness_in_singlets_vs_tumor_cluster": "greater_than",
          "exhaustion_program_in_singlets_vs_tumor_cluster": "greater_than",
          "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": "less_than",
          "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": "approximately_equal",
          "multichain_tcr_rate_in_singlets_vs_tumor_cluster": "greater_than",
          "naive_memory_program_in_singlets_vs_tumor_cluster": "less_than",
          "proliferation_program_in_singlets_vs_tumor_cluster": "approximately_equal"
        },
        "cd8_states_enriched_in_singlet_fraction": [
          {
            "functional_category": "stem_like_exhausted",
            "state_label": "TCF7_stem_like_Tex"
          },
          {
            "functional_category": "terminal_exhausted",
            "state_label": "LAG3high_Tex"
          },
          {
            "functional_category": "proliferating",
            "state_label": "MKI67high_Tex_Tprol"
          }
        ],
        "cd8_states_enriched_in_tumor_cluster_fraction": [
          {
            "functional_category": "naive_memory",
            "state_label": "Tn_Tmem"
          },
          {
            "functional_category": "intermediate_exhausted_gzmk",
            "state_label": "GZMKhigh_Tex"
          },
          {
            "functional_category": "early_effector_memory",
            "state_label": "Tem_GZMH"
          }
        ],
        "clonotype_expansion_finding": "singlet_modestly_more_expanded_than_cluster"
      },
      "nTurns": 1,
      "checkpointSignals": {
        "clustering": true,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": false,
        "nerveJointAnchor": false,
        "multiomeAssembly": false,
        "depthNormalizedAtac": true,
        "regulonInference": true,
        "cd8SubsetIntegration": true,
        "perPatientEnrichment": true,
        "tcrRichnessMetrics": true,
        "orthologProjection": false,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": true,
        "mousePseudobulkDesign": false,
        "cancerEpithelialDenominator": true,
        "geneLevelDe": false,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "trajectoryFile": "scbench-long/trajectories/10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1__gpt-5.4__openai-codex__r3.json",
      "runId": "gpt-5.4__openai-codex__10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1__r3",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-gpt-5-4-pi",
      "modelName": "gpt-5.4",
      "harness": "pi",
      "task": "10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 1,
      "passed": false,
      "score": 14,
      "durationS": 2180.73,
      "cost": 2.1854134999999997,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_singlet_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "TOXhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "LAG3high_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "GZMKhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "intermediate_exhausted_gzmk"
                    ],
                    "canonical": "intermediate_exhausted_gzmk"
                  }
                }
              },
              {
                "key": "TCF7_stem_like_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "stem_like_exhausted"
                    ],
                    "canonical": "stem_like_exhausted"
                  }
                }
              },
              {
                "key": "MKI67_pos_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              },
              {
                "key": "MKI67high_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_tumor_cluster_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "Tn",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "Tn_Tmem",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "early_Tem_GZMK",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              },
              {
                "key": "Tem_GZMH",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              }
            ]
          },
          {
            "name": "clonotype_expansion_direction_data_true",
            "role": "gate",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_substantially_more_expanded_than_cluster",
              "singlet_modestly_more_expanded_than_cluster"
            ]
          },
          {
            "name": "biological_inference_data_true",
            "role": "gate",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_antigen_experienced_cluster_bystander"
            ]
          },
          {
            "name": "singlet_top3_contains_naive_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Tn",
              "Tn_Tmem",
              "early_Tem_GZMK",
              "Tem_GZMH"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "TOXhigh_Tex",
              "LAG3high_Tex",
              "GZMKhigh_Tex",
              "TCF7_stem_like_Tex",
              "MKI67_pos_Tex_Tprol",
              "MKI67high_Tex_Tprol"
            ]
          },
          {
            "name": "singlet_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "TOXhigh_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "TCF7_stem_like_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "MKI67_state_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "Tn_or_Tn_Tmem_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_stemlike_precursor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_reported_canonical_expansion_direction",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_substantially_more_expanded_than_singlet",
              "cluster_modestly_more_expanded_than_singlet"
            ]
          },
          {
            "name": "cf_expansion_lazy_defer",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "expansion_pattern_not_detectable"
            ]
          },
          {
            "name": "cf_reported_canonical_expected_biology",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_antigen_experienced_singlet_bystander"
            ]
          },
          {
            "name": "cf_biological_inference_null_artifact_or_lineage",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "differences_reflect_dissociation_or_sorting_artifact",
              "cluster_and_singlet_capture_same_biology",
              "cluster_and_singlet_capture_distinct_cd8_lineages"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "biological_relations",
            "kind": "dict",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_greater_than",
                  "greater_than"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "less_than",
                  "much_less_than"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than",
                  "approximately_equal"
                ]
              },
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal"
                ]
              },
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal",
                  "greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_exhaustion_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_naive_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_proliferation_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_expansion_richness_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_multichain_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_intermediate_gzmk_overgeneralized",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_melanocytic_conjugate_overcall",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "values": [
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_relations_mass_hedge",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              }
            }
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: FAIL\n  + list_match: list_match (answer_field='cd8_states_enriched_in_singlet_fraction'): PASS  (score=3.0)\n  + list_match: list_match (answer_field='cd8_states_enriched_in_tumor_cluster_fraction'): PASS  (score=3.0)\n  + clonotype_expansion_direction_data_true: passed=True, score=1.0\n  + biological_inference_data_true: passed=True, score=1.0\n  x dict_match: dict_match (answer_field='biological_relations'): FAIL  (score=6.0)\n  + hard_fail 'singlet_top3_contains_naive_state_prior_pushing': not triggered\n  + hard_fail 'tumor_cluster_top3_contains_exhausted_state_prior_pushing': not triggered\n  + hard_fail 'singlet_top3_contains_decoy_token': not triggered\n  + hard_fail 'tumor_cluster_top3_contains_decoy_token': not triggered\n  + hard_fail 'TOXhigh_Tex_wrong_functional_category': not triggered\n  + hard_fail 'TCF7_stem_like_Tex_wrong_functional_category': not triggered\n  + hard_fail 'MKI67_state_wrong_functional_category': not triggered\n  + hard_fail 'Tn_or_Tn_Tmem_wrong_functional_category': not triggered\n  + hard_fail 'missing_exhausted_or_proliferating_anchor_in_singlet_top3': not triggered\n  + hard_fail 'missing_stemlike_precursor_in_singlet_top3': not triggered\n  + hard_fail 'missing_naive_or_early_memory_anchor_in_tumor_cluster_top3': not triggered\n  + hard_fail 'cf_reported_canonical_expansion_direction': not triggered\n  + hard_fail 'cf_expansion_lazy_defer': not triggered\n  + hard_fail 'cf_reported_canonical_expected_biology': not triggered\n  + hard_fail 'cf_biological_inference_null_artifact_or_lineage': not triggered\n  + hard_fail 'cf_exhaustion_inverted_prior_push': not triggered\n  + hard_fail 'cf_naive_inverted_prior_push': not triggered\n  + hard_fail 'cf_proliferation_inverted_prior_push': not triggered\n  + hard_fail 'cf_expansion_richness_inverted_prior_push': not triggered\n  + hard_fail 'cf_multichain_inverted_prior_push': not triggered\n  x hard_fail 'cf_intermediate_gzmk_overgeneralized': TRIGGERED\n  + hard_fail 'cf_melanocytic_conjugate_overcall': not triggered\n  + hard_fail 'cf_relations_mass_hedge': not triggered",
      "metrics": {
        "hard_fail_triggered": [
          "cf_intermediate_gzmk_overgeneralized"
        ],
        "pass_rule": "all",
        "scoring_count": 5,
        "scoring_passed": 4,
        "scoring_total_score": 14
      },
      "fieldScores": {
        "biological_inference_data_true": 1,
        "clonotype_expansion_direction_data_true": 1,
        "dict_match": 6,
        "list_match": 3
      },
      "graderChecks": [
        {
          "name": "cd8_states_enriched_in_singlet_fraction",
          "kind": "list_match",
          "passed": true,
          "score": 3
        },
        {
          "name": "cd8_states_enriched_in_tumor_cluster_fraction",
          "kind": "list_match",
          "passed": true,
          "score": 3
        },
        {
          "name": "clonotype_expansion_direction_data_true",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "biological_inference_data_true",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "biological_relations",
          "kind": "list_match",
          "passed": false,
          "score": 6
        },
        {
          "name": "singlet_top3_contains_naive_state_prior_pushing",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "singlet_top3_contains_decoy_token",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "tumor_cluster_top3_contains_decoy_token",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "TOXhigh_Tex_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "TCF7_stem_like_Tex_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "MKI67_state_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "Tn_or_Tn_Tmem_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_stemlike_precursor_in_singlet_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_reported_canonical_expansion_direction",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_expansion_lazy_defer",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_reported_canonical_expected_biology",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_biological_inference_null_artifact_or_lineage",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_exhaustion_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_naive_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_proliferation_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_expansion_richness_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_multichain_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_intermediate_gzmk_overgeneralized",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "cf_melanocytic_conjugate_overcall",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_relations_mass_hedge",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "biological_inference": "singlet_antigen_experienced_cluster_bystander",
        "biological_relations": {
          "clonal_expansion_richness_in_singlets_vs_tumor_cluster": "greater_than",
          "exhaustion_program_in_singlets_vs_tumor_cluster": "greater_than",
          "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": "greater_than",
          "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": "approximately_equal",
          "multichain_tcr_rate_in_singlets_vs_tumor_cluster": "greater_than",
          "naive_memory_program_in_singlets_vs_tumor_cluster": "less_than",
          "proliferation_program_in_singlets_vs_tumor_cluster": "greater_than"
        },
        "cd8_states_enriched_in_singlet_fraction": [
          {
            "functional_category": "stem_like_exhausted",
            "state_label": "TCF7_stem_like_Tex"
          },
          {
            "functional_category": "intermediate_exhausted_gzmk",
            "state_label": "GZMKhigh_Tex"
          },
          {
            "functional_category": "terminal_exhausted",
            "state_label": "LAG3high_Tex"
          }
        ],
        "cd8_states_enriched_in_tumor_cluster_fraction": [
          {
            "functional_category": "naive_memory",
            "state_label": "Tn_Tmem"
          },
          {
            "functional_category": "early_effector_memory",
            "state_label": "early_Tem_GZMK"
          },
          {
            "functional_category": "early_effector_memory",
            "state_label": "Tem_GZMH"
          }
        ],
        "clonotype_expansion_finding": "singlet_modestly_more_expanded_than_cluster"
      },
      "nTurns": 50,
      "checkpointSignals": {
        "clustering": true,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": true,
        "nerveJointAnchor": false,
        "multiomeAssembly": true,
        "depthNormalizedAtac": true,
        "regulonInference": true,
        "cd8SubsetIntegration": true,
        "perPatientEnrichment": true,
        "tcrRichnessMetrics": true,
        "orthologProjection": false,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": true,
        "mousePseudobulkDesign": true,
        "cancerEpithelialDenominator": true,
        "geneLevelDe": true,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "trajectoryFile": "scbench-long/trajectories/10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1__gpt-5.4__pi__r1.json",
      "runId": "gpt-5.4__pi__10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1__r1",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-gpt-5-4-pi",
      "modelName": "gpt-5.4",
      "harness": "pi",
      "task": "10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 2,
      "passed": false,
      "score": 13,
      "durationS": 1666.41,
      "cost": 1.6531635,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_singlet_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "TOXhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "LAG3high_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "GZMKhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "intermediate_exhausted_gzmk"
                    ],
                    "canonical": "intermediate_exhausted_gzmk"
                  }
                }
              },
              {
                "key": "TCF7_stem_like_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "stem_like_exhausted"
                    ],
                    "canonical": "stem_like_exhausted"
                  }
                }
              },
              {
                "key": "MKI67_pos_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              },
              {
                "key": "MKI67high_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_tumor_cluster_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "Tn",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "Tn_Tmem",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "early_Tem_GZMK",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              },
              {
                "key": "Tem_GZMH",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              }
            ]
          },
          {
            "name": "clonotype_expansion_direction_data_true",
            "role": "gate",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_substantially_more_expanded_than_cluster",
              "singlet_modestly_more_expanded_than_cluster"
            ]
          },
          {
            "name": "biological_inference_data_true",
            "role": "gate",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_antigen_experienced_cluster_bystander"
            ]
          },
          {
            "name": "singlet_top3_contains_naive_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Tn",
              "Tn_Tmem",
              "early_Tem_GZMK",
              "Tem_GZMH"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "TOXhigh_Tex",
              "LAG3high_Tex",
              "GZMKhigh_Tex",
              "TCF7_stem_like_Tex",
              "MKI67_pos_Tex_Tprol",
              "MKI67high_Tex_Tprol"
            ]
          },
          {
            "name": "singlet_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "TOXhigh_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "TCF7_stem_like_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "MKI67_state_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "Tn_or_Tn_Tmem_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_stemlike_precursor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_reported_canonical_expansion_direction",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_substantially_more_expanded_than_singlet",
              "cluster_modestly_more_expanded_than_singlet"
            ]
          },
          {
            "name": "cf_expansion_lazy_defer",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "expansion_pattern_not_detectable"
            ]
          },
          {
            "name": "cf_reported_canonical_expected_biology",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_antigen_experienced_singlet_bystander"
            ]
          },
          {
            "name": "cf_biological_inference_null_artifact_or_lineage",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "differences_reflect_dissociation_or_sorting_artifact",
              "cluster_and_singlet_capture_same_biology",
              "cluster_and_singlet_capture_distinct_cd8_lineages"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "biological_relations",
            "kind": "dict",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_greater_than",
                  "greater_than"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "less_than",
                  "much_less_than"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than",
                  "approximately_equal"
                ]
              },
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal"
                ]
              },
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal",
                  "greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_exhaustion_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_naive_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_proliferation_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_expansion_richness_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_multichain_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_intermediate_gzmk_overgeneralized",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_melanocytic_conjugate_overcall",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "values": [
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_relations_mass_hedge",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              }
            }
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: FAIL\n  + list_match: list_match (answer_field='cd8_states_enriched_in_singlet_fraction'): PASS  (score=3.0)\n  + list_match: list_match (answer_field='cd8_states_enriched_in_tumor_cluster_fraction'): PASS  (score=3.0)\n  + clonotype_expansion_direction_data_true: passed=True, score=1.0\n  + biological_inference_data_true: passed=True, score=1.0\n  x dict_match: dict_match (answer_field='biological_relations'): FAIL  (score=5.0)\n  + hard_fail 'singlet_top3_contains_naive_state_prior_pushing': not triggered\n  + hard_fail 'tumor_cluster_top3_contains_exhausted_state_prior_pushing': not triggered\n  + hard_fail 'singlet_top3_contains_decoy_token': not triggered\n  + hard_fail 'tumor_cluster_top3_contains_decoy_token': not triggered\n  + hard_fail 'TOXhigh_Tex_wrong_functional_category': not triggered\n  + hard_fail 'TCF7_stem_like_Tex_wrong_functional_category': not triggered\n  + hard_fail 'MKI67_state_wrong_functional_category': not triggered\n  + hard_fail 'Tn_or_Tn_Tmem_wrong_functional_category': not triggered\n  + hard_fail 'missing_exhausted_or_proliferating_anchor_in_singlet_top3': not triggered\n  + hard_fail 'missing_stemlike_precursor_in_singlet_top3': not triggered\n  + hard_fail 'missing_naive_or_early_memory_anchor_in_tumor_cluster_top3': not triggered\n  + hard_fail 'cf_reported_canonical_expansion_direction': not triggered\n  + hard_fail 'cf_expansion_lazy_defer': not triggered\n  + hard_fail 'cf_reported_canonical_expected_biology': not triggered\n  + hard_fail 'cf_biological_inference_null_artifact_or_lineage': not triggered\n  + hard_fail 'cf_exhaustion_inverted_prior_push': not triggered\n  + hard_fail 'cf_naive_inverted_prior_push': not triggered\n  + hard_fail 'cf_proliferation_inverted_prior_push': not triggered\n  + hard_fail 'cf_expansion_richness_inverted_prior_push': not triggered\n  x hard_fail 'cf_multichain_inverted_prior_push': TRIGGERED\n  x hard_fail 'cf_intermediate_gzmk_overgeneralized': TRIGGERED\n  + hard_fail 'cf_melanocytic_conjugate_overcall': not triggered\n  + hard_fail 'cf_relations_mass_hedge': not triggered",
      "metrics": {
        "hard_fail_triggered": [
          "cf_multichain_inverted_prior_push",
          "cf_intermediate_gzmk_overgeneralized"
        ],
        "pass_rule": "all",
        "scoring_count": 5,
        "scoring_passed": 4,
        "scoring_total_score": 13
      },
      "fieldScores": {
        "biological_inference_data_true": 1,
        "clonotype_expansion_direction_data_true": 1,
        "dict_match": 5,
        "list_match": 3
      },
      "graderChecks": [
        {
          "name": "cd8_states_enriched_in_singlet_fraction",
          "kind": "list_match",
          "passed": true,
          "score": 3
        },
        {
          "name": "cd8_states_enriched_in_tumor_cluster_fraction",
          "kind": "list_match",
          "passed": true,
          "score": 3
        },
        {
          "name": "clonotype_expansion_direction_data_true",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "biological_inference_data_true",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "biological_relations",
          "kind": "list_match",
          "passed": false,
          "score": 5
        },
        {
          "name": "singlet_top3_contains_naive_state_prior_pushing",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "singlet_top3_contains_decoy_token",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "tumor_cluster_top3_contains_decoy_token",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "TOXhigh_Tex_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "TCF7_stem_like_Tex_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "MKI67_state_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "Tn_or_Tn_Tmem_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_stemlike_precursor_in_singlet_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_reported_canonical_expansion_direction",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_expansion_lazy_defer",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_reported_canonical_expected_biology",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_biological_inference_null_artifact_or_lineage",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_exhaustion_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_naive_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_proliferation_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_expansion_richness_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_multichain_inverted_prior_push",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "cf_intermediate_gzmk_overgeneralized",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "cf_melanocytic_conjugate_overcall",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_relations_mass_hedge",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "biological_inference": "singlet_antigen_experienced_cluster_bystander",
        "biological_relations": {
          "clonal_expansion_richness_in_singlets_vs_tumor_cluster": "greater_than",
          "exhaustion_program_in_singlets_vs_tumor_cluster": "greater_than",
          "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": "greater_than",
          "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": "greater_than",
          "multichain_tcr_rate_in_singlets_vs_tumor_cluster": "less_than",
          "naive_memory_program_in_singlets_vs_tumor_cluster": "less_than",
          "proliferation_program_in_singlets_vs_tumor_cluster": "greater_than"
        },
        "cd8_states_enriched_in_singlet_fraction": [
          {
            "functional_category": "stem_like_exhausted",
            "state_label": "TCF7_stem_like_Tex"
          },
          {
            "functional_category": "intermediate_exhausted_gzmk",
            "state_label": "GZMKhigh_Tex"
          },
          {
            "functional_category": "terminal_exhausted",
            "state_label": "LAG3high_Tex"
          }
        ],
        "cd8_states_enriched_in_tumor_cluster_fraction": [
          {
            "functional_category": "naive_memory",
            "state_label": "Tn_Tmem"
          },
          {
            "functional_category": "early_effector_memory",
            "state_label": "early_Tem_GZMK"
          },
          {
            "functional_category": "early_effector_memory",
            "state_label": "Tem_GZMH"
          }
        ],
        "clonotype_expansion_finding": "singlet_modestly_more_expanded_than_cluster"
      },
      "nTurns": 35,
      "checkpointSignals": {
        "clustering": true,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": true,
        "nerveJointAnchor": false,
        "multiomeAssembly": false,
        "depthNormalizedAtac": true,
        "regulonInference": false,
        "cd8SubsetIntegration": true,
        "perPatientEnrichment": true,
        "tcrRichnessMetrics": true,
        "orthologProjection": false,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": true,
        "mousePseudobulkDesign": true,
        "cancerEpithelialDenominator": false,
        "geneLevelDe": true,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "trajectoryFile": "scbench-long/trajectories/10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1__gpt-5.4__pi__r2.json",
      "runId": "gpt-5.4__pi__10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1__r2",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-gpt-5-4-pi",
      "modelName": "gpt-5.4",
      "harness": "pi",
      "task": "10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": true,
      "score": 15,
      "durationS": 1589,
      "cost": 2.3057044999999996,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_singlet_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "TOXhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "LAG3high_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "GZMKhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "intermediate_exhausted_gzmk"
                    ],
                    "canonical": "intermediate_exhausted_gzmk"
                  }
                }
              },
              {
                "key": "TCF7_stem_like_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "stem_like_exhausted"
                    ],
                    "canonical": "stem_like_exhausted"
                  }
                }
              },
              {
                "key": "MKI67_pos_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              },
              {
                "key": "MKI67high_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_tumor_cluster_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "Tn",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "Tn_Tmem",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "early_Tem_GZMK",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              },
              {
                "key": "Tem_GZMH",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              }
            ]
          },
          {
            "name": "clonotype_expansion_direction_data_true",
            "role": "gate",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_substantially_more_expanded_than_cluster",
              "singlet_modestly_more_expanded_than_cluster"
            ]
          },
          {
            "name": "biological_inference_data_true",
            "role": "gate",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_antigen_experienced_cluster_bystander"
            ]
          },
          {
            "name": "singlet_top3_contains_naive_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Tn",
              "Tn_Tmem",
              "early_Tem_GZMK",
              "Tem_GZMH"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "TOXhigh_Tex",
              "LAG3high_Tex",
              "GZMKhigh_Tex",
              "TCF7_stem_like_Tex",
              "MKI67_pos_Tex_Tprol",
              "MKI67high_Tex_Tprol"
            ]
          },
          {
            "name": "singlet_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "TOXhigh_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "TCF7_stem_like_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "MKI67_state_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "Tn_or_Tn_Tmem_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_stemlike_precursor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_reported_canonical_expansion_direction",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_substantially_more_expanded_than_singlet",
              "cluster_modestly_more_expanded_than_singlet"
            ]
          },
          {
            "name": "cf_expansion_lazy_defer",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "expansion_pattern_not_detectable"
            ]
          },
          {
            "name": "cf_reported_canonical_expected_biology",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_antigen_experienced_singlet_bystander"
            ]
          },
          {
            "name": "cf_biological_inference_null_artifact_or_lineage",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "differences_reflect_dissociation_or_sorting_artifact",
              "cluster_and_singlet_capture_same_biology",
              "cluster_and_singlet_capture_distinct_cd8_lineages"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "biological_relations",
            "kind": "dict",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_greater_than",
                  "greater_than"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "less_than",
                  "much_less_than"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than",
                  "approximately_equal"
                ]
              },
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal"
                ]
              },
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal",
                  "greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_exhaustion_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_naive_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_proliferation_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_expansion_richness_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_multichain_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_intermediate_gzmk_overgeneralized",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_melanocytic_conjugate_overcall",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "values": [
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_relations_mass_hedge",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              }
            }
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: PASS\n  + list_match: list_match (answer_field='cd8_states_enriched_in_singlet_fraction'): PASS  (score=3.0)\n  + list_match: list_match (answer_field='cd8_states_enriched_in_tumor_cluster_fraction'): PASS  (score=3.0)\n  + clonotype_expansion_direction_data_true: passed=True, score=1.0\n  + biological_inference_data_true: passed=True, score=1.0\n  + dict_match: dict_match (answer_field='biological_relations'): PASS  (score=7.0)\n  + hard_fail 'singlet_top3_contains_naive_state_prior_pushing': not triggered\n  + hard_fail 'tumor_cluster_top3_contains_exhausted_state_prior_pushing': not triggered\n  + hard_fail 'singlet_top3_contains_decoy_token': not triggered\n  + hard_fail 'tumor_cluster_top3_contains_decoy_token': not triggered\n  + hard_fail 'TOXhigh_Tex_wrong_functional_category': not triggered\n  + hard_fail 'TCF7_stem_like_Tex_wrong_functional_category': not triggered\n  + hard_fail 'MKI67_state_wrong_functional_category': not triggered\n  + hard_fail 'Tn_or_Tn_Tmem_wrong_functional_category': not triggered\n  + hard_fail 'missing_exhausted_or_proliferating_anchor_in_singlet_top3': not triggered\n  + hard_fail 'missing_stemlike_precursor_in_singlet_top3': not triggered\n  + hard_fail 'missing_naive_or_early_memory_anchor_in_tumor_cluster_top3': not triggered\n  + hard_fail 'cf_reported_canonical_expansion_direction': not triggered\n  + hard_fail 'cf_expansion_lazy_defer': not triggered\n  + hard_fail 'cf_reported_canonical_expected_biology': not triggered\n  + hard_fail 'cf_biological_inference_null_artifact_or_lineage': not triggered\n  + hard_fail 'cf_exhaustion_inverted_prior_push': not triggered\n  + hard_fail 'cf_naive_inverted_prior_push': not triggered\n  + hard_fail 'cf_proliferation_inverted_prior_push': not triggered\n  + hard_fail 'cf_expansion_richness_inverted_prior_push': not triggered\n  + hard_fail 'cf_multichain_inverted_prior_push': not triggered\n  + hard_fail 'cf_intermediate_gzmk_overgeneralized': not triggered\n  + hard_fail 'cf_melanocytic_conjugate_overcall': not triggered\n  + hard_fail 'cf_relations_mass_hedge': not triggered",
      "metrics": {
        "hard_fail_triggered": [],
        "pass_rule": "all",
        "scoring_count": 5,
        "scoring_passed": 5,
        "scoring_total_score": 15
      },
      "fieldScores": {
        "biological_inference_data_true": 1,
        "clonotype_expansion_direction_data_true": 1,
        "dict_match": 7,
        "list_match": 3
      },
      "graderChecks": [
        {
          "name": "cd8_states_enriched_in_singlet_fraction",
          "kind": "list_match",
          "passed": true,
          "score": 3
        },
        {
          "name": "cd8_states_enriched_in_tumor_cluster_fraction",
          "kind": "list_match",
          "passed": true,
          "score": 3
        },
        {
          "name": "clonotype_expansion_direction_data_true",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "biological_inference_data_true",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "biological_relations",
          "kind": "list_match",
          "passed": true,
          "score": 7
        },
        {
          "name": "singlet_top3_contains_naive_state_prior_pushing",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "singlet_top3_contains_decoy_token",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "tumor_cluster_top3_contains_decoy_token",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "TOXhigh_Tex_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "TCF7_stem_like_Tex_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "MKI67_state_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "Tn_or_Tn_Tmem_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_stemlike_precursor_in_singlet_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_reported_canonical_expansion_direction",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_expansion_lazy_defer",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_reported_canonical_expected_biology",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_biological_inference_null_artifact_or_lineage",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_exhaustion_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_naive_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_proliferation_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_expansion_richness_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_multichain_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_intermediate_gzmk_overgeneralized",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_melanocytic_conjugate_overcall",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_relations_mass_hedge",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "biological_inference": "singlet_antigen_experienced_cluster_bystander",
        "biological_relations": {
          "clonal_expansion_richness_in_singlets_vs_tumor_cluster": "greater_than",
          "exhaustion_program_in_singlets_vs_tumor_cluster": "greater_than",
          "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": "approximately_equal",
          "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": "approximately_equal",
          "multichain_tcr_rate_in_singlets_vs_tumor_cluster": "approximately_equal",
          "naive_memory_program_in_singlets_vs_tumor_cluster": "less_than",
          "proliferation_program_in_singlets_vs_tumor_cluster": "greater_than"
        },
        "cd8_states_enriched_in_singlet_fraction": [
          {
            "functional_category": "stem_like_exhausted",
            "state_label": "TCF7_stem_like_Tex"
          },
          {
            "functional_category": "intermediate_exhausted_gzmk",
            "state_label": "GZMKhigh_Tex"
          },
          {
            "functional_category": "terminal_exhausted",
            "state_label": "LAG3high_Tex"
          }
        ],
        "cd8_states_enriched_in_tumor_cluster_fraction": [
          {
            "functional_category": "naive_memory",
            "state_label": "Tn"
          },
          {
            "functional_category": "naive_memory",
            "state_label": "Tn_Tmem"
          },
          {
            "functional_category": "early_effector_memory",
            "state_label": "early_Tem_GZMK"
          }
        ],
        "clonotype_expansion_finding": "singlet_modestly_more_expanded_than_cluster"
      },
      "nTurns": 50,
      "checkpointSignals": {
        "clustering": true,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": true,
        "nerveJointAnchor": false,
        "multiomeAssembly": true,
        "depthNormalizedAtac": false,
        "regulonInference": true,
        "cd8SubsetIntegration": true,
        "perPatientEnrichment": true,
        "tcrRichnessMetrics": true,
        "orthologProjection": false,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": true,
        "mousePseudobulkDesign": false,
        "cancerEpithelialDenominator": false,
        "geneLevelDe": false,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "trajectoryFile": "scbench-long/trajectories/10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1__gpt-5.4__pi__r3.json",
      "runId": "gpt-5.4__pi__10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1__r3",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-gpt-5-5-codex",
      "modelName": "gpt-5.5",
      "harness": "openai-codex",
      "task": "10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 1,
      "passed": false,
      "score": 14,
      "durationS": 522.18,
      "cost": 1.900979,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_singlet_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "TOXhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "LAG3high_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "GZMKhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "intermediate_exhausted_gzmk"
                    ],
                    "canonical": "intermediate_exhausted_gzmk"
                  }
                }
              },
              {
                "key": "TCF7_stem_like_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "stem_like_exhausted"
                    ],
                    "canonical": "stem_like_exhausted"
                  }
                }
              },
              {
                "key": "MKI67_pos_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              },
              {
                "key": "MKI67high_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_tumor_cluster_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "Tn",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "Tn_Tmem",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "early_Tem_GZMK",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              },
              {
                "key": "Tem_GZMH",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              }
            ]
          },
          {
            "name": "clonotype_expansion_direction_data_true",
            "role": "gate",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_substantially_more_expanded_than_cluster",
              "singlet_modestly_more_expanded_than_cluster"
            ]
          },
          {
            "name": "biological_inference_data_true",
            "role": "gate",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_antigen_experienced_cluster_bystander"
            ]
          },
          {
            "name": "singlet_top3_contains_naive_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Tn",
              "Tn_Tmem",
              "early_Tem_GZMK",
              "Tem_GZMH"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "TOXhigh_Tex",
              "LAG3high_Tex",
              "GZMKhigh_Tex",
              "TCF7_stem_like_Tex",
              "MKI67_pos_Tex_Tprol",
              "MKI67high_Tex_Tprol"
            ]
          },
          {
            "name": "singlet_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "TOXhigh_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "TCF7_stem_like_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "MKI67_state_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "Tn_or_Tn_Tmem_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_stemlike_precursor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_reported_canonical_expansion_direction",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_substantially_more_expanded_than_singlet",
              "cluster_modestly_more_expanded_than_singlet"
            ]
          },
          {
            "name": "cf_expansion_lazy_defer",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "expansion_pattern_not_detectable"
            ]
          },
          {
            "name": "cf_reported_canonical_expected_biology",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_antigen_experienced_singlet_bystander"
            ]
          },
          {
            "name": "cf_biological_inference_null_artifact_or_lineage",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "differences_reflect_dissociation_or_sorting_artifact",
              "cluster_and_singlet_capture_same_biology",
              "cluster_and_singlet_capture_distinct_cd8_lineages"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "biological_relations",
            "kind": "dict",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_greater_than",
                  "greater_than"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "less_than",
                  "much_less_than"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than",
                  "approximately_equal"
                ]
              },
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal"
                ]
              },
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal",
                  "greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_exhaustion_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_naive_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_proliferation_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_expansion_richness_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_multichain_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_intermediate_gzmk_overgeneralized",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_melanocytic_conjugate_overcall",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "values": [
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_relations_mass_hedge",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              }
            }
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: FAIL\n  + list_match: list_match (answer_field='cd8_states_enriched_in_singlet_fraction'): PASS  (score=3.0)\n  + list_match: list_match (answer_field='cd8_states_enriched_in_tumor_cluster_fraction'): PASS  (score=3.0)\n  + clonotype_expansion_direction_data_true: passed=True, score=1.0\n  + biological_inference_data_true: passed=True, score=1.0\n  x dict_match: dict_match (answer_field='biological_relations'): FAIL  (score=6.0)\n  + hard_fail 'singlet_top3_contains_naive_state_prior_pushing': not triggered\n  + hard_fail 'tumor_cluster_top3_contains_exhausted_state_prior_pushing': not triggered\n  + hard_fail 'singlet_top3_contains_decoy_token': not triggered\n  + hard_fail 'tumor_cluster_top3_contains_decoy_token': not triggered\n  + hard_fail 'TOXhigh_Tex_wrong_functional_category': not triggered\n  + hard_fail 'TCF7_stem_like_Tex_wrong_functional_category': not triggered\n  + hard_fail 'MKI67_state_wrong_functional_category': not triggered\n  + hard_fail 'Tn_or_Tn_Tmem_wrong_functional_category': not triggered\n  + hard_fail 'missing_exhausted_or_proliferating_anchor_in_singlet_top3': not triggered\n  + hard_fail 'missing_stemlike_precursor_in_singlet_top3': not triggered\n  + hard_fail 'missing_naive_or_early_memory_anchor_in_tumor_cluster_top3': not triggered\n  + hard_fail 'cf_reported_canonical_expansion_direction': not triggered\n  + hard_fail 'cf_expansion_lazy_defer': not triggered\n  + hard_fail 'cf_reported_canonical_expected_biology': not triggered\n  + hard_fail 'cf_biological_inference_null_artifact_or_lineage': not triggered\n  + hard_fail 'cf_exhaustion_inverted_prior_push': not triggered\n  + hard_fail 'cf_naive_inverted_prior_push': not triggered\n  + hard_fail 'cf_proliferation_inverted_prior_push': not triggered\n  + hard_fail 'cf_expansion_richness_inverted_prior_push': not triggered\n  + hard_fail 'cf_multichain_inverted_prior_push': not triggered\n  x hard_fail 'cf_intermediate_gzmk_overgeneralized': TRIGGERED\n  + hard_fail 'cf_melanocytic_conjugate_overcall': not triggered\n  + hard_fail 'cf_relations_mass_hedge': not triggered",
      "metrics": {
        "hard_fail_triggered": [
          "cf_intermediate_gzmk_overgeneralized"
        ],
        "pass_rule": "all",
        "scoring_count": 5,
        "scoring_passed": 4,
        "scoring_total_score": 14
      },
      "fieldScores": {
        "biological_inference_data_true": 1,
        "clonotype_expansion_direction_data_true": 1,
        "dict_match": 6,
        "list_match": 3
      },
      "graderChecks": [
        {
          "name": "cd8_states_enriched_in_singlet_fraction",
          "kind": "list_match",
          "passed": true,
          "score": 3
        },
        {
          "name": "cd8_states_enriched_in_tumor_cluster_fraction",
          "kind": "list_match",
          "passed": true,
          "score": 3
        },
        {
          "name": "clonotype_expansion_direction_data_true",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "biological_inference_data_true",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "biological_relations",
          "kind": "list_match",
          "passed": false,
          "score": 6
        },
        {
          "name": "singlet_top3_contains_naive_state_prior_pushing",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "singlet_top3_contains_decoy_token",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "tumor_cluster_top3_contains_decoy_token",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "TOXhigh_Tex_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "TCF7_stem_like_Tex_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "MKI67_state_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "Tn_or_Tn_Tmem_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_stemlike_precursor_in_singlet_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_reported_canonical_expansion_direction",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_expansion_lazy_defer",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_reported_canonical_expected_biology",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_biological_inference_null_artifact_or_lineage",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_exhaustion_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_naive_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_proliferation_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_expansion_richness_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_multichain_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_intermediate_gzmk_overgeneralized",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "cf_melanocytic_conjugate_overcall",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_relations_mass_hedge",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "biological_inference": "singlet_antigen_experienced_cluster_bystander",
        "biological_relations": {
          "clonal_expansion_richness_in_singlets_vs_tumor_cluster": "greater_than",
          "exhaustion_program_in_singlets_vs_tumor_cluster": "greater_than",
          "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": "greater_than",
          "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": "less_than",
          "multichain_tcr_rate_in_singlets_vs_tumor_cluster": "greater_than",
          "naive_memory_program_in_singlets_vs_tumor_cluster": "less_than",
          "proliferation_program_in_singlets_vs_tumor_cluster": "greater_than"
        },
        "cd8_states_enriched_in_singlet_fraction": [
          {
            "functional_category": "stem_like_exhausted",
            "state_label": "TCF7_stem_like_Tex"
          },
          {
            "functional_category": "intermediate_exhausted_gzmk",
            "state_label": "GZMKhigh_Tex"
          },
          {
            "functional_category": "terminal_exhausted",
            "state_label": "LAG3high_Tex"
          }
        ],
        "cd8_states_enriched_in_tumor_cluster_fraction": [
          {
            "functional_category": "naive_memory",
            "state_label": "Tn"
          },
          {
            "functional_category": "naive_memory",
            "state_label": "Tn_Tmem"
          },
          {
            "functional_category": "early_effector_memory",
            "state_label": "early_Tem_GZMK"
          }
        ],
        "clonotype_expansion_finding": "singlet_modestly_more_expanded_than_cluster"
      },
      "nTurns": 1,
      "checkpointSignals": {
        "clustering": false,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": true,
        "nerveJointAnchor": false,
        "multiomeAssembly": false,
        "depthNormalizedAtac": true,
        "regulonInference": false,
        "cd8SubsetIntegration": true,
        "perPatientEnrichment": true,
        "tcrRichnessMetrics": true,
        "orthologProjection": false,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": false,
        "mousePseudobulkDesign": false,
        "cancerEpithelialDenominator": true,
        "geneLevelDe": false,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "trajectoryFile": "scbench-long/trajectories/10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1__gpt-5.5__openai-codex__r1.json",
      "runId": "gpt-5.5__openai-codex__10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1__r1",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-gpt-5-5-codex",
      "modelName": "gpt-5.5",
      "harness": "openai-codex",
      "task": "10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 2,
      "passed": false,
      "score": 13,
      "durationS": 1379.3,
      "cost": 1.812786,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_singlet_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "TOXhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "LAG3high_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "GZMKhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "intermediate_exhausted_gzmk"
                    ],
                    "canonical": "intermediate_exhausted_gzmk"
                  }
                }
              },
              {
                "key": "TCF7_stem_like_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "stem_like_exhausted"
                    ],
                    "canonical": "stem_like_exhausted"
                  }
                }
              },
              {
                "key": "MKI67_pos_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              },
              {
                "key": "MKI67high_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_tumor_cluster_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "Tn",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "Tn_Tmem",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "early_Tem_GZMK",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              },
              {
                "key": "Tem_GZMH",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              }
            ]
          },
          {
            "name": "clonotype_expansion_direction_data_true",
            "role": "gate",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_substantially_more_expanded_than_cluster",
              "singlet_modestly_more_expanded_than_cluster"
            ]
          },
          {
            "name": "biological_inference_data_true",
            "role": "gate",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_antigen_experienced_cluster_bystander"
            ]
          },
          {
            "name": "singlet_top3_contains_naive_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Tn",
              "Tn_Tmem",
              "early_Tem_GZMK",
              "Tem_GZMH"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "TOXhigh_Tex",
              "LAG3high_Tex",
              "GZMKhigh_Tex",
              "TCF7_stem_like_Tex",
              "MKI67_pos_Tex_Tprol",
              "MKI67high_Tex_Tprol"
            ]
          },
          {
            "name": "singlet_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "TOXhigh_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "TCF7_stem_like_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "MKI67_state_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "Tn_or_Tn_Tmem_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_stemlike_precursor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_reported_canonical_expansion_direction",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_substantially_more_expanded_than_singlet",
              "cluster_modestly_more_expanded_than_singlet"
            ]
          },
          {
            "name": "cf_expansion_lazy_defer",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "expansion_pattern_not_detectable"
            ]
          },
          {
            "name": "cf_reported_canonical_expected_biology",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_antigen_experienced_singlet_bystander"
            ]
          },
          {
            "name": "cf_biological_inference_null_artifact_or_lineage",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "differences_reflect_dissociation_or_sorting_artifact",
              "cluster_and_singlet_capture_same_biology",
              "cluster_and_singlet_capture_distinct_cd8_lineages"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "biological_relations",
            "kind": "dict",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_greater_than",
                  "greater_than"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "less_than",
                  "much_less_than"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than",
                  "approximately_equal"
                ]
              },
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal"
                ]
              },
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal",
                  "greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_exhaustion_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_naive_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_proliferation_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_expansion_richness_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_multichain_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_intermediate_gzmk_overgeneralized",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_melanocytic_conjugate_overcall",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "values": [
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_relations_mass_hedge",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              }
            }
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: FAIL\n  + list_match: list_match (answer_field='cd8_states_enriched_in_singlet_fraction'): PASS  (score=3.0)\n  + list_match: list_match (answer_field='cd8_states_enriched_in_tumor_cluster_fraction'): PASS  (score=2.0)\n  + clonotype_expansion_direction_data_true: passed=True, score=1.0\n  + biological_inference_data_true: passed=True, score=1.0\n  x dict_match: dict_match (answer_field='biological_relations'): FAIL  (score=6.0)\n  + hard_fail 'singlet_top3_contains_naive_state_prior_pushing': not triggered\n  + hard_fail 'tumor_cluster_top3_contains_exhausted_state_prior_pushing': not triggered\n  + hard_fail 'singlet_top3_contains_decoy_token': not triggered\n  + hard_fail 'tumor_cluster_top3_contains_decoy_token': not triggered\n  + hard_fail 'TOXhigh_Tex_wrong_functional_category': not triggered\n  + hard_fail 'TCF7_stem_like_Tex_wrong_functional_category': not triggered\n  + hard_fail 'MKI67_state_wrong_functional_category': not triggered\n  + hard_fail 'Tn_or_Tn_Tmem_wrong_functional_category': not triggered\n  + hard_fail 'missing_exhausted_or_proliferating_anchor_in_singlet_top3': not triggered\n  + hard_fail 'missing_stemlike_precursor_in_singlet_top3': not triggered\n  + hard_fail 'missing_naive_or_early_memory_anchor_in_tumor_cluster_top3': not triggered\n  + hard_fail 'cf_reported_canonical_expansion_direction': not triggered\n  + hard_fail 'cf_expansion_lazy_defer': not triggered\n  + hard_fail 'cf_reported_canonical_expected_biology': not triggered\n  + hard_fail 'cf_biological_inference_null_artifact_or_lineage': not triggered\n  + hard_fail 'cf_exhaustion_inverted_prior_push': not triggered\n  + hard_fail 'cf_naive_inverted_prior_push': not triggered\n  + hard_fail 'cf_proliferation_inverted_prior_push': not triggered\n  + hard_fail 'cf_expansion_richness_inverted_prior_push': not triggered\n  + hard_fail 'cf_multichain_inverted_prior_push': not triggered\n  x hard_fail 'cf_intermediate_gzmk_overgeneralized': TRIGGERED\n  + hard_fail 'cf_melanocytic_conjugate_overcall': not triggered\n  + hard_fail 'cf_relations_mass_hedge': not triggered",
      "metrics": {
        "hard_fail_triggered": [
          "cf_intermediate_gzmk_overgeneralized"
        ],
        "pass_rule": "all",
        "scoring_count": 5,
        "scoring_passed": 4,
        "scoring_total_score": 13
      },
      "fieldScores": {
        "biological_inference_data_true": 1,
        "clonotype_expansion_direction_data_true": 1,
        "dict_match": 6,
        "list_match": 2
      },
      "graderChecks": [
        {
          "name": "cd8_states_enriched_in_singlet_fraction",
          "kind": "list_match",
          "passed": true,
          "score": 3
        },
        {
          "name": "cd8_states_enriched_in_tumor_cluster_fraction",
          "kind": "list_match",
          "passed": true,
          "score": 2
        },
        {
          "name": "clonotype_expansion_direction_data_true",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "biological_inference_data_true",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "biological_relations",
          "kind": "list_match",
          "passed": false,
          "score": 6
        },
        {
          "name": "singlet_top3_contains_naive_state_prior_pushing",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "singlet_top3_contains_decoy_token",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "tumor_cluster_top3_contains_decoy_token",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "TOXhigh_Tex_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "TCF7_stem_like_Tex_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "MKI67_state_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "Tn_or_Tn_Tmem_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_stemlike_precursor_in_singlet_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_reported_canonical_expansion_direction",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_expansion_lazy_defer",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_reported_canonical_expected_biology",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_biological_inference_null_artifact_or_lineage",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_exhaustion_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_naive_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_proliferation_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_expansion_richness_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_multichain_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_intermediate_gzmk_overgeneralized",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "cf_melanocytic_conjugate_overcall",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_relations_mass_hedge",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "biological_inference": "singlet_antigen_experienced_cluster_bystander",
        "biological_relations": {
          "clonal_expansion_richness_in_singlets_vs_tumor_cluster": "greater_than",
          "exhaustion_program_in_singlets_vs_tumor_cluster": "greater_than",
          "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": "greater_than",
          "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": "less_than",
          "multichain_tcr_rate_in_singlets_vs_tumor_cluster": "approximately_equal",
          "naive_memory_program_in_singlets_vs_tumor_cluster": "less_than",
          "proliferation_program_in_singlets_vs_tumor_cluster": "greater_than"
        },
        "cd8_states_enriched_in_singlet_fraction": [
          {
            "functional_category": "stem_like_exhausted",
            "state_label": "TCF7_stem_like_Tex"
          },
          {
            "functional_category": "intermediate_exhausted_gzmk",
            "state_label": "GZMKhigh_Tex"
          },
          {
            "functional_category": "terminal_exhausted",
            "state_label": "TOXhigh_Tex"
          }
        ],
        "cd8_states_enriched_in_tumor_cluster_fraction": [
          {
            "functional_category": "naive_memory",
            "state_label": "Tn"
          },
          {
            "functional_category": "naive_memory",
            "state_label": "Tn_Tmem"
          },
          {
            "functional_category": "innate_like_tc17_mait",
            "state_label": "Tc17_MAIT"
          }
        ],
        "clonotype_expansion_finding": "singlet_modestly_more_expanded_than_cluster"
      },
      "nTurns": 1,
      "checkpointSignals": {
        "clustering": false,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": true,
        "nerveJointAnchor": false,
        "multiomeAssembly": false,
        "depthNormalizedAtac": false,
        "regulonInference": false,
        "cd8SubsetIntegration": true,
        "perPatientEnrichment": true,
        "tcrRichnessMetrics": true,
        "orthologProjection": false,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": true,
        "mousePseudobulkDesign": false,
        "cancerEpithelialDenominator": false,
        "geneLevelDe": false,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "trajectoryFile": "scbench-long/trajectories/10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1__gpt-5.5__openai-codex__r2.json",
      "runId": "gpt-5.5__openai-codex__10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1__r2",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-gpt-5-5-codex",
      "modelName": "gpt-5.5",
      "harness": "openai-codex",
      "task": "10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": true,
      "score": 14,
      "durationS": 940.22,
      "cost": 2.093881,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_singlet_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "TOXhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "LAG3high_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "GZMKhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "intermediate_exhausted_gzmk"
                    ],
                    "canonical": "intermediate_exhausted_gzmk"
                  }
                }
              },
              {
                "key": "TCF7_stem_like_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "stem_like_exhausted"
                    ],
                    "canonical": "stem_like_exhausted"
                  }
                }
              },
              {
                "key": "MKI67_pos_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              },
              {
                "key": "MKI67high_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_tumor_cluster_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "Tn",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "Tn_Tmem",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "early_Tem_GZMK",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              },
              {
                "key": "Tem_GZMH",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              }
            ]
          },
          {
            "name": "clonotype_expansion_direction_data_true",
            "role": "gate",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_substantially_more_expanded_than_cluster",
              "singlet_modestly_more_expanded_than_cluster"
            ]
          },
          {
            "name": "biological_inference_data_true",
            "role": "gate",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_antigen_experienced_cluster_bystander"
            ]
          },
          {
            "name": "singlet_top3_contains_naive_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Tn",
              "Tn_Tmem",
              "early_Tem_GZMK",
              "Tem_GZMH"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "TOXhigh_Tex",
              "LAG3high_Tex",
              "GZMKhigh_Tex",
              "TCF7_stem_like_Tex",
              "MKI67_pos_Tex_Tprol",
              "MKI67high_Tex_Tprol"
            ]
          },
          {
            "name": "singlet_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "TOXhigh_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "TCF7_stem_like_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "MKI67_state_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "Tn_or_Tn_Tmem_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_stemlike_precursor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_reported_canonical_expansion_direction",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_substantially_more_expanded_than_singlet",
              "cluster_modestly_more_expanded_than_singlet"
            ]
          },
          {
            "name": "cf_expansion_lazy_defer",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "expansion_pattern_not_detectable"
            ]
          },
          {
            "name": "cf_reported_canonical_expected_biology",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_antigen_experienced_singlet_bystander"
            ]
          },
          {
            "name": "cf_biological_inference_null_artifact_or_lineage",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "differences_reflect_dissociation_or_sorting_artifact",
              "cluster_and_singlet_capture_same_biology",
              "cluster_and_singlet_capture_distinct_cd8_lineages"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "biological_relations",
            "kind": "dict",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_greater_than",
                  "greater_than"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "less_than",
                  "much_less_than"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than",
                  "approximately_equal"
                ]
              },
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal"
                ]
              },
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal",
                  "greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_exhaustion_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_naive_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_proliferation_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_expansion_richness_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_multichain_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_intermediate_gzmk_overgeneralized",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_melanocytic_conjugate_overcall",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "values": [
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_relations_mass_hedge",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              }
            }
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: PASS\n  + list_match: list_match (answer_field='cd8_states_enriched_in_singlet_fraction'): PASS  (score=3.0)\n  + list_match: list_match (answer_field='cd8_states_enriched_in_tumor_cluster_fraction'): PASS  (score=2.0)\n  + clonotype_expansion_direction_data_true: passed=True, score=1.0\n  + biological_inference_data_true: passed=True, score=1.0\n  + dict_match: dict_match (answer_field='biological_relations'): PASS  (score=7.0)\n  + hard_fail 'singlet_top3_contains_naive_state_prior_pushing': not triggered\n  + hard_fail 'tumor_cluster_top3_contains_exhausted_state_prior_pushing': not triggered\n  + hard_fail 'singlet_top3_contains_decoy_token': not triggered\n  + hard_fail 'tumor_cluster_top3_contains_decoy_token': not triggered\n  + hard_fail 'TOXhigh_Tex_wrong_functional_category': not triggered\n  + hard_fail 'TCF7_stem_like_Tex_wrong_functional_category': not triggered\n  + hard_fail 'MKI67_state_wrong_functional_category': not triggered\n  + hard_fail 'Tn_or_Tn_Tmem_wrong_functional_category': not triggered\n  + hard_fail 'missing_exhausted_or_proliferating_anchor_in_singlet_top3': not triggered\n  + hard_fail 'missing_stemlike_precursor_in_singlet_top3': not triggered\n  + hard_fail 'missing_naive_or_early_memory_anchor_in_tumor_cluster_top3': not triggered\n  + hard_fail 'cf_reported_canonical_expansion_direction': not triggered\n  + hard_fail 'cf_expansion_lazy_defer': not triggered\n  + hard_fail 'cf_reported_canonical_expected_biology': not triggered\n  + hard_fail 'cf_biological_inference_null_artifact_or_lineage': not triggered\n  + hard_fail 'cf_exhaustion_inverted_prior_push': not triggered\n  + hard_fail 'cf_naive_inverted_prior_push': not triggered\n  + hard_fail 'cf_proliferation_inverted_prior_push': not triggered\n  + hard_fail 'cf_expansion_richness_inverted_prior_push': not triggered\n  + hard_fail 'cf_multichain_inverted_prior_push': not triggered\n  + hard_fail 'cf_intermediate_gzmk_overgeneralized': not triggered\n  + hard_fail 'cf_melanocytic_conjugate_overcall': not triggered\n  + hard_fail 'cf_relations_mass_hedge': not triggered",
      "metrics": {
        "hard_fail_triggered": [],
        "pass_rule": "all",
        "scoring_count": 5,
        "scoring_passed": 5,
        "scoring_total_score": 14
      },
      "fieldScores": {
        "biological_inference_data_true": 1,
        "clonotype_expansion_direction_data_true": 1,
        "dict_match": 7,
        "list_match": 2
      },
      "graderChecks": [
        {
          "name": "cd8_states_enriched_in_singlet_fraction",
          "kind": "list_match",
          "passed": true,
          "score": 3
        },
        {
          "name": "cd8_states_enriched_in_tumor_cluster_fraction",
          "kind": "list_match",
          "passed": true,
          "score": 2
        },
        {
          "name": "clonotype_expansion_direction_data_true",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "biological_inference_data_true",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "biological_relations",
          "kind": "list_match",
          "passed": true,
          "score": 7
        },
        {
          "name": "singlet_top3_contains_naive_state_prior_pushing",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "singlet_top3_contains_decoy_token",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "tumor_cluster_top3_contains_decoy_token",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "TOXhigh_Tex_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "TCF7_stem_like_Tex_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "MKI67_state_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "Tn_or_Tn_Tmem_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_stemlike_precursor_in_singlet_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_reported_canonical_expansion_direction",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_expansion_lazy_defer",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_reported_canonical_expected_biology",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_biological_inference_null_artifact_or_lineage",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_exhaustion_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_naive_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_proliferation_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_expansion_richness_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_multichain_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_intermediate_gzmk_overgeneralized",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_melanocytic_conjugate_overcall",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_relations_mass_hedge",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "biological_inference": "singlet_antigen_experienced_cluster_bystander",
        "biological_relations": {
          "clonal_expansion_richness_in_singlets_vs_tumor_cluster": "greater_than",
          "exhaustion_program_in_singlets_vs_tumor_cluster": "greater_than",
          "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": "approximately_equal",
          "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": "approximately_equal",
          "multichain_tcr_rate_in_singlets_vs_tumor_cluster": "approximately_equal",
          "naive_memory_program_in_singlets_vs_tumor_cluster": "less_than",
          "proliferation_program_in_singlets_vs_tumor_cluster": "greater_than"
        },
        "cd8_states_enriched_in_singlet_fraction": [
          {
            "functional_category": "stem_like_exhausted",
            "state_label": "TCF7_stem_like_Tex"
          },
          {
            "functional_category": "intermediate_exhausted_gzmk",
            "state_label": "GZMKhigh_Tex"
          },
          {
            "functional_category": "terminal_exhausted",
            "state_label": "TOXhigh_Tex"
          }
        ],
        "cd8_states_enriched_in_tumor_cluster_fraction": [
          {
            "functional_category": "naive_memory",
            "state_label": "Tn_Tmem"
          },
          {
            "functional_category": "naive_memory",
            "state_label": "Tn"
          },
          {
            "functional_category": "early_effector_memory",
            "state_label": "Tem_NK_like"
          }
        ],
        "clonotype_expansion_finding": "singlet_modestly_more_expanded_than_cluster"
      },
      "nTurns": 1,
      "checkpointSignals": {
        "clustering": false,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": true,
        "nerveJointAnchor": false,
        "multiomeAssembly": true,
        "depthNormalizedAtac": true,
        "regulonInference": true,
        "cd8SubsetIntegration": true,
        "perPatientEnrichment": true,
        "tcrRichnessMetrics": true,
        "orthologProjection": false,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": false,
        "mousePseudobulkDesign": false,
        "cancerEpithelialDenominator": true,
        "geneLevelDe": false,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "trajectoryFile": "scbench-long/trajectories/10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1__gpt-5.5__openai-codex__r3.json",
      "runId": "gpt-5.5__openai-codex__10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1__r3",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-gpt-5-5-pi",
      "modelName": "gpt-5.5",
      "harness": "pi",
      "task": "10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 1,
      "passed": false,
      "score": 14,
      "durationS": 844.41,
      "cost": 1.6138150000000002,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_singlet_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "TOXhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "LAG3high_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "GZMKhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "intermediate_exhausted_gzmk"
                    ],
                    "canonical": "intermediate_exhausted_gzmk"
                  }
                }
              },
              {
                "key": "TCF7_stem_like_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "stem_like_exhausted"
                    ],
                    "canonical": "stem_like_exhausted"
                  }
                }
              },
              {
                "key": "MKI67_pos_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              },
              {
                "key": "MKI67high_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_tumor_cluster_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "Tn",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "Tn_Tmem",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "early_Tem_GZMK",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              },
              {
                "key": "Tem_GZMH",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              }
            ]
          },
          {
            "name": "clonotype_expansion_direction_data_true",
            "role": "gate",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_substantially_more_expanded_than_cluster",
              "singlet_modestly_more_expanded_than_cluster"
            ]
          },
          {
            "name": "biological_inference_data_true",
            "role": "gate",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_antigen_experienced_cluster_bystander"
            ]
          },
          {
            "name": "singlet_top3_contains_naive_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Tn",
              "Tn_Tmem",
              "early_Tem_GZMK",
              "Tem_GZMH"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "TOXhigh_Tex",
              "LAG3high_Tex",
              "GZMKhigh_Tex",
              "TCF7_stem_like_Tex",
              "MKI67_pos_Tex_Tprol",
              "MKI67high_Tex_Tprol"
            ]
          },
          {
            "name": "singlet_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "TOXhigh_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "TCF7_stem_like_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "MKI67_state_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "Tn_or_Tn_Tmem_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_stemlike_precursor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_reported_canonical_expansion_direction",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_substantially_more_expanded_than_singlet",
              "cluster_modestly_more_expanded_than_singlet"
            ]
          },
          {
            "name": "cf_expansion_lazy_defer",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "expansion_pattern_not_detectable"
            ]
          },
          {
            "name": "cf_reported_canonical_expected_biology",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_antigen_experienced_singlet_bystander"
            ]
          },
          {
            "name": "cf_biological_inference_null_artifact_or_lineage",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "differences_reflect_dissociation_or_sorting_artifact",
              "cluster_and_singlet_capture_same_biology",
              "cluster_and_singlet_capture_distinct_cd8_lineages"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "biological_relations",
            "kind": "dict",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_greater_than",
                  "greater_than"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "less_than",
                  "much_less_than"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than",
                  "approximately_equal"
                ]
              },
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal"
                ]
              },
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal",
                  "greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_exhaustion_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_naive_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_proliferation_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_expansion_richness_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_multichain_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_intermediate_gzmk_overgeneralized",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_melanocytic_conjugate_overcall",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "values": [
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_relations_mass_hedge",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              }
            }
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: FAIL\n  + list_match: list_match (answer_field='cd8_states_enriched_in_singlet_fraction'): PASS  (score=3.0)\n  + list_match: list_match (answer_field='cd8_states_enriched_in_tumor_cluster_fraction'): PASS  (score=3.0)\n  + clonotype_expansion_direction_data_true: passed=True, score=1.0\n  + biological_inference_data_true: passed=True, score=1.0\n  x dict_match: dict_match (answer_field='biological_relations'): FAIL  (score=6.0)\n  + hard_fail 'singlet_top3_contains_naive_state_prior_pushing': not triggered\n  + hard_fail 'tumor_cluster_top3_contains_exhausted_state_prior_pushing': not triggered\n  + hard_fail 'singlet_top3_contains_decoy_token': not triggered\n  + hard_fail 'tumor_cluster_top3_contains_decoy_token': not triggered\n  + hard_fail 'TOXhigh_Tex_wrong_functional_category': not triggered\n  + hard_fail 'TCF7_stem_like_Tex_wrong_functional_category': not triggered\n  + hard_fail 'MKI67_state_wrong_functional_category': not triggered\n  + hard_fail 'Tn_or_Tn_Tmem_wrong_functional_category': not triggered\n  + hard_fail 'missing_exhausted_or_proliferating_anchor_in_singlet_top3': not triggered\n  + hard_fail 'missing_stemlike_precursor_in_singlet_top3': not triggered\n  + hard_fail 'missing_naive_or_early_memory_anchor_in_tumor_cluster_top3': not triggered\n  + hard_fail 'cf_reported_canonical_expansion_direction': not triggered\n  + hard_fail 'cf_expansion_lazy_defer': not triggered\n  + hard_fail 'cf_reported_canonical_expected_biology': not triggered\n  + hard_fail 'cf_biological_inference_null_artifact_or_lineage': not triggered\n  + hard_fail 'cf_exhaustion_inverted_prior_push': not triggered\n  + hard_fail 'cf_naive_inverted_prior_push': not triggered\n  + hard_fail 'cf_proliferation_inverted_prior_push': not triggered\n  + hard_fail 'cf_expansion_richness_inverted_prior_push': not triggered\n  + hard_fail 'cf_multichain_inverted_prior_push': not triggered\n  x hard_fail 'cf_intermediate_gzmk_overgeneralized': TRIGGERED\n  + hard_fail 'cf_melanocytic_conjugate_overcall': not triggered\n  + hard_fail 'cf_relations_mass_hedge': not triggered",
      "metrics": {
        "hard_fail_triggered": [
          "cf_intermediate_gzmk_overgeneralized"
        ],
        "pass_rule": "all",
        "scoring_count": 5,
        "scoring_passed": 4,
        "scoring_total_score": 14
      },
      "fieldScores": {
        "biological_inference_data_true": 1,
        "clonotype_expansion_direction_data_true": 1,
        "dict_match": 6,
        "list_match": 3
      },
      "graderChecks": [
        {
          "name": "cd8_states_enriched_in_singlet_fraction",
          "kind": "list_match",
          "passed": true,
          "score": 3
        },
        {
          "name": "cd8_states_enriched_in_tumor_cluster_fraction",
          "kind": "list_match",
          "passed": true,
          "score": 3
        },
        {
          "name": "clonotype_expansion_direction_data_true",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "biological_inference_data_true",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "biological_relations",
          "kind": "list_match",
          "passed": false,
          "score": 6
        },
        {
          "name": "singlet_top3_contains_naive_state_prior_pushing",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "singlet_top3_contains_decoy_token",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "tumor_cluster_top3_contains_decoy_token",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "TOXhigh_Tex_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "TCF7_stem_like_Tex_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "MKI67_state_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "Tn_or_Tn_Tmem_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_stemlike_precursor_in_singlet_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_reported_canonical_expansion_direction",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_expansion_lazy_defer",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_reported_canonical_expected_biology",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_biological_inference_null_artifact_or_lineage",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_exhaustion_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_naive_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_proliferation_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_expansion_richness_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_multichain_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_intermediate_gzmk_overgeneralized",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "cf_melanocytic_conjugate_overcall",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_relations_mass_hedge",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "biological_inference": "singlet_antigen_experienced_cluster_bystander",
        "biological_relations": {
          "clonal_expansion_richness_in_singlets_vs_tumor_cluster": "greater_than",
          "exhaustion_program_in_singlets_vs_tumor_cluster": "greater_than",
          "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": "greater_than",
          "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": "approximately_equal",
          "multichain_tcr_rate_in_singlets_vs_tumor_cluster": "greater_than",
          "naive_memory_program_in_singlets_vs_tumor_cluster": "less_than",
          "proliferation_program_in_singlets_vs_tumor_cluster": "greater_than"
        },
        "cd8_states_enriched_in_singlet_fraction": [
          {
            "functional_category": "stem_like_exhausted",
            "state_label": "TCF7_stem_like_Tex"
          },
          {
            "functional_category": "intermediate_exhausted_gzmk",
            "state_label": "GZMKhigh_Tex"
          },
          {
            "functional_category": "terminal_exhausted",
            "state_label": "TOXhigh_Tex"
          }
        ],
        "cd8_states_enriched_in_tumor_cluster_fraction": [
          {
            "functional_category": "naive_memory",
            "state_label": "Tn"
          },
          {
            "functional_category": "naive_memory",
            "state_label": "Tn_Tmem"
          },
          {
            "functional_category": "early_effector_memory",
            "state_label": "early_Tem_GZMK"
          }
        ],
        "clonotype_expansion_finding": "singlet_modestly_more_expanded_than_cluster"
      },
      "nTurns": 23,
      "checkpointSignals": {
        "clustering": true,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": true,
        "nerveJointAnchor": false,
        "multiomeAssembly": false,
        "depthNormalizedAtac": false,
        "regulonInference": false,
        "cd8SubsetIntegration": true,
        "perPatientEnrichment": true,
        "tcrRichnessMetrics": true,
        "orthologProjection": false,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": false,
        "mousePseudobulkDesign": false,
        "cancerEpithelialDenominator": false,
        "geneLevelDe": false,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "trajectoryFile": "scbench-long/trajectories/10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1__gpt-5.5__pi__r1.json",
      "runId": "gpt-5.5__pi__10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1__r1",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-gpt-5-5-pi",
      "modelName": "gpt-5.5",
      "harness": "pi",
      "task": "10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 2,
      "passed": false,
      "score": 14,
      "durationS": 714.34,
      "cost": 2.0988529999999996,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_singlet_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "TOXhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "LAG3high_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "GZMKhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "intermediate_exhausted_gzmk"
                    ],
                    "canonical": "intermediate_exhausted_gzmk"
                  }
                }
              },
              {
                "key": "TCF7_stem_like_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "stem_like_exhausted"
                    ],
                    "canonical": "stem_like_exhausted"
                  }
                }
              },
              {
                "key": "MKI67_pos_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              },
              {
                "key": "MKI67high_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_tumor_cluster_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "Tn",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "Tn_Tmem",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "early_Tem_GZMK",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              },
              {
                "key": "Tem_GZMH",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              }
            ]
          },
          {
            "name": "clonotype_expansion_direction_data_true",
            "role": "gate",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_substantially_more_expanded_than_cluster",
              "singlet_modestly_more_expanded_than_cluster"
            ]
          },
          {
            "name": "biological_inference_data_true",
            "role": "gate",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_antigen_experienced_cluster_bystander"
            ]
          },
          {
            "name": "singlet_top3_contains_naive_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Tn",
              "Tn_Tmem",
              "early_Tem_GZMK",
              "Tem_GZMH"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "TOXhigh_Tex",
              "LAG3high_Tex",
              "GZMKhigh_Tex",
              "TCF7_stem_like_Tex",
              "MKI67_pos_Tex_Tprol",
              "MKI67high_Tex_Tprol"
            ]
          },
          {
            "name": "singlet_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "TOXhigh_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "TCF7_stem_like_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "MKI67_state_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "Tn_or_Tn_Tmem_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_stemlike_precursor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_reported_canonical_expansion_direction",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_substantially_more_expanded_than_singlet",
              "cluster_modestly_more_expanded_than_singlet"
            ]
          },
          {
            "name": "cf_expansion_lazy_defer",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "expansion_pattern_not_detectable"
            ]
          },
          {
            "name": "cf_reported_canonical_expected_biology",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_antigen_experienced_singlet_bystander"
            ]
          },
          {
            "name": "cf_biological_inference_null_artifact_or_lineage",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "differences_reflect_dissociation_or_sorting_artifact",
              "cluster_and_singlet_capture_same_biology",
              "cluster_and_singlet_capture_distinct_cd8_lineages"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "biological_relations",
            "kind": "dict",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_greater_than",
                  "greater_than"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "less_than",
                  "much_less_than"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than",
                  "approximately_equal"
                ]
              },
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal"
                ]
              },
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal",
                  "greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_exhaustion_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_naive_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_proliferation_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_expansion_richness_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_multichain_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_intermediate_gzmk_overgeneralized",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_melanocytic_conjugate_overcall",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "values": [
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_relations_mass_hedge",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              }
            }
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: FAIL\n  + list_match: list_match (answer_field='cd8_states_enriched_in_singlet_fraction'): PASS  (score=3.0)\n  + list_match: list_match (answer_field='cd8_states_enriched_in_tumor_cluster_fraction'): PASS  (score=3.0)\n  + clonotype_expansion_direction_data_true: passed=True, score=1.0\n  + biological_inference_data_true: passed=True, score=1.0\n  x dict_match: dict_match (answer_field='biological_relations'): FAIL  (score=6.0)\n  + hard_fail 'singlet_top3_contains_naive_state_prior_pushing': not triggered\n  + hard_fail 'tumor_cluster_top3_contains_exhausted_state_prior_pushing': not triggered\n  + hard_fail 'singlet_top3_contains_decoy_token': not triggered\n  + hard_fail 'tumor_cluster_top3_contains_decoy_token': not triggered\n  + hard_fail 'TOXhigh_Tex_wrong_functional_category': not triggered\n  + hard_fail 'TCF7_stem_like_Tex_wrong_functional_category': not triggered\n  + hard_fail 'MKI67_state_wrong_functional_category': not triggered\n  + hard_fail 'Tn_or_Tn_Tmem_wrong_functional_category': not triggered\n  + hard_fail 'missing_exhausted_or_proliferating_anchor_in_singlet_top3': not triggered\n  + hard_fail 'missing_stemlike_precursor_in_singlet_top3': not triggered\n  + hard_fail 'missing_naive_or_early_memory_anchor_in_tumor_cluster_top3': not triggered\n  + hard_fail 'cf_reported_canonical_expansion_direction': not triggered\n  + hard_fail 'cf_expansion_lazy_defer': not triggered\n  + hard_fail 'cf_reported_canonical_expected_biology': not triggered\n  + hard_fail 'cf_biological_inference_null_artifact_or_lineage': not triggered\n  + hard_fail 'cf_exhaustion_inverted_prior_push': not triggered\n  + hard_fail 'cf_naive_inverted_prior_push': not triggered\n  + hard_fail 'cf_proliferation_inverted_prior_push': not triggered\n  + hard_fail 'cf_expansion_richness_inverted_prior_push': not triggered\n  + hard_fail 'cf_multichain_inverted_prior_push': not triggered\n  x hard_fail 'cf_intermediate_gzmk_overgeneralized': TRIGGERED\n  + hard_fail 'cf_melanocytic_conjugate_overcall': not triggered\n  + hard_fail 'cf_relations_mass_hedge': not triggered",
      "metrics": {
        "hard_fail_triggered": [
          "cf_intermediate_gzmk_overgeneralized"
        ],
        "pass_rule": "all",
        "scoring_count": 5,
        "scoring_passed": 4,
        "scoring_total_score": 14
      },
      "fieldScores": {
        "biological_inference_data_true": 1,
        "clonotype_expansion_direction_data_true": 1,
        "dict_match": 6,
        "list_match": 3
      },
      "graderChecks": [
        {
          "name": "cd8_states_enriched_in_singlet_fraction",
          "kind": "list_match",
          "passed": true,
          "score": 3
        },
        {
          "name": "cd8_states_enriched_in_tumor_cluster_fraction",
          "kind": "list_match",
          "passed": true,
          "score": 3
        },
        {
          "name": "clonotype_expansion_direction_data_true",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "biological_inference_data_true",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "biological_relations",
          "kind": "list_match",
          "passed": false,
          "score": 6
        },
        {
          "name": "singlet_top3_contains_naive_state_prior_pushing",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "singlet_top3_contains_decoy_token",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "tumor_cluster_top3_contains_decoy_token",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "TOXhigh_Tex_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "TCF7_stem_like_Tex_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "MKI67_state_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "Tn_or_Tn_Tmem_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_stemlike_precursor_in_singlet_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_reported_canonical_expansion_direction",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_expansion_lazy_defer",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_reported_canonical_expected_biology",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_biological_inference_null_artifact_or_lineage",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_exhaustion_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_naive_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_proliferation_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_expansion_richness_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_multichain_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_intermediate_gzmk_overgeneralized",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "cf_melanocytic_conjugate_overcall",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_relations_mass_hedge",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "biological_inference": "singlet_antigen_experienced_cluster_bystander",
        "biological_relations": {
          "clonal_expansion_richness_in_singlets_vs_tumor_cluster": "greater_than",
          "exhaustion_program_in_singlets_vs_tumor_cluster": "greater_than",
          "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": "greater_than",
          "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": "less_than",
          "multichain_tcr_rate_in_singlets_vs_tumor_cluster": "greater_than",
          "naive_memory_program_in_singlets_vs_tumor_cluster": "less_than",
          "proliferation_program_in_singlets_vs_tumor_cluster": "greater_than"
        },
        "cd8_states_enriched_in_singlet_fraction": [
          {
            "functional_category": "stem_like_exhausted",
            "state_label": "TCF7_stem_like_Tex"
          },
          {
            "functional_category": "intermediate_exhausted_gzmk",
            "state_label": "GZMKhigh_Tex"
          },
          {
            "functional_category": "terminal_exhausted",
            "state_label": "TOXhigh_Tex"
          }
        ],
        "cd8_states_enriched_in_tumor_cluster_fraction": [
          {
            "functional_category": "naive_memory",
            "state_label": "Tn"
          },
          {
            "functional_category": "naive_memory",
            "state_label": "Tn_Tmem"
          },
          {
            "functional_category": "early_effector_memory",
            "state_label": "early_Tem_GZMK"
          }
        ],
        "clonotype_expansion_finding": "singlet_modestly_more_expanded_than_cluster"
      },
      "nTurns": 23,
      "checkpointSignals": {
        "clustering": true,
        "markerAnnotation": true,
        "scoreBasedAnnotation": true,
        "normalization": false,
        "nerveJointAnchor": false,
        "multiomeAssembly": true,
        "depthNormalizedAtac": true,
        "regulonInference": true,
        "cd8SubsetIntegration": true,
        "perPatientEnrichment": true,
        "tcrRichnessMetrics": true,
        "orthologProjection": false,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": false,
        "mousePseudobulkDesign": false,
        "cancerEpithelialDenominator": false,
        "geneLevelDe": true,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "trajectoryFile": "scbench-long/trajectories/10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1__gpt-5.5__pi__r2.json",
      "runId": "gpt-5.5__pi__10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1__r2",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-gpt-5-5-pi",
      "modelName": "gpt-5.5",
      "harness": "pi",
      "task": "10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": false,
      "score": 14,
      "durationS": 2007.57,
      "cost": 1.924292,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_singlet_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "TOXhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "LAG3high_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "GZMKhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "intermediate_exhausted_gzmk"
                    ],
                    "canonical": "intermediate_exhausted_gzmk"
                  }
                }
              },
              {
                "key": "TCF7_stem_like_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "stem_like_exhausted"
                    ],
                    "canonical": "stem_like_exhausted"
                  }
                }
              },
              {
                "key": "MKI67_pos_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              },
              {
                "key": "MKI67high_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_tumor_cluster_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "Tn",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "Tn_Tmem",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "early_Tem_GZMK",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              },
              {
                "key": "Tem_GZMH",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              }
            ]
          },
          {
            "name": "clonotype_expansion_direction_data_true",
            "role": "gate",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_substantially_more_expanded_than_cluster",
              "singlet_modestly_more_expanded_than_cluster"
            ]
          },
          {
            "name": "biological_inference_data_true",
            "role": "gate",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_antigen_experienced_cluster_bystander"
            ]
          },
          {
            "name": "singlet_top3_contains_naive_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Tn",
              "Tn_Tmem",
              "early_Tem_GZMK",
              "Tem_GZMH"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "TOXhigh_Tex",
              "LAG3high_Tex",
              "GZMKhigh_Tex",
              "TCF7_stem_like_Tex",
              "MKI67_pos_Tex_Tprol",
              "MKI67high_Tex_Tprol"
            ]
          },
          {
            "name": "singlet_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "TOXhigh_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "TCF7_stem_like_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "MKI67_state_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "Tn_or_Tn_Tmem_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_stemlike_precursor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_reported_canonical_expansion_direction",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_substantially_more_expanded_than_singlet",
              "cluster_modestly_more_expanded_than_singlet"
            ]
          },
          {
            "name": "cf_expansion_lazy_defer",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "expansion_pattern_not_detectable"
            ]
          },
          {
            "name": "cf_reported_canonical_expected_biology",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_antigen_experienced_singlet_bystander"
            ]
          },
          {
            "name": "cf_biological_inference_null_artifact_or_lineage",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "differences_reflect_dissociation_or_sorting_artifact",
              "cluster_and_singlet_capture_same_biology",
              "cluster_and_singlet_capture_distinct_cd8_lineages"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "biological_relations",
            "kind": "dict",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_greater_than",
                  "greater_than"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "less_than",
                  "much_less_than"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than",
                  "approximately_equal"
                ]
              },
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal"
                ]
              },
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal",
                  "greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_exhaustion_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_naive_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_proliferation_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_expansion_richness_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_multichain_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_intermediate_gzmk_overgeneralized",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_melanocytic_conjugate_overcall",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "values": [
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_relations_mass_hedge",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              }
            }
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: FAIL\n  + list_match: list_match (answer_field='cd8_states_enriched_in_singlet_fraction'): PASS  (score=3.0)\n  + list_match: list_match (answer_field='cd8_states_enriched_in_tumor_cluster_fraction'): PASS  (score=3.0)\n  + clonotype_expansion_direction_data_true: passed=True, score=1.0\n  + biological_inference_data_true: passed=True, score=1.0\n  x dict_match: dict_match (answer_field='biological_relations'): FAIL  (score=6.0)\n  + hard_fail 'singlet_top3_contains_naive_state_prior_pushing': not triggered\n  + hard_fail 'tumor_cluster_top3_contains_exhausted_state_prior_pushing': not triggered\n  + hard_fail 'singlet_top3_contains_decoy_token': not triggered\n  + hard_fail 'tumor_cluster_top3_contains_decoy_token': not triggered\n  + hard_fail 'TOXhigh_Tex_wrong_functional_category': not triggered\n  + hard_fail 'TCF7_stem_like_Tex_wrong_functional_category': not triggered\n  + hard_fail 'MKI67_state_wrong_functional_category': not triggered\n  + hard_fail 'Tn_or_Tn_Tmem_wrong_functional_category': not triggered\n  + hard_fail 'missing_exhausted_or_proliferating_anchor_in_singlet_top3': not triggered\n  + hard_fail 'missing_stemlike_precursor_in_singlet_top3': not triggered\n  + hard_fail 'missing_naive_or_early_memory_anchor_in_tumor_cluster_top3': not triggered\n  + hard_fail 'cf_reported_canonical_expansion_direction': not triggered\n  + hard_fail 'cf_expansion_lazy_defer': not triggered\n  + hard_fail 'cf_reported_canonical_expected_biology': not triggered\n  + hard_fail 'cf_biological_inference_null_artifact_or_lineage': not triggered\n  + hard_fail 'cf_exhaustion_inverted_prior_push': not triggered\n  + hard_fail 'cf_naive_inverted_prior_push': not triggered\n  + hard_fail 'cf_proliferation_inverted_prior_push': not triggered\n  + hard_fail 'cf_expansion_richness_inverted_prior_push': not triggered\n  + hard_fail 'cf_multichain_inverted_prior_push': not triggered\n  x hard_fail 'cf_intermediate_gzmk_overgeneralized': TRIGGERED\n  + hard_fail 'cf_melanocytic_conjugate_overcall': not triggered\n  + hard_fail 'cf_relations_mass_hedge': not triggered",
      "metrics": {
        "hard_fail_triggered": [
          "cf_intermediate_gzmk_overgeneralized"
        ],
        "pass_rule": "all",
        "scoring_count": 5,
        "scoring_passed": 4,
        "scoring_total_score": 14
      },
      "fieldScores": {
        "biological_inference_data_true": 1,
        "clonotype_expansion_direction_data_true": 1,
        "dict_match": 6,
        "list_match": 3
      },
      "graderChecks": [
        {
          "name": "cd8_states_enriched_in_singlet_fraction",
          "kind": "list_match",
          "passed": true,
          "score": 3
        },
        {
          "name": "cd8_states_enriched_in_tumor_cluster_fraction",
          "kind": "list_match",
          "passed": true,
          "score": 3
        },
        {
          "name": "clonotype_expansion_direction_data_true",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "biological_inference_data_true",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "biological_relations",
          "kind": "list_match",
          "passed": false,
          "score": 6
        },
        {
          "name": "singlet_top3_contains_naive_state_prior_pushing",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "singlet_top3_contains_decoy_token",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "tumor_cluster_top3_contains_decoy_token",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "TOXhigh_Tex_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "TCF7_stem_like_Tex_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "MKI67_state_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "Tn_or_Tn_Tmem_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_stemlike_precursor_in_singlet_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_reported_canonical_expansion_direction",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_expansion_lazy_defer",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_reported_canonical_expected_biology",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_biological_inference_null_artifact_or_lineage",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_exhaustion_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_naive_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_proliferation_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_expansion_richness_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_multichain_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_intermediate_gzmk_overgeneralized",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "cf_melanocytic_conjugate_overcall",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_relations_mass_hedge",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "biological_inference": "singlet_antigen_experienced_cluster_bystander",
        "biological_relations": {
          "clonal_expansion_richness_in_singlets_vs_tumor_cluster": "greater_than",
          "exhaustion_program_in_singlets_vs_tumor_cluster": "greater_than",
          "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": "greater_than",
          "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": "approximately_equal",
          "multichain_tcr_rate_in_singlets_vs_tumor_cluster": "greater_than",
          "naive_memory_program_in_singlets_vs_tumor_cluster": "less_than",
          "proliferation_program_in_singlets_vs_tumor_cluster": "greater_than"
        },
        "cd8_states_enriched_in_singlet_fraction": [
          {
            "functional_category": "stem_like_exhausted",
            "state_label": "TCF7_stem_like_Tex"
          },
          {
            "functional_category": "intermediate_exhausted_gzmk",
            "state_label": "GZMKhigh_Tex"
          },
          {
            "functional_category": "terminal_exhausted",
            "state_label": "LAG3high_Tex"
          }
        ],
        "cd8_states_enriched_in_tumor_cluster_fraction": [
          {
            "functional_category": "naive_memory",
            "state_label": "Tn"
          },
          {
            "functional_category": "naive_memory",
            "state_label": "Tn_Tmem"
          },
          {
            "functional_category": "early_effector_memory",
            "state_label": "early_Tem_GZMK"
          }
        ],
        "clonotype_expansion_finding": "singlet_modestly_more_expanded_than_cluster"
      },
      "nTurns": 24,
      "checkpointSignals": {
        "clustering": true,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": false,
        "nerveJointAnchor": false,
        "multiomeAssembly": true,
        "depthNormalizedAtac": false,
        "regulonInference": true,
        "cd8SubsetIntegration": true,
        "perPatientEnrichment": true,
        "tcrRichnessMetrics": true,
        "orthologProjection": false,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": false,
        "mousePseudobulkDesign": false,
        "cancerEpithelialDenominator": true,
        "geneLevelDe": false,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "trajectoryFile": "scbench-long/trajectories/10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1__gpt-5.5__pi__r3.json",
      "runId": "gpt-5.5__pi__10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1__r3",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-grok-4-20-beta-reasoning-pi",
      "modelName": "grok-4.20-beta-0309-reasoning",
      "harness": "pi",
      "task": "10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 1,
      "passed": false,
      "score": 5,
      "durationS": 166.27,
      "cost": 0.1179871,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_singlet_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "TOXhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "LAG3high_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "GZMKhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "intermediate_exhausted_gzmk"
                    ],
                    "canonical": "intermediate_exhausted_gzmk"
                  }
                }
              },
              {
                "key": "TCF7_stem_like_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "stem_like_exhausted"
                    ],
                    "canonical": "stem_like_exhausted"
                  }
                }
              },
              {
                "key": "MKI67_pos_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              },
              {
                "key": "MKI67high_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_tumor_cluster_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "Tn",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "Tn_Tmem",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "early_Tem_GZMK",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              },
              {
                "key": "Tem_GZMH",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              }
            ]
          },
          {
            "name": "clonotype_expansion_direction_data_true",
            "role": "gate",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_substantially_more_expanded_than_cluster",
              "singlet_modestly_more_expanded_than_cluster"
            ]
          },
          {
            "name": "biological_inference_data_true",
            "role": "gate",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_antigen_experienced_cluster_bystander"
            ]
          },
          {
            "name": "singlet_top3_contains_naive_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Tn",
              "Tn_Tmem",
              "early_Tem_GZMK",
              "Tem_GZMH"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "TOXhigh_Tex",
              "LAG3high_Tex",
              "GZMKhigh_Tex",
              "TCF7_stem_like_Tex",
              "MKI67_pos_Tex_Tprol",
              "MKI67high_Tex_Tprol"
            ]
          },
          {
            "name": "singlet_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "TOXhigh_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "TCF7_stem_like_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "MKI67_state_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "Tn_or_Tn_Tmem_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_stemlike_precursor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_reported_canonical_expansion_direction",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_substantially_more_expanded_than_singlet",
              "cluster_modestly_more_expanded_than_singlet"
            ]
          },
          {
            "name": "cf_expansion_lazy_defer",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "expansion_pattern_not_detectable"
            ]
          },
          {
            "name": "cf_reported_canonical_expected_biology",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_antigen_experienced_singlet_bystander"
            ]
          },
          {
            "name": "cf_biological_inference_null_artifact_or_lineage",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "differences_reflect_dissociation_or_sorting_artifact",
              "cluster_and_singlet_capture_same_biology",
              "cluster_and_singlet_capture_distinct_cd8_lineages"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "biological_relations",
            "kind": "dict",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_greater_than",
                  "greater_than"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "less_than",
                  "much_less_than"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than",
                  "approximately_equal"
                ]
              },
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal"
                ]
              },
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal",
                  "greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_exhaustion_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_naive_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_proliferation_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_expansion_richness_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_multichain_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_intermediate_gzmk_overgeneralized",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_melanocytic_conjugate_overcall",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "values": [
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_relations_mass_hedge",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              }
            }
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: FAIL\n  x list_match: list_match (answer_field='cd8_states_enriched_in_singlet_fraction'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='cd8_states_enriched_in_tumor_cluster_fraction'): FAIL  (score=0.0)\n  x clonotype_expansion_direction_data_true: passed=False, score=0.0\n  x biological_inference_data_true: passed=False, score=0.0\n  x dict_match: dict_match (answer_field='biological_relations'): FAIL  (score=5.0)\n  x hard_fail 'singlet_top3_contains_naive_state_prior_pushing': TRIGGERED\n  x hard_fail 'tumor_cluster_top3_contains_exhausted_state_prior_pushing': TRIGGERED\n  + hard_fail 'singlet_top3_contains_decoy_token': not triggered\n  + hard_fail 'tumor_cluster_top3_contains_decoy_token': not triggered\n  + hard_fail 'TOXhigh_Tex_wrong_functional_category': not triggered\n  + hard_fail 'TCF7_stem_like_Tex_wrong_functional_category': not triggered\n  + hard_fail 'MKI67_state_wrong_functional_category': not triggered\n  + hard_fail 'Tn_or_Tn_Tmem_wrong_functional_category': not triggered\n  x hard_fail 'missing_exhausted_or_proliferating_anchor_in_singlet_top3': TRIGGERED\n  x hard_fail 'missing_stemlike_precursor_in_singlet_top3': TRIGGERED\n  x hard_fail 'missing_naive_or_early_memory_anchor_in_tumor_cluster_top3': TRIGGERED\n  + hard_fail 'cf_reported_canonical_expansion_direction': not triggered\n  + hard_fail 'cf_expansion_lazy_defer': not triggered\n  x hard_fail 'cf_reported_canonical_expected_biology': TRIGGERED\n  + hard_fail 'cf_biological_inference_null_artifact_or_lineage': not triggered\n  + hard_fail 'cf_exhaustion_inverted_prior_push': not triggered\n  + hard_fail 'cf_naive_inverted_prior_push': not triggered\n  + hard_fail 'cf_proliferation_inverted_prior_push': not triggered\n  + hard_fail 'cf_expansion_richness_inverted_prior_push': not triggered\n  + hard_fail 'cf_multichain_inverted_prior_push': not triggered\n  x hard_fail 'cf_intermediate_gzmk_overgeneralized': TRIGGERED\n  + hard_fail 'cf_melanocytic_conjugate_overcall': not triggered\n  + hard_fail 'cf_relations_mass_hedge': not triggered",
      "metrics": {
        "hard_fail_triggered": [
          "singlet_top3_contains_naive_state_prior_pushing",
          "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
          "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
          "missing_stemlike_precursor_in_singlet_top3",
          "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
          "cf_reported_canonical_expected_biology",
          "cf_intermediate_gzmk_overgeneralized"
        ],
        "pass_rule": "all",
        "scoring_count": 5,
        "scoring_passed": 0,
        "scoring_total_score": 5
      },
      "fieldScores": {
        "biological_inference_data_true": 0,
        "clonotype_expansion_direction_data_true": 0,
        "dict_match": 5,
        "list_match": 0
      },
      "graderChecks": [
        {
          "name": "cd8_states_enriched_in_singlet_fraction",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "cd8_states_enriched_in_tumor_cluster_fraction",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "clonotype_expansion_direction_data_true",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "biological_inference_data_true",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "biological_relations",
          "kind": "list_match",
          "passed": false,
          "score": 5
        },
        {
          "name": "singlet_top3_contains_naive_state_prior_pushing",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "singlet_top3_contains_decoy_token",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "tumor_cluster_top3_contains_decoy_token",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "TOXhigh_Tex_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "TCF7_stem_like_Tex_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "MKI67_state_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "Tn_or_Tn_Tmem_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "missing_stemlike_precursor_in_singlet_top3",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "cf_reported_canonical_expansion_direction",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_expansion_lazy_defer",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_reported_canonical_expected_biology",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "cf_biological_inference_null_artifact_or_lineage",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_exhaustion_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_naive_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_proliferation_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_expansion_richness_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_multichain_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_intermediate_gzmk_overgeneralized",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "cf_melanocytic_conjugate_overcall",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_relations_mass_hedge",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "biological_inference": "cluster_antigen_experienced_singlet_bystander",
        "biological_relations": {
          "clonal_expansion_richness_in_singlets_vs_tumor_cluster": "approximately_equal",
          "exhaustion_program_in_singlets_vs_tumor_cluster": "greater_than",
          "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": "greater_than",
          "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": "less_than",
          "multichain_tcr_rate_in_singlets_vs_tumor_cluster": "greater_than",
          "naive_memory_program_in_singlets_vs_tumor_cluster": "less_than",
          "proliferation_program_in_singlets_vs_tumor_cluster": "greater_than"
        },
        "cd8_states_enriched_in_singlet_fraction": [
          {
            "functional_category": "naive_memory",
            "state_label": "Tn"
          },
          {
            "functional_category": "naive_memory",
            "state_label": "Tn_Tmem"
          },
          {
            "functional_category": "early_effector_memory",
            "state_label": "early_Tem_GZMK"
          }
        ],
        "cd8_states_enriched_in_tumor_cluster_fraction": [
          {
            "functional_category": "stem_like_exhausted",
            "state_label": "TCF7_stem_like_Tex"
          },
          {
            "functional_category": "terminal_exhausted",
            "state_label": "TOXhigh_Tex"
          },
          {
            "functional_category": "proliferating",
            "state_label": "MKI67_pos_Tex_Tprol"
          }
        ],
        "clonotype_expansion_finding": "cluster_and_singlet_similarly_expanded"
      },
      "nTurns": 21,
      "checkpointSignals": {
        "clustering": false,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": false,
        "nerveJointAnchor": false,
        "multiomeAssembly": false,
        "depthNormalizedAtac": false,
        "regulonInference": false,
        "cd8SubsetIntegration": true,
        "perPatientEnrichment": true,
        "tcrRichnessMetrics": true,
        "orthologProjection": false,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": false,
        "mousePseudobulkDesign": false,
        "cancerEpithelialDenominator": false,
        "geneLevelDe": false,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "trajectoryFile": "scbench-long/trajectories/10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1__grok-4.20-beta-0309-reasoning__pi__r1.json",
      "runId": "grok-4.20-beta-0309-reasoning__pi__10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1__r1",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-grok-4-20-beta-reasoning-pi",
      "modelName": "grok-4.20-beta-0309-reasoning",
      "harness": "pi",
      "task": "10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 2,
      "passed": false,
      "score": 4,
      "durationS": 121.02,
      "cost": 0.12702085,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_singlet_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "TOXhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "LAG3high_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "GZMKhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "intermediate_exhausted_gzmk"
                    ],
                    "canonical": "intermediate_exhausted_gzmk"
                  }
                }
              },
              {
                "key": "TCF7_stem_like_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "stem_like_exhausted"
                    ],
                    "canonical": "stem_like_exhausted"
                  }
                }
              },
              {
                "key": "MKI67_pos_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              },
              {
                "key": "MKI67high_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_tumor_cluster_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "Tn",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "Tn_Tmem",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "early_Tem_GZMK",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              },
              {
                "key": "Tem_GZMH",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              }
            ]
          },
          {
            "name": "clonotype_expansion_direction_data_true",
            "role": "gate",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_substantially_more_expanded_than_cluster",
              "singlet_modestly_more_expanded_than_cluster"
            ]
          },
          {
            "name": "biological_inference_data_true",
            "role": "gate",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_antigen_experienced_cluster_bystander"
            ]
          },
          {
            "name": "singlet_top3_contains_naive_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Tn",
              "Tn_Tmem",
              "early_Tem_GZMK",
              "Tem_GZMH"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "TOXhigh_Tex",
              "LAG3high_Tex",
              "GZMKhigh_Tex",
              "TCF7_stem_like_Tex",
              "MKI67_pos_Tex_Tprol",
              "MKI67high_Tex_Tprol"
            ]
          },
          {
            "name": "singlet_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "TOXhigh_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "TCF7_stem_like_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "MKI67_state_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "Tn_or_Tn_Tmem_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_stemlike_precursor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_reported_canonical_expansion_direction",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_substantially_more_expanded_than_singlet",
              "cluster_modestly_more_expanded_than_singlet"
            ]
          },
          {
            "name": "cf_expansion_lazy_defer",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "expansion_pattern_not_detectable"
            ]
          },
          {
            "name": "cf_reported_canonical_expected_biology",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_antigen_experienced_singlet_bystander"
            ]
          },
          {
            "name": "cf_biological_inference_null_artifact_or_lineage",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "differences_reflect_dissociation_or_sorting_artifact",
              "cluster_and_singlet_capture_same_biology",
              "cluster_and_singlet_capture_distinct_cd8_lineages"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "biological_relations",
            "kind": "dict",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_greater_than",
                  "greater_than"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "less_than",
                  "much_less_than"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than",
                  "approximately_equal"
                ]
              },
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal"
                ]
              },
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal",
                  "greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_exhaustion_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_naive_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_proliferation_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_expansion_richness_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_multichain_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_intermediate_gzmk_overgeneralized",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_melanocytic_conjugate_overcall",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "values": [
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_relations_mass_hedge",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              }
            }
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: FAIL\n  x list_match: list_match (answer_field='cd8_states_enriched_in_singlet_fraction'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='cd8_states_enriched_in_tumor_cluster_fraction'): FAIL  (score=0.0)\n  x clonotype_expansion_direction_data_true: passed=False, score=0.0\n  x biological_inference_data_true: passed=False, score=0.0\n  x dict_match: dict_match (answer_field='biological_relations'): FAIL  (score=4.0)\n  x hard_fail 'singlet_top3_contains_naive_state_prior_pushing': TRIGGERED\n  x hard_fail 'tumor_cluster_top3_contains_exhausted_state_prior_pushing': TRIGGERED\n  + hard_fail 'singlet_top3_contains_decoy_token': not triggered\n  + hard_fail 'tumor_cluster_top3_contains_decoy_token': not triggered\n  + hard_fail 'TOXhigh_Tex_wrong_functional_category': not triggered\n  + hard_fail 'TCF7_stem_like_Tex_wrong_functional_category': not triggered\n  + hard_fail 'MKI67_state_wrong_functional_category': not triggered\n  + hard_fail 'Tn_or_Tn_Tmem_wrong_functional_category': not triggered\n  x hard_fail 'missing_exhausted_or_proliferating_anchor_in_singlet_top3': TRIGGERED\n  x hard_fail 'missing_stemlike_precursor_in_singlet_top3': TRIGGERED\n  x hard_fail 'missing_naive_or_early_memory_anchor_in_tumor_cluster_top3': TRIGGERED\n  x hard_fail 'cf_reported_canonical_expansion_direction': TRIGGERED\n  + hard_fail 'cf_expansion_lazy_defer': not triggered\n  x hard_fail 'cf_reported_canonical_expected_biology': TRIGGERED\n  + hard_fail 'cf_biological_inference_null_artifact_or_lineage': not triggered\n  + hard_fail 'cf_exhaustion_inverted_prior_push': not triggered\n  x hard_fail 'cf_naive_inverted_prior_push': TRIGGERED\n  + hard_fail 'cf_proliferation_inverted_prior_push': not triggered\n  x hard_fail 'cf_expansion_richness_inverted_prior_push': TRIGGERED\n  + hard_fail 'cf_multichain_inverted_prior_push': not triggered\n  + hard_fail 'cf_intermediate_gzmk_overgeneralized': not triggered\n  + hard_fail 'cf_melanocytic_conjugate_overcall': not triggered\n  + hard_fail 'cf_relations_mass_hedge': not triggered",
      "metrics": {
        "hard_fail_triggered": [
          "singlet_top3_contains_naive_state_prior_pushing",
          "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
          "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
          "missing_stemlike_precursor_in_singlet_top3",
          "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
          "cf_reported_canonical_expansion_direction",
          "cf_reported_canonical_expected_biology",
          "cf_naive_inverted_prior_push",
          "cf_expansion_richness_inverted_prior_push"
        ],
        "pass_rule": "all",
        "scoring_count": 5,
        "scoring_passed": 0,
        "scoring_total_score": 4
      },
      "fieldScores": {
        "biological_inference_data_true": 0,
        "clonotype_expansion_direction_data_true": 0,
        "dict_match": 4,
        "list_match": 0
      },
      "graderChecks": [
        {
          "name": "cd8_states_enriched_in_singlet_fraction",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "cd8_states_enriched_in_tumor_cluster_fraction",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "clonotype_expansion_direction_data_true",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "biological_inference_data_true",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "biological_relations",
          "kind": "list_match",
          "passed": false,
          "score": 4
        },
        {
          "name": "singlet_top3_contains_naive_state_prior_pushing",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "singlet_top3_contains_decoy_token",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "tumor_cluster_top3_contains_decoy_token",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "TOXhigh_Tex_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "TCF7_stem_like_Tex_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "MKI67_state_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "Tn_or_Tn_Tmem_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "missing_stemlike_precursor_in_singlet_top3",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "cf_reported_canonical_expansion_direction",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "cf_expansion_lazy_defer",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_reported_canonical_expected_biology",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "cf_biological_inference_null_artifact_or_lineage",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_exhaustion_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_naive_inverted_prior_push",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "cf_proliferation_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_expansion_richness_inverted_prior_push",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "cf_multichain_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_intermediate_gzmk_overgeneralized",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_melanocytic_conjugate_overcall",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_relations_mass_hedge",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "biological_inference": "cluster_antigen_experienced_singlet_bystander",
        "biological_relations": {
          "clonal_expansion_richness_in_singlets_vs_tumor_cluster": "less_than",
          "exhaustion_program_in_singlets_vs_tumor_cluster": "approximately_equal",
          "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": "approximately_equal",
          "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": "less_than",
          "multichain_tcr_rate_in_singlets_vs_tumor_cluster": "approximately_equal",
          "naive_memory_program_in_singlets_vs_tumor_cluster": "greater_than",
          "proliferation_program_in_singlets_vs_tumor_cluster": "greater_than"
        },
        "cd8_states_enriched_in_singlet_fraction": [
          {
            "functional_category": "naive_memory",
            "state_label": "Tn_Tmem"
          },
          {
            "functional_category": "early_effector_memory",
            "state_label": "early_Tem_GZMK"
          },
          {
            "functional_category": "early_effector_memory",
            "state_label": "Tem_GZMH"
          }
        ],
        "cd8_states_enriched_in_tumor_cluster_fraction": [
          {
            "functional_category": "stem_like_exhausted",
            "state_label": "TCF7_stem_like_Tex"
          },
          {
            "functional_category": "intermediate_exhausted_gzmk",
            "state_label": "GZMKhigh_Tex"
          },
          {
            "functional_category": "terminal_exhausted",
            "state_label": "TOXhigh_Tex"
          }
        ],
        "clonotype_expansion_finding": "cluster_substantially_more_expanded_than_singlet"
      },
      "nTurns": 28,
      "checkpointSignals": {
        "clustering": true,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": false,
        "nerveJointAnchor": false,
        "multiomeAssembly": false,
        "depthNormalizedAtac": false,
        "regulonInference": false,
        "cd8SubsetIntegration": true,
        "perPatientEnrichment": true,
        "tcrRichnessMetrics": true,
        "orthologProjection": false,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": false,
        "mousePseudobulkDesign": false,
        "cancerEpithelialDenominator": false,
        "geneLevelDe": false,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "trajectoryFile": "scbench-long/trajectories/10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1__grok-4.20-beta-0309-reasoning__pi__r2.json",
      "runId": "grok-4.20-beta-0309-reasoning__pi__10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1__r2",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-grok-4-20-beta-reasoning-pi",
      "modelName": "grok-4.20-beta-0309-reasoning",
      "harness": "pi",
      "task": "10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": false,
      "score": 3,
      "durationS": 342.18,
      "cost": 0.055558050000000005,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_singlet_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "TOXhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "LAG3high_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "GZMKhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "intermediate_exhausted_gzmk"
                    ],
                    "canonical": "intermediate_exhausted_gzmk"
                  }
                }
              },
              {
                "key": "TCF7_stem_like_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "stem_like_exhausted"
                    ],
                    "canonical": "stem_like_exhausted"
                  }
                }
              },
              {
                "key": "MKI67_pos_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              },
              {
                "key": "MKI67high_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_tumor_cluster_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "Tn",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "Tn_Tmem",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "early_Tem_GZMK",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              },
              {
                "key": "Tem_GZMH",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              }
            ]
          },
          {
            "name": "clonotype_expansion_direction_data_true",
            "role": "gate",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_substantially_more_expanded_than_cluster",
              "singlet_modestly_more_expanded_than_cluster"
            ]
          },
          {
            "name": "biological_inference_data_true",
            "role": "gate",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_antigen_experienced_cluster_bystander"
            ]
          },
          {
            "name": "singlet_top3_contains_naive_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Tn",
              "Tn_Tmem",
              "early_Tem_GZMK",
              "Tem_GZMH"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "TOXhigh_Tex",
              "LAG3high_Tex",
              "GZMKhigh_Tex",
              "TCF7_stem_like_Tex",
              "MKI67_pos_Tex_Tprol",
              "MKI67high_Tex_Tprol"
            ]
          },
          {
            "name": "singlet_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "TOXhigh_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "TCF7_stem_like_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "MKI67_state_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "Tn_or_Tn_Tmem_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_stemlike_precursor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_reported_canonical_expansion_direction",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_substantially_more_expanded_than_singlet",
              "cluster_modestly_more_expanded_than_singlet"
            ]
          },
          {
            "name": "cf_expansion_lazy_defer",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "expansion_pattern_not_detectable"
            ]
          },
          {
            "name": "cf_reported_canonical_expected_biology",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_antigen_experienced_singlet_bystander"
            ]
          },
          {
            "name": "cf_biological_inference_null_artifact_or_lineage",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "differences_reflect_dissociation_or_sorting_artifact",
              "cluster_and_singlet_capture_same_biology",
              "cluster_and_singlet_capture_distinct_cd8_lineages"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "biological_relations",
            "kind": "dict",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_greater_than",
                  "greater_than"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "less_than",
                  "much_less_than"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than",
                  "approximately_equal"
                ]
              },
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal"
                ]
              },
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal",
                  "greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_exhaustion_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_naive_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_proliferation_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_expansion_richness_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_multichain_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_intermediate_gzmk_overgeneralized",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_melanocytic_conjugate_overcall",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "values": [
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_relations_mass_hedge",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              }
            }
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: FAIL\n  x list_match: list_match (answer_field='cd8_states_enriched_in_singlet_fraction'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='cd8_states_enriched_in_tumor_cluster_fraction'): FAIL  (score=0.0)\n  x clonotype_expansion_direction_data_true: passed=False, score=0.0\n  x biological_inference_data_true: passed=False, score=0.0\n  x dict_match: dict_match (answer_field='biological_relations'): FAIL  (score=3.0)\n  x hard_fail 'singlet_top3_contains_naive_state_prior_pushing': TRIGGERED\n  x hard_fail 'tumor_cluster_top3_contains_exhausted_state_prior_pushing': TRIGGERED\n  + hard_fail 'singlet_top3_contains_decoy_token': not triggered\n  + hard_fail 'tumor_cluster_top3_contains_decoy_token': not triggered\n  + hard_fail 'TOXhigh_Tex_wrong_functional_category': not triggered\n  + hard_fail 'TCF7_stem_like_Tex_wrong_functional_category': not triggered\n  + hard_fail 'MKI67_state_wrong_functional_category': not triggered\n  + hard_fail 'Tn_or_Tn_Tmem_wrong_functional_category': not triggered\n  x hard_fail 'missing_exhausted_or_proliferating_anchor_in_singlet_top3': TRIGGERED\n  x hard_fail 'missing_stemlike_precursor_in_singlet_top3': TRIGGERED\n  x hard_fail 'missing_naive_or_early_memory_anchor_in_tumor_cluster_top3': TRIGGERED\n  x hard_fail 'cf_reported_canonical_expansion_direction': TRIGGERED\n  + hard_fail 'cf_expansion_lazy_defer': not triggered\n  x hard_fail 'cf_reported_canonical_expected_biology': TRIGGERED\n  + hard_fail 'cf_biological_inference_null_artifact_or_lineage': not triggered\n  x hard_fail 'cf_exhaustion_inverted_prior_push': TRIGGERED\n  x hard_fail 'cf_naive_inverted_prior_push': TRIGGERED\n  x hard_fail 'cf_proliferation_inverted_prior_push': TRIGGERED\n  x hard_fail 'cf_expansion_richness_inverted_prior_push': TRIGGERED\n  + hard_fail 'cf_multichain_inverted_prior_push': not triggered\n  + hard_fail 'cf_intermediate_gzmk_overgeneralized': not triggered\n  + hard_fail 'cf_melanocytic_conjugate_overcall': not triggered\n  + hard_fail 'cf_relations_mass_hedge': not triggered",
      "metrics": {
        "hard_fail_triggered": [
          "singlet_top3_contains_naive_state_prior_pushing",
          "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
          "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
          "missing_stemlike_precursor_in_singlet_top3",
          "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
          "cf_reported_canonical_expansion_direction",
          "cf_reported_canonical_expected_biology",
          "cf_exhaustion_inverted_prior_push",
          "cf_naive_inverted_prior_push",
          "cf_proliferation_inverted_prior_push",
          "cf_expansion_richness_inverted_prior_push"
        ],
        "pass_rule": "all",
        "scoring_count": 5,
        "scoring_passed": 0,
        "scoring_total_score": 3
      },
      "fieldScores": {
        "biological_inference_data_true": 0,
        "clonotype_expansion_direction_data_true": 0,
        "dict_match": 3,
        "list_match": 0
      },
      "graderChecks": [
        {
          "name": "cd8_states_enriched_in_singlet_fraction",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "cd8_states_enriched_in_tumor_cluster_fraction",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "clonotype_expansion_direction_data_true",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "biological_inference_data_true",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "biological_relations",
          "kind": "list_match",
          "passed": false,
          "score": 3
        },
        {
          "name": "singlet_top3_contains_naive_state_prior_pushing",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "singlet_top3_contains_decoy_token",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "tumor_cluster_top3_contains_decoy_token",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "TOXhigh_Tex_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "TCF7_stem_like_Tex_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "MKI67_state_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "Tn_or_Tn_Tmem_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "missing_stemlike_precursor_in_singlet_top3",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "cf_reported_canonical_expansion_direction",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "cf_expansion_lazy_defer",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_reported_canonical_expected_biology",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "cf_biological_inference_null_artifact_or_lineage",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_exhaustion_inverted_prior_push",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "cf_naive_inverted_prior_push",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "cf_proliferation_inverted_prior_push",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "cf_expansion_richness_inverted_prior_push",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "cf_multichain_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_intermediate_gzmk_overgeneralized",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_melanocytic_conjugate_overcall",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_relations_mass_hedge",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "biological_inference": "cluster_antigen_experienced_singlet_bystander",
        "biological_relations": {
          "clonal_expansion_richness_in_singlets_vs_tumor_cluster": "less_than",
          "exhaustion_program_in_singlets_vs_tumor_cluster": "less_than",
          "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": "less_than",
          "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": "greater_than",
          "multichain_tcr_rate_in_singlets_vs_tumor_cluster": "approximately_equal",
          "naive_memory_program_in_singlets_vs_tumor_cluster": "greater_than",
          "proliferation_program_in_singlets_vs_tumor_cluster": "less_than"
        },
        "cd8_states_enriched_in_singlet_fraction": [
          {
            "functional_category": "naive_memory",
            "state_label": "Tn"
          },
          {
            "functional_category": "naive_memory",
            "state_label": "Tn_Tmem"
          },
          {
            "functional_category": "early_effector_memory",
            "state_label": "early_Tem_GZMK"
          }
        ],
        "cd8_states_enriched_in_tumor_cluster_fraction": [
          {
            "functional_category": "stem_like_exhausted",
            "state_label": "TCF7_stem_like_Tex"
          },
          {
            "functional_category": "intermediate_exhausted_gzmk",
            "state_label": "GZMKhigh_Tex"
          },
          {
            "functional_category": "proliferating",
            "state_label": "MKI67_pos_Tex_Tprol"
          }
        ],
        "clonotype_expansion_finding": "cluster_substantially_more_expanded_than_singlet"
      },
      "nTurns": 14,
      "checkpointSignals": {
        "clustering": true,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": false,
        "nerveJointAnchor": false,
        "multiomeAssembly": false,
        "depthNormalizedAtac": false,
        "regulonInference": false,
        "cd8SubsetIntegration": true,
        "perPatientEnrichment": true,
        "tcrRichnessMetrics": true,
        "orthologProjection": false,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": false,
        "mousePseudobulkDesign": false,
        "cancerEpithelialDenominator": false,
        "geneLevelDe": false,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "trajectoryFile": "scbench-long/trajectories/10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1__grok-4.20-beta-0309-reasoning__pi__r3.json",
      "runId": "grok-4.20-beta-0309-reasoning__pi__10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1__r3",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-grok-4-3-pi",
      "modelName": "grok-4.3",
      "harness": "pi",
      "task": "10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 1,
      "passed": false,
      "score": 3,
      "durationS": 323.32,
      "cost": 0.058377000000000005,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_singlet_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "TOXhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "LAG3high_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "GZMKhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "intermediate_exhausted_gzmk"
                    ],
                    "canonical": "intermediate_exhausted_gzmk"
                  }
                }
              },
              {
                "key": "TCF7_stem_like_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "stem_like_exhausted"
                    ],
                    "canonical": "stem_like_exhausted"
                  }
                }
              },
              {
                "key": "MKI67_pos_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              },
              {
                "key": "MKI67high_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_tumor_cluster_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "Tn",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "Tn_Tmem",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "early_Tem_GZMK",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              },
              {
                "key": "Tem_GZMH",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              }
            ]
          },
          {
            "name": "clonotype_expansion_direction_data_true",
            "role": "gate",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_substantially_more_expanded_than_cluster",
              "singlet_modestly_more_expanded_than_cluster"
            ]
          },
          {
            "name": "biological_inference_data_true",
            "role": "gate",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_antigen_experienced_cluster_bystander"
            ]
          },
          {
            "name": "singlet_top3_contains_naive_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Tn",
              "Tn_Tmem",
              "early_Tem_GZMK",
              "Tem_GZMH"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "TOXhigh_Tex",
              "LAG3high_Tex",
              "GZMKhigh_Tex",
              "TCF7_stem_like_Tex",
              "MKI67_pos_Tex_Tprol",
              "MKI67high_Tex_Tprol"
            ]
          },
          {
            "name": "singlet_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "TOXhigh_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "TCF7_stem_like_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "MKI67_state_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "Tn_or_Tn_Tmem_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_stemlike_precursor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_reported_canonical_expansion_direction",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_substantially_more_expanded_than_singlet",
              "cluster_modestly_more_expanded_than_singlet"
            ]
          },
          {
            "name": "cf_expansion_lazy_defer",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "expansion_pattern_not_detectable"
            ]
          },
          {
            "name": "cf_reported_canonical_expected_biology",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_antigen_experienced_singlet_bystander"
            ]
          },
          {
            "name": "cf_biological_inference_null_artifact_or_lineage",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "differences_reflect_dissociation_or_sorting_artifact",
              "cluster_and_singlet_capture_same_biology",
              "cluster_and_singlet_capture_distinct_cd8_lineages"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "biological_relations",
            "kind": "dict",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_greater_than",
                  "greater_than"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "less_than",
                  "much_less_than"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than",
                  "approximately_equal"
                ]
              },
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal"
                ]
              },
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal",
                  "greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_exhaustion_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_naive_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_proliferation_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_expansion_richness_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_multichain_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_intermediate_gzmk_overgeneralized",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_melanocytic_conjugate_overcall",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "values": [
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_relations_mass_hedge",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              }
            }
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: FAIL\n  x list_match: list_match (answer_field='cd8_states_enriched_in_singlet_fraction'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='cd8_states_enriched_in_tumor_cluster_fraction'): FAIL  (score=0.0)\n  x clonotype_expansion_direction_data_true: passed=False, score=0.0\n  x biological_inference_data_true: passed=False, score=0.0\n  x dict_match: dict_match (answer_field='biological_relations'): FAIL  (score=3.0)\n  x hard_fail 'singlet_top3_contains_naive_state_prior_pushing': TRIGGERED\n  x hard_fail 'tumor_cluster_top3_contains_exhausted_state_prior_pushing': TRIGGERED\n  + hard_fail 'singlet_top3_contains_decoy_token': not triggered\n  + hard_fail 'tumor_cluster_top3_contains_decoy_token': not triggered\n  + hard_fail 'TOXhigh_Tex_wrong_functional_category': not triggered\n  + hard_fail 'TCF7_stem_like_Tex_wrong_functional_category': not triggered\n  + hard_fail 'MKI67_state_wrong_functional_category': not triggered\n  + hard_fail 'Tn_or_Tn_Tmem_wrong_functional_category': not triggered\n  x hard_fail 'missing_exhausted_or_proliferating_anchor_in_singlet_top3': TRIGGERED\n  x hard_fail 'missing_stemlike_precursor_in_singlet_top3': TRIGGERED\n  x hard_fail 'missing_naive_or_early_memory_anchor_in_tumor_cluster_top3': TRIGGERED\n  x hard_fail 'cf_reported_canonical_expansion_direction': TRIGGERED\n  + hard_fail 'cf_expansion_lazy_defer': not triggered\n  x hard_fail 'cf_reported_canonical_expected_biology': TRIGGERED\n  + hard_fail 'cf_biological_inference_null_artifact_or_lineage': not triggered\n  x hard_fail 'cf_exhaustion_inverted_prior_push': TRIGGERED\n  x hard_fail 'cf_naive_inverted_prior_push': TRIGGERED\n  x hard_fail 'cf_proliferation_inverted_prior_push': TRIGGERED\n  x hard_fail 'cf_expansion_richness_inverted_prior_push': TRIGGERED\n  + hard_fail 'cf_multichain_inverted_prior_push': not triggered\n  + hard_fail 'cf_intermediate_gzmk_overgeneralized': not triggered\n  + hard_fail 'cf_melanocytic_conjugate_overcall': not triggered\n  + hard_fail 'cf_relations_mass_hedge': not triggered",
      "metrics": {
        "hard_fail_triggered": [
          "singlet_top3_contains_naive_state_prior_pushing",
          "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
          "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
          "missing_stemlike_precursor_in_singlet_top3",
          "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
          "cf_reported_canonical_expansion_direction",
          "cf_reported_canonical_expected_biology",
          "cf_exhaustion_inverted_prior_push",
          "cf_naive_inverted_prior_push",
          "cf_proliferation_inverted_prior_push",
          "cf_expansion_richness_inverted_prior_push"
        ],
        "pass_rule": "all",
        "scoring_count": 5,
        "scoring_passed": 0,
        "scoring_total_score": 3
      },
      "fieldScores": {
        "biological_inference_data_true": 0,
        "clonotype_expansion_direction_data_true": 0,
        "dict_match": 3,
        "list_match": 0
      },
      "graderChecks": [
        {
          "name": "cd8_states_enriched_in_singlet_fraction",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "cd8_states_enriched_in_tumor_cluster_fraction",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "clonotype_expansion_direction_data_true",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "biological_inference_data_true",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "biological_relations",
          "kind": "list_match",
          "passed": false,
          "score": 3
        },
        {
          "name": "singlet_top3_contains_naive_state_prior_pushing",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "singlet_top3_contains_decoy_token",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "tumor_cluster_top3_contains_decoy_token",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "TOXhigh_Tex_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "TCF7_stem_like_Tex_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "MKI67_state_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "Tn_or_Tn_Tmem_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "missing_stemlike_precursor_in_singlet_top3",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "cf_reported_canonical_expansion_direction",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "cf_expansion_lazy_defer",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_reported_canonical_expected_biology",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "cf_biological_inference_null_artifact_or_lineage",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_exhaustion_inverted_prior_push",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "cf_naive_inverted_prior_push",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "cf_proliferation_inverted_prior_push",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "cf_expansion_richness_inverted_prior_push",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "cf_multichain_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_intermediate_gzmk_overgeneralized",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_melanocytic_conjugate_overcall",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_relations_mass_hedge",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "biological_inference": "cluster_antigen_experienced_singlet_bystander",
        "biological_relations": {
          "clonal_expansion_richness_in_singlets_vs_tumor_cluster": "much_less_than",
          "exhaustion_program_in_singlets_vs_tumor_cluster": "much_less_than",
          "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": "less_than",
          "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": "greater_than",
          "multichain_tcr_rate_in_singlets_vs_tumor_cluster": "approximately_equal",
          "naive_memory_program_in_singlets_vs_tumor_cluster": "much_greater_than",
          "proliferation_program_in_singlets_vs_tumor_cluster": "less_than"
        },
        "cd8_states_enriched_in_singlet_fraction": [
          {
            "functional_category": "naive_memory",
            "state_label": "Tn"
          },
          {
            "functional_category": "naive_memory",
            "state_label": "Tn_Tmem"
          },
          {
            "functional_category": "early_effector_memory",
            "state_label": "early_Tem_GZMK"
          }
        ],
        "cd8_states_enriched_in_tumor_cluster_fraction": [
          {
            "functional_category": "stem_like_exhausted",
            "state_label": "TCF7_stem_like_Tex"
          },
          {
            "functional_category": "intermediate_exhausted_gzmk",
            "state_label": "GZMKhigh_Tex"
          },
          {
            "functional_category": "terminal_exhausted",
            "state_label": "TOXhigh_Tex"
          }
        ],
        "clonotype_expansion_finding": "cluster_substantially_more_expanded_than_singlet"
      },
      "nTurns": 9,
      "checkpointSignals": {
        "clustering": true,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": false,
        "nerveJointAnchor": false,
        "multiomeAssembly": false,
        "depthNormalizedAtac": false,
        "regulonInference": false,
        "cd8SubsetIntegration": true,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": true,
        "orthologProjection": false,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": false,
        "mousePseudobulkDesign": false,
        "cancerEpithelialDenominator": false,
        "geneLevelDe": false,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "trajectoryFile": "scbench-long/trajectories/10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1__grok-4.3__pi__r1.json",
      "runId": "grok-4.3__pi__10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1__r1",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-grok-4-3-pi",
      "modelName": "grok-4.3",
      "harness": "pi",
      "task": "10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 2,
      "passed": false,
      "score": 3,
      "durationS": 89.02,
      "cost": 0.05494715000000001,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_singlet_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "TOXhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "LAG3high_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "GZMKhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "intermediate_exhausted_gzmk"
                    ],
                    "canonical": "intermediate_exhausted_gzmk"
                  }
                }
              },
              {
                "key": "TCF7_stem_like_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "stem_like_exhausted"
                    ],
                    "canonical": "stem_like_exhausted"
                  }
                }
              },
              {
                "key": "MKI67_pos_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              },
              {
                "key": "MKI67high_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_tumor_cluster_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "Tn",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "Tn_Tmem",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "early_Tem_GZMK",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              },
              {
                "key": "Tem_GZMH",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              }
            ]
          },
          {
            "name": "clonotype_expansion_direction_data_true",
            "role": "gate",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_substantially_more_expanded_than_cluster",
              "singlet_modestly_more_expanded_than_cluster"
            ]
          },
          {
            "name": "biological_inference_data_true",
            "role": "gate",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_antigen_experienced_cluster_bystander"
            ]
          },
          {
            "name": "singlet_top3_contains_naive_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Tn",
              "Tn_Tmem",
              "early_Tem_GZMK",
              "Tem_GZMH"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "TOXhigh_Tex",
              "LAG3high_Tex",
              "GZMKhigh_Tex",
              "TCF7_stem_like_Tex",
              "MKI67_pos_Tex_Tprol",
              "MKI67high_Tex_Tprol"
            ]
          },
          {
            "name": "singlet_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "TOXhigh_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "TCF7_stem_like_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "MKI67_state_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "Tn_or_Tn_Tmem_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_stemlike_precursor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_reported_canonical_expansion_direction",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_substantially_more_expanded_than_singlet",
              "cluster_modestly_more_expanded_than_singlet"
            ]
          },
          {
            "name": "cf_expansion_lazy_defer",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "expansion_pattern_not_detectable"
            ]
          },
          {
            "name": "cf_reported_canonical_expected_biology",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_antigen_experienced_singlet_bystander"
            ]
          },
          {
            "name": "cf_biological_inference_null_artifact_or_lineage",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "differences_reflect_dissociation_or_sorting_artifact",
              "cluster_and_singlet_capture_same_biology",
              "cluster_and_singlet_capture_distinct_cd8_lineages"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "biological_relations",
            "kind": "dict",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_greater_than",
                  "greater_than"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "less_than",
                  "much_less_than"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than",
                  "approximately_equal"
                ]
              },
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal"
                ]
              },
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal",
                  "greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_exhaustion_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_naive_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_proliferation_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_expansion_richness_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_multichain_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_intermediate_gzmk_overgeneralized",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_melanocytic_conjugate_overcall",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "values": [
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_relations_mass_hedge",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              }
            }
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: FAIL\n  x list_match: list_match (answer_field='cd8_states_enriched_in_singlet_fraction'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='cd8_states_enriched_in_tumor_cluster_fraction'): FAIL  (score=0.0)\n  x clonotype_expansion_direction_data_true: passed=False, score=0.0\n  x biological_inference_data_true: passed=False, score=0.0\n  x dict_match: dict_match (answer_field='biological_relations'): FAIL  (score=3.0)\n  x hard_fail 'singlet_top3_contains_naive_state_prior_pushing': TRIGGERED\n  x hard_fail 'tumor_cluster_top3_contains_exhausted_state_prior_pushing': TRIGGERED\n  + hard_fail 'singlet_top3_contains_decoy_token': not triggered\n  + hard_fail 'tumor_cluster_top3_contains_decoy_token': not triggered\n  + hard_fail 'TOXhigh_Tex_wrong_functional_category': not triggered\n  + hard_fail 'TCF7_stem_like_Tex_wrong_functional_category': not triggered\n  + hard_fail 'MKI67_state_wrong_functional_category': not triggered\n  + hard_fail 'Tn_or_Tn_Tmem_wrong_functional_category': not triggered\n  x hard_fail 'missing_exhausted_or_proliferating_anchor_in_singlet_top3': TRIGGERED\n  x hard_fail 'missing_stemlike_precursor_in_singlet_top3': TRIGGERED\n  x hard_fail 'missing_naive_or_early_memory_anchor_in_tumor_cluster_top3': TRIGGERED\n  x hard_fail 'cf_reported_canonical_expansion_direction': TRIGGERED\n  + hard_fail 'cf_expansion_lazy_defer': not triggered\n  x hard_fail 'cf_reported_canonical_expected_biology': TRIGGERED\n  + hard_fail 'cf_biological_inference_null_artifact_or_lineage': not triggered\n  x hard_fail 'cf_exhaustion_inverted_prior_push': TRIGGERED\n  x hard_fail 'cf_naive_inverted_prior_push': TRIGGERED\n  x hard_fail 'cf_proliferation_inverted_prior_push': TRIGGERED\n  x hard_fail 'cf_expansion_richness_inverted_prior_push': TRIGGERED\n  + hard_fail 'cf_multichain_inverted_prior_push': not triggered\n  + hard_fail 'cf_intermediate_gzmk_overgeneralized': not triggered\n  + hard_fail 'cf_melanocytic_conjugate_overcall': not triggered\n  + hard_fail 'cf_relations_mass_hedge': not triggered",
      "metrics": {
        "hard_fail_triggered": [
          "singlet_top3_contains_naive_state_prior_pushing",
          "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
          "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
          "missing_stemlike_precursor_in_singlet_top3",
          "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
          "cf_reported_canonical_expansion_direction",
          "cf_reported_canonical_expected_biology",
          "cf_exhaustion_inverted_prior_push",
          "cf_naive_inverted_prior_push",
          "cf_proliferation_inverted_prior_push",
          "cf_expansion_richness_inverted_prior_push"
        ],
        "pass_rule": "all",
        "scoring_count": 5,
        "scoring_passed": 0,
        "scoring_total_score": 3
      },
      "fieldScores": {
        "biological_inference_data_true": 0,
        "clonotype_expansion_direction_data_true": 0,
        "dict_match": 3,
        "list_match": 0
      },
      "graderChecks": [
        {
          "name": "cd8_states_enriched_in_singlet_fraction",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "cd8_states_enriched_in_tumor_cluster_fraction",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "clonotype_expansion_direction_data_true",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "biological_inference_data_true",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "biological_relations",
          "kind": "list_match",
          "passed": false,
          "score": 3
        },
        {
          "name": "singlet_top3_contains_naive_state_prior_pushing",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "singlet_top3_contains_decoy_token",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "tumor_cluster_top3_contains_decoy_token",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "TOXhigh_Tex_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "TCF7_stem_like_Tex_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "MKI67_state_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "Tn_or_Tn_Tmem_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "missing_stemlike_precursor_in_singlet_top3",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "cf_reported_canonical_expansion_direction",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "cf_expansion_lazy_defer",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_reported_canonical_expected_biology",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "cf_biological_inference_null_artifact_or_lineage",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_exhaustion_inverted_prior_push",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "cf_naive_inverted_prior_push",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "cf_proliferation_inverted_prior_push",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "cf_expansion_richness_inverted_prior_push",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "cf_multichain_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_intermediate_gzmk_overgeneralized",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_melanocytic_conjugate_overcall",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_relations_mass_hedge",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "biological_inference": "cluster_antigen_experienced_singlet_bystander",
        "biological_relations": {
          "clonal_expansion_richness_in_singlets_vs_tumor_cluster": "less_than",
          "exhaustion_program_in_singlets_vs_tumor_cluster": "much_less_than",
          "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": "less_than",
          "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": "greater_than",
          "multichain_tcr_rate_in_singlets_vs_tumor_cluster": "approximately_equal",
          "naive_memory_program_in_singlets_vs_tumor_cluster": "much_greater_than",
          "proliferation_program_in_singlets_vs_tumor_cluster": "less_than"
        },
        "cd8_states_enriched_in_singlet_fraction": [
          {
            "functional_category": "naive_memory",
            "state_label": "Tn"
          },
          {
            "functional_category": "naive_memory",
            "state_label": "Tn_Tmem"
          },
          {
            "functional_category": "early_effector_memory",
            "state_label": "early_Tem_GZMK"
          }
        ],
        "cd8_states_enriched_in_tumor_cluster_fraction": [
          {
            "functional_category": "stem_like_exhausted",
            "state_label": "TCF7_stem_like_Tex"
          },
          {
            "functional_category": "intermediate_exhausted_gzmk",
            "state_label": "GZMKhigh_Tex"
          },
          {
            "functional_category": "terminal_exhausted",
            "state_label": "TOXhigh_Tex"
          }
        ],
        "clonotype_expansion_finding": "cluster_substantially_more_expanded_than_singlet"
      },
      "nTurns": 12,
      "checkpointSignals": {
        "clustering": false,
        "markerAnnotation": true,
        "scoreBasedAnnotation": true,
        "normalization": false,
        "nerveJointAnchor": false,
        "multiomeAssembly": false,
        "depthNormalizedAtac": false,
        "regulonInference": false,
        "cd8SubsetIntegration": true,
        "perPatientEnrichment": true,
        "tcrRichnessMetrics": true,
        "orthologProjection": false,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": false,
        "mousePseudobulkDesign": false,
        "cancerEpithelialDenominator": false,
        "geneLevelDe": false,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "trajectoryFile": "scbench-long/trajectories/10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1__grok-4.3__pi__r2.json",
      "runId": "grok-4.3__pi__10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1__r2",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-grok-4-3-pi",
      "modelName": "grok-4.3",
      "harness": "pi",
      "task": "10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": false,
      "score": 3,
      "durationS": 45.07,
      "cost": 0.034517950000000006,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_singlet_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "TOXhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "LAG3high_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "GZMKhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "intermediate_exhausted_gzmk"
                    ],
                    "canonical": "intermediate_exhausted_gzmk"
                  }
                }
              },
              {
                "key": "TCF7_stem_like_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "stem_like_exhausted"
                    ],
                    "canonical": "stem_like_exhausted"
                  }
                }
              },
              {
                "key": "MKI67_pos_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              },
              {
                "key": "MKI67high_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_tumor_cluster_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "Tn",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "Tn_Tmem",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "early_Tem_GZMK",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              },
              {
                "key": "Tem_GZMH",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              }
            ]
          },
          {
            "name": "clonotype_expansion_direction_data_true",
            "role": "gate",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_substantially_more_expanded_than_cluster",
              "singlet_modestly_more_expanded_than_cluster"
            ]
          },
          {
            "name": "biological_inference_data_true",
            "role": "gate",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_antigen_experienced_cluster_bystander"
            ]
          },
          {
            "name": "singlet_top3_contains_naive_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Tn",
              "Tn_Tmem",
              "early_Tem_GZMK",
              "Tem_GZMH"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "TOXhigh_Tex",
              "LAG3high_Tex",
              "GZMKhigh_Tex",
              "TCF7_stem_like_Tex",
              "MKI67_pos_Tex_Tprol",
              "MKI67high_Tex_Tprol"
            ]
          },
          {
            "name": "singlet_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "TOXhigh_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "TCF7_stem_like_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "MKI67_state_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "Tn_or_Tn_Tmem_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_stemlike_precursor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_reported_canonical_expansion_direction",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_substantially_more_expanded_than_singlet",
              "cluster_modestly_more_expanded_than_singlet"
            ]
          },
          {
            "name": "cf_expansion_lazy_defer",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "expansion_pattern_not_detectable"
            ]
          },
          {
            "name": "cf_reported_canonical_expected_biology",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_antigen_experienced_singlet_bystander"
            ]
          },
          {
            "name": "cf_biological_inference_null_artifact_or_lineage",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "differences_reflect_dissociation_or_sorting_artifact",
              "cluster_and_singlet_capture_same_biology",
              "cluster_and_singlet_capture_distinct_cd8_lineages"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "biological_relations",
            "kind": "dict",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_greater_than",
                  "greater_than"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "less_than",
                  "much_less_than"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than",
                  "approximately_equal"
                ]
              },
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal"
                ]
              },
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal",
                  "greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_exhaustion_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_naive_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_proliferation_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_expansion_richness_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_multichain_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_intermediate_gzmk_overgeneralized",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_melanocytic_conjugate_overcall",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "values": [
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_relations_mass_hedge",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              }
            }
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: FAIL\n  x list_match: list_match (answer_field='cd8_states_enriched_in_singlet_fraction'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='cd8_states_enriched_in_tumor_cluster_fraction'): FAIL  (score=0.0)\n  x clonotype_expansion_direction_data_true: passed=False, score=0.0\n  x biological_inference_data_true: passed=False, score=0.0\n  x dict_match: dict_match (answer_field='biological_relations'): FAIL  (score=3.0)\n  x hard_fail 'singlet_top3_contains_naive_state_prior_pushing': TRIGGERED\n  x hard_fail 'tumor_cluster_top3_contains_exhausted_state_prior_pushing': TRIGGERED\n  + hard_fail 'singlet_top3_contains_decoy_token': not triggered\n  + hard_fail 'tumor_cluster_top3_contains_decoy_token': not triggered\n  + hard_fail 'TOXhigh_Tex_wrong_functional_category': not triggered\n  + hard_fail 'TCF7_stem_like_Tex_wrong_functional_category': not triggered\n  + hard_fail 'MKI67_state_wrong_functional_category': not triggered\n  + hard_fail 'Tn_or_Tn_Tmem_wrong_functional_category': not triggered\n  x hard_fail 'missing_exhausted_or_proliferating_anchor_in_singlet_top3': TRIGGERED\n  x hard_fail 'missing_stemlike_precursor_in_singlet_top3': TRIGGERED\n  x hard_fail 'missing_naive_or_early_memory_anchor_in_tumor_cluster_top3': TRIGGERED\n  x hard_fail 'cf_reported_canonical_expansion_direction': TRIGGERED\n  + hard_fail 'cf_expansion_lazy_defer': not triggered\n  x hard_fail 'cf_reported_canonical_expected_biology': TRIGGERED\n  + hard_fail 'cf_biological_inference_null_artifact_or_lineage': not triggered\n  x hard_fail 'cf_exhaustion_inverted_prior_push': TRIGGERED\n  x hard_fail 'cf_naive_inverted_prior_push': TRIGGERED\n  x hard_fail 'cf_proliferation_inverted_prior_push': TRIGGERED\n  x hard_fail 'cf_expansion_richness_inverted_prior_push': TRIGGERED\n  + hard_fail 'cf_multichain_inverted_prior_push': not triggered\n  + hard_fail 'cf_intermediate_gzmk_overgeneralized': not triggered\n  + hard_fail 'cf_melanocytic_conjugate_overcall': not triggered\n  + hard_fail 'cf_relations_mass_hedge': not triggered",
      "metrics": {
        "hard_fail_triggered": [
          "singlet_top3_contains_naive_state_prior_pushing",
          "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
          "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
          "missing_stemlike_precursor_in_singlet_top3",
          "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
          "cf_reported_canonical_expansion_direction",
          "cf_reported_canonical_expected_biology",
          "cf_exhaustion_inverted_prior_push",
          "cf_naive_inverted_prior_push",
          "cf_proliferation_inverted_prior_push",
          "cf_expansion_richness_inverted_prior_push"
        ],
        "pass_rule": "all",
        "scoring_count": 5,
        "scoring_passed": 0,
        "scoring_total_score": 3
      },
      "fieldScores": {
        "biological_inference_data_true": 0,
        "clonotype_expansion_direction_data_true": 0,
        "dict_match": 3,
        "list_match": 0
      },
      "graderChecks": [
        {
          "name": "cd8_states_enriched_in_singlet_fraction",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "cd8_states_enriched_in_tumor_cluster_fraction",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "clonotype_expansion_direction_data_true",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "biological_inference_data_true",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "biological_relations",
          "kind": "list_match",
          "passed": false,
          "score": 3
        },
        {
          "name": "singlet_top3_contains_naive_state_prior_pushing",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "singlet_top3_contains_decoy_token",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "tumor_cluster_top3_contains_decoy_token",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "TOXhigh_Tex_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "TCF7_stem_like_Tex_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "MKI67_state_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "Tn_or_Tn_Tmem_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "missing_stemlike_precursor_in_singlet_top3",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "cf_reported_canonical_expansion_direction",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "cf_expansion_lazy_defer",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_reported_canonical_expected_biology",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "cf_biological_inference_null_artifact_or_lineage",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_exhaustion_inverted_prior_push",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "cf_naive_inverted_prior_push",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "cf_proliferation_inverted_prior_push",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "cf_expansion_richness_inverted_prior_push",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "cf_multichain_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_intermediate_gzmk_overgeneralized",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_melanocytic_conjugate_overcall",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_relations_mass_hedge",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "biological_inference": "cluster_antigen_experienced_singlet_bystander",
        "biological_relations": {
          "clonal_expansion_richness_in_singlets_vs_tumor_cluster": "much_less_than",
          "exhaustion_program_in_singlets_vs_tumor_cluster": "much_less_than",
          "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": "less_than",
          "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": "greater_than",
          "multichain_tcr_rate_in_singlets_vs_tumor_cluster": "approximately_equal",
          "naive_memory_program_in_singlets_vs_tumor_cluster": "much_greater_than",
          "proliferation_program_in_singlets_vs_tumor_cluster": "less_than"
        },
        "cd8_states_enriched_in_singlet_fraction": [
          {
            "functional_category": "naive_memory",
            "state_label": "Tn"
          },
          {
            "functional_category": "naive_memory",
            "state_label": "Tn_Tmem"
          },
          {
            "functional_category": "early_effector_memory",
            "state_label": "early_Tem_GZMK"
          }
        ],
        "cd8_states_enriched_in_tumor_cluster_fraction": [
          {
            "functional_category": "stem_like_exhausted",
            "state_label": "TCF7_stem_like_Tex"
          },
          {
            "functional_category": "intermediate_exhausted_gzmk",
            "state_label": "GZMKhigh_Tex"
          },
          {
            "functional_category": "terminal_exhausted",
            "state_label": "TOXhigh_Tex"
          }
        ],
        "clonotype_expansion_finding": "cluster_substantially_more_expanded_than_singlet"
      },
      "nTurns": 9,
      "checkpointSignals": {
        "clustering": true,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": false,
        "nerveJointAnchor": false,
        "multiomeAssembly": false,
        "depthNormalizedAtac": false,
        "regulonInference": false,
        "cd8SubsetIntegration": true,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": true,
        "orthologProjection": false,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": false,
        "mousePseudobulkDesign": false,
        "cancerEpithelialDenominator": false,
        "geneLevelDe": false,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "trajectoryFile": "scbench-long/trajectories/10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1__grok-4.3__pi__r3.json",
      "runId": "grok-4.3__pi__10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1__r3",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-kimi-k2p6-pi",
      "modelName": "kimi-k2p6",
      "harness": "pi",
      "task": "10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 1,
      "passed": false,
      "score": 13,
      "durationS": 1371.24,
      "cost": 0.81246065,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_singlet_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "TOXhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "LAG3high_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "GZMKhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "intermediate_exhausted_gzmk"
                    ],
                    "canonical": "intermediate_exhausted_gzmk"
                  }
                }
              },
              {
                "key": "TCF7_stem_like_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "stem_like_exhausted"
                    ],
                    "canonical": "stem_like_exhausted"
                  }
                }
              },
              {
                "key": "MKI67_pos_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              },
              {
                "key": "MKI67high_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_tumor_cluster_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "Tn",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "Tn_Tmem",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "early_Tem_GZMK",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              },
              {
                "key": "Tem_GZMH",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              }
            ]
          },
          {
            "name": "clonotype_expansion_direction_data_true",
            "role": "gate",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_substantially_more_expanded_than_cluster",
              "singlet_modestly_more_expanded_than_cluster"
            ]
          },
          {
            "name": "biological_inference_data_true",
            "role": "gate",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_antigen_experienced_cluster_bystander"
            ]
          },
          {
            "name": "singlet_top3_contains_naive_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Tn",
              "Tn_Tmem",
              "early_Tem_GZMK",
              "Tem_GZMH"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "TOXhigh_Tex",
              "LAG3high_Tex",
              "GZMKhigh_Tex",
              "TCF7_stem_like_Tex",
              "MKI67_pos_Tex_Tprol",
              "MKI67high_Tex_Tprol"
            ]
          },
          {
            "name": "singlet_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "TOXhigh_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "TCF7_stem_like_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "MKI67_state_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "Tn_or_Tn_Tmem_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_stemlike_precursor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_reported_canonical_expansion_direction",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_substantially_more_expanded_than_singlet",
              "cluster_modestly_more_expanded_than_singlet"
            ]
          },
          {
            "name": "cf_expansion_lazy_defer",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "expansion_pattern_not_detectable"
            ]
          },
          {
            "name": "cf_reported_canonical_expected_biology",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_antigen_experienced_singlet_bystander"
            ]
          },
          {
            "name": "cf_biological_inference_null_artifact_or_lineage",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "differences_reflect_dissociation_or_sorting_artifact",
              "cluster_and_singlet_capture_same_biology",
              "cluster_and_singlet_capture_distinct_cd8_lineages"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "biological_relations",
            "kind": "dict",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_greater_than",
                  "greater_than"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "less_than",
                  "much_less_than"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than",
                  "approximately_equal"
                ]
              },
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal"
                ]
              },
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal",
                  "greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_exhaustion_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_naive_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_proliferation_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_expansion_richness_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_multichain_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_intermediate_gzmk_overgeneralized",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_melanocytic_conjugate_overcall",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "values": [
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_relations_mass_hedge",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              }
            }
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: FAIL\n  + list_match: list_match (answer_field='cd8_states_enriched_in_singlet_fraction'): PASS  (score=3.0)\n  + list_match: list_match (answer_field='cd8_states_enriched_in_tumor_cluster_fraction'): PASS  (score=3.0)\n  + clonotype_expansion_direction_data_true: passed=True, score=1.0\n  + biological_inference_data_true: passed=True, score=1.0\n  x dict_match: dict_match (answer_field='biological_relations'): FAIL  (score=5.0)\n  + hard_fail 'singlet_top3_contains_naive_state_prior_pushing': not triggered\n  + hard_fail 'tumor_cluster_top3_contains_exhausted_state_prior_pushing': not triggered\n  + hard_fail 'singlet_top3_contains_decoy_token': not triggered\n  + hard_fail 'tumor_cluster_top3_contains_decoy_token': not triggered\n  + hard_fail 'TOXhigh_Tex_wrong_functional_category': not triggered\n  + hard_fail 'TCF7_stem_like_Tex_wrong_functional_category': not triggered\n  + hard_fail 'MKI67_state_wrong_functional_category': not triggered\n  + hard_fail 'Tn_or_Tn_Tmem_wrong_functional_category': not triggered\n  + hard_fail 'missing_exhausted_or_proliferating_anchor_in_singlet_top3': not triggered\n  + hard_fail 'missing_stemlike_precursor_in_singlet_top3': not triggered\n  + hard_fail 'missing_naive_or_early_memory_anchor_in_tumor_cluster_top3': not triggered\n  + hard_fail 'cf_reported_canonical_expansion_direction': not triggered\n  + hard_fail 'cf_expansion_lazy_defer': not triggered\n  + hard_fail 'cf_reported_canonical_expected_biology': not triggered\n  + hard_fail 'cf_biological_inference_null_artifact_or_lineage': not triggered\n  + hard_fail 'cf_exhaustion_inverted_prior_push': not triggered\n  + hard_fail 'cf_naive_inverted_prior_push': not triggered\n  + hard_fail 'cf_proliferation_inverted_prior_push': not triggered\n  + hard_fail 'cf_expansion_richness_inverted_prior_push': not triggered\n  + hard_fail 'cf_multichain_inverted_prior_push': not triggered\n  + hard_fail 'cf_intermediate_gzmk_overgeneralized': not triggered\n  + hard_fail 'cf_melanocytic_conjugate_overcall': not triggered\n  + hard_fail 'cf_relations_mass_hedge': not triggered",
      "metrics": {
        "hard_fail_triggered": [],
        "pass_rule": "all",
        "scoring_count": 5,
        "scoring_passed": 4,
        "scoring_total_score": 13
      },
      "fieldScores": {
        "biological_inference_data_true": 1,
        "clonotype_expansion_direction_data_true": 1,
        "dict_match": 5,
        "list_match": 3
      },
      "graderChecks": [
        {
          "name": "cd8_states_enriched_in_singlet_fraction",
          "kind": "list_match",
          "passed": true,
          "score": 3
        },
        {
          "name": "cd8_states_enriched_in_tumor_cluster_fraction",
          "kind": "list_match",
          "passed": true,
          "score": 3
        },
        {
          "name": "clonotype_expansion_direction_data_true",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "biological_inference_data_true",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "biological_relations",
          "kind": "list_match",
          "passed": false,
          "score": 5
        },
        {
          "name": "singlet_top3_contains_naive_state_prior_pushing",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "singlet_top3_contains_decoy_token",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "tumor_cluster_top3_contains_decoy_token",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "TOXhigh_Tex_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "TCF7_stem_like_Tex_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "MKI67_state_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "Tn_or_Tn_Tmem_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_stemlike_precursor_in_singlet_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_reported_canonical_expansion_direction",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_expansion_lazy_defer",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_reported_canonical_expected_biology",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_biological_inference_null_artifact_or_lineage",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_exhaustion_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_naive_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_proliferation_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_expansion_richness_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_multichain_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_intermediate_gzmk_overgeneralized",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_melanocytic_conjugate_overcall",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_relations_mass_hedge",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "biological_inference": "singlet_antigen_experienced_cluster_bystander",
        "biological_relations": {
          "clonal_expansion_richness_in_singlets_vs_tumor_cluster": "approximately_equal",
          "exhaustion_program_in_singlets_vs_tumor_cluster": "greater_than",
          "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": "approximately_equal",
          "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": "approximately_equal",
          "multichain_tcr_rate_in_singlets_vs_tumor_cluster": "approximately_equal",
          "naive_memory_program_in_singlets_vs_tumor_cluster": "less_than",
          "proliferation_program_in_singlets_vs_tumor_cluster": "approximately_equal"
        },
        "cd8_states_enriched_in_singlet_fraction": [
          {
            "functional_category": "stem_like_exhausted",
            "state_label": "TCF7_stem_like_Tex"
          },
          {
            "functional_category": "intermediate_exhausted_gzmk",
            "state_label": "GZMKhigh_Tex"
          },
          {
            "functional_category": "terminal_exhausted",
            "state_label": "LAG3high_Tex"
          }
        ],
        "cd8_states_enriched_in_tumor_cluster_fraction": [
          {
            "functional_category": "early_effector_memory",
            "state_label": "Tem_GZMH"
          },
          {
            "functional_category": "early_effector_memory",
            "state_label": "early_Tem_GZMK"
          },
          {
            "functional_category": "naive_memory",
            "state_label": "Tn"
          }
        ],
        "clonotype_expansion_finding": "singlet_modestly_more_expanded_than_cluster"
      },
      "nTurns": 23,
      "checkpointSignals": {
        "clustering": false,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": false,
        "nerveJointAnchor": false,
        "multiomeAssembly": false,
        "depthNormalizedAtac": false,
        "regulonInference": false,
        "cd8SubsetIntegration": true,
        "perPatientEnrichment": true,
        "tcrRichnessMetrics": true,
        "orthologProjection": false,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": false,
        "mousePseudobulkDesign": false,
        "cancerEpithelialDenominator": false,
        "geneLevelDe": false,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "trajectoryFile": "scbench-long/trajectories/10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1__kimi-k2p6__pi__r1.json",
      "runId": "kimi-k2p6__pi__10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1__r1",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-kimi-k2p6-pi",
      "modelName": "kimi-k2p6",
      "harness": "pi",
      "task": "10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 2,
      "passed": false,
      "score": 4,
      "durationS": 618.24,
      "cost": 0.25057771,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_singlet_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "TOXhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "LAG3high_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "GZMKhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "intermediate_exhausted_gzmk"
                    ],
                    "canonical": "intermediate_exhausted_gzmk"
                  }
                }
              },
              {
                "key": "TCF7_stem_like_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "stem_like_exhausted"
                    ],
                    "canonical": "stem_like_exhausted"
                  }
                }
              },
              {
                "key": "MKI67_pos_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              },
              {
                "key": "MKI67high_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_tumor_cluster_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "Tn",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "Tn_Tmem",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "early_Tem_GZMK",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              },
              {
                "key": "Tem_GZMH",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              }
            ]
          },
          {
            "name": "clonotype_expansion_direction_data_true",
            "role": "gate",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_substantially_more_expanded_than_cluster",
              "singlet_modestly_more_expanded_than_cluster"
            ]
          },
          {
            "name": "biological_inference_data_true",
            "role": "gate",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_antigen_experienced_cluster_bystander"
            ]
          },
          {
            "name": "singlet_top3_contains_naive_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Tn",
              "Tn_Tmem",
              "early_Tem_GZMK",
              "Tem_GZMH"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "TOXhigh_Tex",
              "LAG3high_Tex",
              "GZMKhigh_Tex",
              "TCF7_stem_like_Tex",
              "MKI67_pos_Tex_Tprol",
              "MKI67high_Tex_Tprol"
            ]
          },
          {
            "name": "singlet_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "TOXhigh_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "TCF7_stem_like_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "MKI67_state_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "Tn_or_Tn_Tmem_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_stemlike_precursor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_reported_canonical_expansion_direction",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_substantially_more_expanded_than_singlet",
              "cluster_modestly_more_expanded_than_singlet"
            ]
          },
          {
            "name": "cf_expansion_lazy_defer",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "expansion_pattern_not_detectable"
            ]
          },
          {
            "name": "cf_reported_canonical_expected_biology",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_antigen_experienced_singlet_bystander"
            ]
          },
          {
            "name": "cf_biological_inference_null_artifact_or_lineage",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "differences_reflect_dissociation_or_sorting_artifact",
              "cluster_and_singlet_capture_same_biology",
              "cluster_and_singlet_capture_distinct_cd8_lineages"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "biological_relations",
            "kind": "dict",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_greater_than",
                  "greater_than"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "less_than",
                  "much_less_than"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than",
                  "approximately_equal"
                ]
              },
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal"
                ]
              },
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal",
                  "greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_exhaustion_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_naive_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_proliferation_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_expansion_richness_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_multichain_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_intermediate_gzmk_overgeneralized",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_melanocytic_conjugate_overcall",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "values": [
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_relations_mass_hedge",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              }
            }
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: FAIL\n  x list_match: list_match (answer_field='cd8_states_enriched_in_singlet_fraction'): FAIL  (score=1.0)\n  x list_match: list_match (answer_field='cd8_states_enriched_in_tumor_cluster_fraction'): FAIL  (score=0.0)\n  x clonotype_expansion_direction_data_true: passed=False, score=0.0\n  x biological_inference_data_true: passed=False, score=0.0\n  x dict_match: dict_match (answer_field='biological_relations'): FAIL  (score=3.0)\n  x hard_fail 'singlet_top3_contains_naive_state_prior_pushing': TRIGGERED\n  x hard_fail 'tumor_cluster_top3_contains_exhausted_state_prior_pushing': TRIGGERED\n  + hard_fail 'singlet_top3_contains_decoy_token': not triggered\n  + hard_fail 'tumor_cluster_top3_contains_decoy_token': not triggered\n  + hard_fail 'TOXhigh_Tex_wrong_functional_category': not triggered\n  + hard_fail 'TCF7_stem_like_Tex_wrong_functional_category': not triggered\n  + hard_fail 'MKI67_state_wrong_functional_category': not triggered\n  + hard_fail 'Tn_or_Tn_Tmem_wrong_functional_category': not triggered\n  + hard_fail 'missing_exhausted_or_proliferating_anchor_in_singlet_top3': not triggered\n  x hard_fail 'missing_stemlike_precursor_in_singlet_top3': TRIGGERED\n  x hard_fail 'missing_naive_or_early_memory_anchor_in_tumor_cluster_top3': TRIGGERED\n  + hard_fail 'cf_reported_canonical_expansion_direction': not triggered\n  + hard_fail 'cf_expansion_lazy_defer': not triggered\n  x hard_fail 'cf_reported_canonical_expected_biology': TRIGGERED\n  + hard_fail 'cf_biological_inference_null_artifact_or_lineage': not triggered\n  + hard_fail 'cf_exhaustion_inverted_prior_push': not triggered\n  + hard_fail 'cf_naive_inverted_prior_push': not triggered\n  + hard_fail 'cf_proliferation_inverted_prior_push': not triggered\n  + hard_fail 'cf_expansion_richness_inverted_prior_push': not triggered\n  + hard_fail 'cf_multichain_inverted_prior_push': not triggered\n  + hard_fail 'cf_intermediate_gzmk_overgeneralized': not triggered\n  + hard_fail 'cf_melanocytic_conjugate_overcall': not triggered\n  x hard_fail 'cf_relations_mass_hedge': TRIGGERED",
      "metrics": {
        "hard_fail_triggered": [
          "singlet_top3_contains_naive_state_prior_pushing",
          "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
          "missing_stemlike_precursor_in_singlet_top3",
          "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
          "cf_reported_canonical_expected_biology",
          "cf_relations_mass_hedge"
        ],
        "pass_rule": "all",
        "scoring_count": 5,
        "scoring_passed": 0,
        "scoring_total_score": 4
      },
      "fieldScores": {
        "biological_inference_data_true": 0,
        "clonotype_expansion_direction_data_true": 0,
        "dict_match": 3,
        "list_match": 0
      },
      "graderChecks": [
        {
          "name": "cd8_states_enriched_in_singlet_fraction",
          "kind": "list_match",
          "passed": false,
          "score": 1
        },
        {
          "name": "cd8_states_enriched_in_tumor_cluster_fraction",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "clonotype_expansion_direction_data_true",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "biological_inference_data_true",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "biological_relations",
          "kind": "list_match",
          "passed": false,
          "score": 3
        },
        {
          "name": "singlet_top3_contains_naive_state_prior_pushing",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "singlet_top3_contains_decoy_token",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "tumor_cluster_top3_contains_decoy_token",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "TOXhigh_Tex_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "TCF7_stem_like_Tex_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "MKI67_state_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "Tn_or_Tn_Tmem_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_stemlike_precursor_in_singlet_top3",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "cf_reported_canonical_expansion_direction",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_expansion_lazy_defer",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_reported_canonical_expected_biology",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "cf_biological_inference_null_artifact_or_lineage",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_exhaustion_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_naive_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_proliferation_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_expansion_richness_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_multichain_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_intermediate_gzmk_overgeneralized",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_melanocytic_conjugate_overcall",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_relations_mass_hedge",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        }
      ],
      "agentAnswer": {
        "biological_inference": "cluster_antigen_experienced_singlet_bystander",
        "biological_relations": {
          "clonal_expansion_richness_in_singlets_vs_tumor_cluster": "approximately_equal",
          "exhaustion_program_in_singlets_vs_tumor_cluster": "approximately_equal",
          "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": "approximately_equal",
          "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": "approximately_equal",
          "multichain_tcr_rate_in_singlets_vs_tumor_cluster": "approximately_equal",
          "naive_memory_program_in_singlets_vs_tumor_cluster": "approximately_equal",
          "proliferation_program_in_singlets_vs_tumor_cluster": "approximately_equal"
        },
        "cd8_states_enriched_in_singlet_fraction": [
          {
            "functional_category": "early_effector_memory",
            "state_label": "Tem_GZMH"
          },
          {
            "functional_category": "early_effector_memory",
            "state_label": "early_Tem_GZMK"
          },
          {
            "functional_category": "terminal_exhausted",
            "state_label": "LAG3high_Tex"
          }
        ],
        "cd8_states_enriched_in_tumor_cluster_fraction": [
          {
            "functional_category": "terminal_exhausted",
            "state_label": "LAG3high_Tex"
          },
          {
            "functional_category": "proliferating",
            "state_label": "MKI67_pos_Tex_Tprol"
          },
          {
            "functional_category": "stem_like_exhausted",
            "state_label": "TCF7_stem_like_Tex"
          }
        ],
        "clonotype_expansion_finding": "cluster_and_singlet_similarly_expanded"
      },
      "nTurns": 15,
      "checkpointSignals": {
        "clustering": true,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": false,
        "nerveJointAnchor": false,
        "multiomeAssembly": true,
        "depthNormalizedAtac": false,
        "regulonInference": true,
        "cd8SubsetIntegration": true,
        "perPatientEnrichment": true,
        "tcrRichnessMetrics": true,
        "orthologProjection": false,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": false,
        "mousePseudobulkDesign": false,
        "cancerEpithelialDenominator": false,
        "geneLevelDe": false,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "trajectoryFile": "scbench-long/trajectories/10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1__kimi-k2p6__pi__r2.json",
      "runId": "kimi-k2p6__pi__10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1__r2",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-kimi-k2p6-pi",
      "modelName": "kimi-k2p6",
      "harness": "pi",
      "task": "10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": true,
      "score": 15,
      "durationS": 1664.26,
      "cost": 0.7610937299999999,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_singlet_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "TOXhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "LAG3high_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "GZMKhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "intermediate_exhausted_gzmk"
                    ],
                    "canonical": "intermediate_exhausted_gzmk"
                  }
                }
              },
              {
                "key": "TCF7_stem_like_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "stem_like_exhausted"
                    ],
                    "canonical": "stem_like_exhausted"
                  }
                }
              },
              {
                "key": "MKI67_pos_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              },
              {
                "key": "MKI67high_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_tumor_cluster_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "Tn",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "Tn_Tmem",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "early_Tem_GZMK",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              },
              {
                "key": "Tem_GZMH",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              }
            ]
          },
          {
            "name": "clonotype_expansion_direction_data_true",
            "role": "gate",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_substantially_more_expanded_than_cluster",
              "singlet_modestly_more_expanded_than_cluster"
            ]
          },
          {
            "name": "biological_inference_data_true",
            "role": "gate",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_antigen_experienced_cluster_bystander"
            ]
          },
          {
            "name": "singlet_top3_contains_naive_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Tn",
              "Tn_Tmem",
              "early_Tem_GZMK",
              "Tem_GZMH"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "TOXhigh_Tex",
              "LAG3high_Tex",
              "GZMKhigh_Tex",
              "TCF7_stem_like_Tex",
              "MKI67_pos_Tex_Tprol",
              "MKI67high_Tex_Tprol"
            ]
          },
          {
            "name": "singlet_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "TOXhigh_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "TCF7_stem_like_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "MKI67_state_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "Tn_or_Tn_Tmem_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_stemlike_precursor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_reported_canonical_expansion_direction",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_substantially_more_expanded_than_singlet",
              "cluster_modestly_more_expanded_than_singlet"
            ]
          },
          {
            "name": "cf_expansion_lazy_defer",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "expansion_pattern_not_detectable"
            ]
          },
          {
            "name": "cf_reported_canonical_expected_biology",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_antigen_experienced_singlet_bystander"
            ]
          },
          {
            "name": "cf_biological_inference_null_artifact_or_lineage",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "differences_reflect_dissociation_or_sorting_artifact",
              "cluster_and_singlet_capture_same_biology",
              "cluster_and_singlet_capture_distinct_cd8_lineages"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "biological_relations",
            "kind": "dict",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_greater_than",
                  "greater_than"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "less_than",
                  "much_less_than"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than",
                  "approximately_equal"
                ]
              },
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal"
                ]
              },
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal",
                  "greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_exhaustion_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_naive_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_proliferation_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_expansion_richness_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_multichain_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_intermediate_gzmk_overgeneralized",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_melanocytic_conjugate_overcall",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "values": [
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_relations_mass_hedge",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              }
            }
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: PASS\n  + list_match: list_match (answer_field='cd8_states_enriched_in_singlet_fraction'): PASS  (score=3.0)\n  + list_match: list_match (answer_field='cd8_states_enriched_in_tumor_cluster_fraction'): PASS  (score=3.0)\n  + clonotype_expansion_direction_data_true: passed=True, score=1.0\n  + biological_inference_data_true: passed=True, score=1.0\n  + dict_match: dict_match (answer_field='biological_relations'): PASS  (score=7.0)\n  + hard_fail 'singlet_top3_contains_naive_state_prior_pushing': not triggered\n  + hard_fail 'tumor_cluster_top3_contains_exhausted_state_prior_pushing': not triggered\n  + hard_fail 'singlet_top3_contains_decoy_token': not triggered\n  + hard_fail 'tumor_cluster_top3_contains_decoy_token': not triggered\n  + hard_fail 'TOXhigh_Tex_wrong_functional_category': not triggered\n  + hard_fail 'TCF7_stem_like_Tex_wrong_functional_category': not triggered\n  + hard_fail 'MKI67_state_wrong_functional_category': not triggered\n  + hard_fail 'Tn_or_Tn_Tmem_wrong_functional_category': not triggered\n  + hard_fail 'missing_exhausted_or_proliferating_anchor_in_singlet_top3': not triggered\n  + hard_fail 'missing_stemlike_precursor_in_singlet_top3': not triggered\n  + hard_fail 'missing_naive_or_early_memory_anchor_in_tumor_cluster_top3': not triggered\n  + hard_fail 'cf_reported_canonical_expansion_direction': not triggered\n  + hard_fail 'cf_expansion_lazy_defer': not triggered\n  + hard_fail 'cf_reported_canonical_expected_biology': not triggered\n  + hard_fail 'cf_biological_inference_null_artifact_or_lineage': not triggered\n  + hard_fail 'cf_exhaustion_inverted_prior_push': not triggered\n  + hard_fail 'cf_naive_inverted_prior_push': not triggered\n  + hard_fail 'cf_proliferation_inverted_prior_push': not triggered\n  + hard_fail 'cf_expansion_richness_inverted_prior_push': not triggered\n  + hard_fail 'cf_multichain_inverted_prior_push': not triggered\n  + hard_fail 'cf_intermediate_gzmk_overgeneralized': not triggered\n  + hard_fail 'cf_melanocytic_conjugate_overcall': not triggered\n  + hard_fail 'cf_relations_mass_hedge': not triggered",
      "metrics": {
        "hard_fail_triggered": [],
        "pass_rule": "all",
        "scoring_count": 5,
        "scoring_passed": 5,
        "scoring_total_score": 15
      },
      "fieldScores": {
        "biological_inference_data_true": 1,
        "clonotype_expansion_direction_data_true": 1,
        "dict_match": 7,
        "list_match": 3
      },
      "graderChecks": [
        {
          "name": "cd8_states_enriched_in_singlet_fraction",
          "kind": "list_match",
          "passed": true,
          "score": 3
        },
        {
          "name": "cd8_states_enriched_in_tumor_cluster_fraction",
          "kind": "list_match",
          "passed": true,
          "score": 3
        },
        {
          "name": "clonotype_expansion_direction_data_true",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "biological_inference_data_true",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "biological_relations",
          "kind": "list_match",
          "passed": true,
          "score": 7
        },
        {
          "name": "singlet_top3_contains_naive_state_prior_pushing",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "singlet_top3_contains_decoy_token",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "tumor_cluster_top3_contains_decoy_token",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "TOXhigh_Tex_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "TCF7_stem_like_Tex_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "MKI67_state_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "Tn_or_Tn_Tmem_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_stemlike_precursor_in_singlet_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_reported_canonical_expansion_direction",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_expansion_lazy_defer",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_reported_canonical_expected_biology",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_biological_inference_null_artifact_or_lineage",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_exhaustion_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_naive_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_proliferation_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_expansion_richness_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_multichain_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_intermediate_gzmk_overgeneralized",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_melanocytic_conjugate_overcall",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_relations_mass_hedge",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "biological_inference": "singlet_antigen_experienced_cluster_bystander",
        "biological_relations": {
          "clonal_expansion_richness_in_singlets_vs_tumor_cluster": "greater_than",
          "exhaustion_program_in_singlets_vs_tumor_cluster": "greater_than",
          "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": "approximately_equal",
          "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": "approximately_equal",
          "multichain_tcr_rate_in_singlets_vs_tumor_cluster": "approximately_equal",
          "naive_memory_program_in_singlets_vs_tumor_cluster": "much_less_than",
          "proliferation_program_in_singlets_vs_tumor_cluster": "greater_than"
        },
        "cd8_states_enriched_in_singlet_fraction": [
          {
            "functional_category": "stem_like_exhausted",
            "state_label": "TCF7_stem_like_Tex"
          },
          {
            "functional_category": "intermediate_exhausted_gzmk",
            "state_label": "GZMKhigh_Tex"
          },
          {
            "functional_category": "terminal_exhausted",
            "state_label": "TOXhigh_Tex"
          }
        ],
        "cd8_states_enriched_in_tumor_cluster_fraction": [
          {
            "functional_category": "naive_memory",
            "state_label": "Tn"
          },
          {
            "functional_category": "early_effector_memory",
            "state_label": "early_Tem_GZMK"
          },
          {
            "functional_category": "early_effector_memory",
            "state_label": "Tem_GZMH"
          }
        ],
        "clonotype_expansion_finding": "singlet_modestly_more_expanded_than_cluster"
      },
      "nTurns": 24,
      "checkpointSignals": {
        "clustering": false,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": false,
        "nerveJointAnchor": false,
        "multiomeAssembly": true,
        "depthNormalizedAtac": true,
        "regulonInference": true,
        "cd8SubsetIntegration": true,
        "perPatientEnrichment": true,
        "tcrRichnessMetrics": true,
        "orthologProjection": false,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": false,
        "mousePseudobulkDesign": false,
        "cancerEpithelialDenominator": false,
        "geneLevelDe": false,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "trajectoryFile": "scbench-long/trajectories/10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1__kimi-k2p6__pi__r3.json",
      "runId": "kimi-k2p6__pi__10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1__r3",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-opus-4-6-claude-code",
      "modelName": "claude-opus-4-6",
      "harness": "claude-code",
      "task": "10x_5p_melanoma_tcr_matched_apc_vs_tumor_exhaustion",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 1082,
      "cost": 1.8870400000000003,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "state_discriminating_genes",
            "kind": "list",
            "matchKey": "gene",
            "k": 3,
            "expected": [
              {
                "key": "CTLA4",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TOX",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "ENTPD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "HAVCR2",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TIGIT",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "PDCD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "difference_attributable_to",
            "role": "gate",
            "field": "difference_attributable_to",
            "kind": "value",
            "expected": "covaries_with_apc_conjugate_state"
          },
          {
            "name": "causal_status",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "direction_unresolved_selection_confound"
          },
          {
            "name": "functional_interpretation",
            "role": "gate",
            "field": "functional_interpretation",
            "kind": "value",
            "expected": "costimulation_signature_function_unresolved"
          },
          {
            "name": "cf_contamination_gene_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "MLANA",
              "PMEL",
              "DCT",
              "TYR",
              "TYRP1",
              "SOX10",
              "MITF",
              "MAGEA4"
            ]
          },
          {
            "name": "cf_apc_partner_contamination_gene",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "HLA-DRA",
              "CD68",
              "LYZ",
              "CD14",
              "C1QA",
              "ITGAX",
              "CD1C"
            ]
          },
          {
            "name": "cf_ligand_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "CD274",
              "PDCD1LG2",
              "CD80",
              "CD86"
            ]
          },
          {
            "name": "cf_attribution_causal_or_baseline",
            "role": "hard_fail",
            "field": "$.difference_attributable_to",
            "kind": "avoid",
            "expected": [
              "apc_engagement",
              "tumor_cell_engagement",
              "clonotype_intrinsic_baseline"
            ]
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "engagement_induces_exhausted_state",
              "pre_exhausted_cells_select_into_apc_conjugates"
            ]
          },
          {
            "name": "cf_causal_fabrication",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "resolved_by_pseudotemporal_ordering",
              "resolved_by_perturbation_experiment"
            ]
          },
          {
            "name": "cf_functional_overclaim",
            "role": "hard_fail",
            "field": "$.functional_interpretation",
            "kind": "avoid",
            "expected": [
              "costimulation_licensed_reinvigoration_competent",
              "costimulation_licensed_checkpoint_responsive",
              "tumor_driven_terminal_exhaustion"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-6-claude-code",
      "modelName": "claude-opus-4-6",
      "harness": "claude-code",
      "task": "10x_5p_melanoma_tcr_matched_apc_vs_tumor_exhaustion",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 1923,
      "cost": 2.4629342499999995,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "state_discriminating_genes",
            "kind": "list",
            "matchKey": "gene",
            "k": 3,
            "expected": [
              {
                "key": "CTLA4",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TOX",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "ENTPD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "HAVCR2",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TIGIT",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "PDCD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "difference_attributable_to",
            "role": "gate",
            "field": "difference_attributable_to",
            "kind": "value",
            "expected": "covaries_with_apc_conjugate_state"
          },
          {
            "name": "causal_status",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "direction_unresolved_selection_confound"
          },
          {
            "name": "functional_interpretation",
            "role": "gate",
            "field": "functional_interpretation",
            "kind": "value",
            "expected": "costimulation_signature_function_unresolved"
          },
          {
            "name": "cf_contamination_gene_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "MLANA",
              "PMEL",
              "DCT",
              "TYR",
              "TYRP1",
              "SOX10",
              "MITF",
              "MAGEA4"
            ]
          },
          {
            "name": "cf_apc_partner_contamination_gene",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "HLA-DRA",
              "CD68",
              "LYZ",
              "CD14",
              "C1QA",
              "ITGAX",
              "CD1C"
            ]
          },
          {
            "name": "cf_ligand_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "CD274",
              "PDCD1LG2",
              "CD80",
              "CD86"
            ]
          },
          {
            "name": "cf_attribution_causal_or_baseline",
            "role": "hard_fail",
            "field": "$.difference_attributable_to",
            "kind": "avoid",
            "expected": [
              "apc_engagement",
              "tumor_cell_engagement",
              "clonotype_intrinsic_baseline"
            ]
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "engagement_induces_exhausted_state",
              "pre_exhausted_cells_select_into_apc_conjugates"
            ]
          },
          {
            "name": "cf_causal_fabrication",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "resolved_by_pseudotemporal_ordering",
              "resolved_by_perturbation_experiment"
            ]
          },
          {
            "name": "cf_functional_overclaim",
            "role": "hard_fail",
            "field": "$.functional_interpretation",
            "kind": "avoid",
            "expected": [
              "costimulation_licensed_reinvigoration_competent",
              "costimulation_licensed_checkpoint_responsive",
              "tumor_driven_terminal_exhaustion"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-6-claude-code",
      "modelName": "claude-opus-4-6",
      "harness": "claude-code",
      "task": "10x_5p_melanoma_tcr_matched_apc_vs_tumor_exhaustion",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1303,
      "cost": 1.88482075,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "state_discriminating_genes",
            "kind": "list",
            "matchKey": "gene",
            "k": 3,
            "expected": [
              {
                "key": "CTLA4",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TOX",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "ENTPD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "HAVCR2",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TIGIT",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "PDCD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "difference_attributable_to",
            "role": "gate",
            "field": "difference_attributable_to",
            "kind": "value",
            "expected": "covaries_with_apc_conjugate_state"
          },
          {
            "name": "causal_status",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "direction_unresolved_selection_confound"
          },
          {
            "name": "functional_interpretation",
            "role": "gate",
            "field": "functional_interpretation",
            "kind": "value",
            "expected": "costimulation_signature_function_unresolved"
          },
          {
            "name": "cf_contamination_gene_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "MLANA",
              "PMEL",
              "DCT",
              "TYR",
              "TYRP1",
              "SOX10",
              "MITF",
              "MAGEA4"
            ]
          },
          {
            "name": "cf_apc_partner_contamination_gene",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "HLA-DRA",
              "CD68",
              "LYZ",
              "CD14",
              "C1QA",
              "ITGAX",
              "CD1C"
            ]
          },
          {
            "name": "cf_ligand_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "CD274",
              "PDCD1LG2",
              "CD80",
              "CD86"
            ]
          },
          {
            "name": "cf_attribution_causal_or_baseline",
            "role": "hard_fail",
            "field": "$.difference_attributable_to",
            "kind": "avoid",
            "expected": [
              "apc_engagement",
              "tumor_cell_engagement",
              "clonotype_intrinsic_baseline"
            ]
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "engagement_induces_exhausted_state",
              "pre_exhausted_cells_select_into_apc_conjugates"
            ]
          },
          {
            "name": "cf_causal_fabrication",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "resolved_by_pseudotemporal_ordering",
              "resolved_by_perturbation_experiment"
            ]
          },
          {
            "name": "cf_functional_overclaim",
            "role": "hard_fail",
            "field": "$.functional_interpretation",
            "kind": "avoid",
            "expected": [
              "costimulation_licensed_reinvigoration_competent",
              "costimulation_licensed_checkpoint_responsive",
              "tumor_driven_terminal_exhaustion"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-6-pi",
      "modelName": "claude-opus-4-6",
      "harness": "pi",
      "task": "10x_5p_melanoma_tcr_matched_apc_vs_tumor_exhaustion",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 954,
      "cost": 1.94983025,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "state_discriminating_genes",
            "kind": "list",
            "matchKey": "gene",
            "k": 3,
            "expected": [
              {
                "key": "CTLA4",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TOX",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "ENTPD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "HAVCR2",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TIGIT",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "PDCD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "difference_attributable_to",
            "role": "gate",
            "field": "difference_attributable_to",
            "kind": "value",
            "expected": "covaries_with_apc_conjugate_state"
          },
          {
            "name": "causal_status",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "direction_unresolved_selection_confound"
          },
          {
            "name": "functional_interpretation",
            "role": "gate",
            "field": "functional_interpretation",
            "kind": "value",
            "expected": "costimulation_signature_function_unresolved"
          },
          {
            "name": "cf_contamination_gene_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "MLANA",
              "PMEL",
              "DCT",
              "TYR",
              "TYRP1",
              "SOX10",
              "MITF",
              "MAGEA4"
            ]
          },
          {
            "name": "cf_apc_partner_contamination_gene",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "HLA-DRA",
              "CD68",
              "LYZ",
              "CD14",
              "C1QA",
              "ITGAX",
              "CD1C"
            ]
          },
          {
            "name": "cf_ligand_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "CD274",
              "PDCD1LG2",
              "CD80",
              "CD86"
            ]
          },
          {
            "name": "cf_attribution_causal_or_baseline",
            "role": "hard_fail",
            "field": "$.difference_attributable_to",
            "kind": "avoid",
            "expected": [
              "apc_engagement",
              "tumor_cell_engagement",
              "clonotype_intrinsic_baseline"
            ]
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "engagement_induces_exhausted_state",
              "pre_exhausted_cells_select_into_apc_conjugates"
            ]
          },
          {
            "name": "cf_causal_fabrication",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "resolved_by_pseudotemporal_ordering",
              "resolved_by_perturbation_experiment"
            ]
          },
          {
            "name": "cf_functional_overclaim",
            "role": "hard_fail",
            "field": "$.functional_interpretation",
            "kind": "avoid",
            "expected": [
              "costimulation_licensed_reinvigoration_competent",
              "costimulation_licensed_checkpoint_responsive",
              "tumor_driven_terminal_exhaustion"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-6-pi",
      "modelName": "claude-opus-4-6",
      "harness": "pi",
      "task": "10x_5p_melanoma_tcr_matched_apc_vs_tumor_exhaustion",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 1446,
      "cost": 2.3045360000000006,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "state_discriminating_genes",
            "kind": "list",
            "matchKey": "gene",
            "k": 3,
            "expected": [
              {
                "key": "CTLA4",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TOX",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "ENTPD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "HAVCR2",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TIGIT",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "PDCD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "difference_attributable_to",
            "role": "gate",
            "field": "difference_attributable_to",
            "kind": "value",
            "expected": "covaries_with_apc_conjugate_state"
          },
          {
            "name": "causal_status",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "direction_unresolved_selection_confound"
          },
          {
            "name": "functional_interpretation",
            "role": "gate",
            "field": "functional_interpretation",
            "kind": "value",
            "expected": "costimulation_signature_function_unresolved"
          },
          {
            "name": "cf_contamination_gene_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "MLANA",
              "PMEL",
              "DCT",
              "TYR",
              "TYRP1",
              "SOX10",
              "MITF",
              "MAGEA4"
            ]
          },
          {
            "name": "cf_apc_partner_contamination_gene",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "HLA-DRA",
              "CD68",
              "LYZ",
              "CD14",
              "C1QA",
              "ITGAX",
              "CD1C"
            ]
          },
          {
            "name": "cf_ligand_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "CD274",
              "PDCD1LG2",
              "CD80",
              "CD86"
            ]
          },
          {
            "name": "cf_attribution_causal_or_baseline",
            "role": "hard_fail",
            "field": "$.difference_attributable_to",
            "kind": "avoid",
            "expected": [
              "apc_engagement",
              "tumor_cell_engagement",
              "clonotype_intrinsic_baseline"
            ]
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "engagement_induces_exhausted_state",
              "pre_exhausted_cells_select_into_apc_conjugates"
            ]
          },
          {
            "name": "cf_causal_fabrication",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "resolved_by_pseudotemporal_ordering",
              "resolved_by_perturbation_experiment"
            ]
          },
          {
            "name": "cf_functional_overclaim",
            "role": "hard_fail",
            "field": "$.functional_interpretation",
            "kind": "avoid",
            "expected": [
              "costimulation_licensed_reinvigoration_competent",
              "costimulation_licensed_checkpoint_responsive",
              "tumor_driven_terminal_exhaustion"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-6-pi",
      "modelName": "claude-opus-4-6",
      "harness": "pi",
      "task": "10x_5p_melanoma_tcr_matched_apc_vs_tumor_exhaustion",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1151,
      "cost": 1.4501977499999998,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "state_discriminating_genes",
            "kind": "list",
            "matchKey": "gene",
            "k": 3,
            "expected": [
              {
                "key": "CTLA4",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TOX",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "ENTPD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "HAVCR2",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TIGIT",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "PDCD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "difference_attributable_to",
            "role": "gate",
            "field": "difference_attributable_to",
            "kind": "value",
            "expected": "covaries_with_apc_conjugate_state"
          },
          {
            "name": "causal_status",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "direction_unresolved_selection_confound"
          },
          {
            "name": "functional_interpretation",
            "role": "gate",
            "field": "functional_interpretation",
            "kind": "value",
            "expected": "costimulation_signature_function_unresolved"
          },
          {
            "name": "cf_contamination_gene_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "MLANA",
              "PMEL",
              "DCT",
              "TYR",
              "TYRP1",
              "SOX10",
              "MITF",
              "MAGEA4"
            ]
          },
          {
            "name": "cf_apc_partner_contamination_gene",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "HLA-DRA",
              "CD68",
              "LYZ",
              "CD14",
              "C1QA",
              "ITGAX",
              "CD1C"
            ]
          },
          {
            "name": "cf_ligand_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "CD274",
              "PDCD1LG2",
              "CD80",
              "CD86"
            ]
          },
          {
            "name": "cf_attribution_causal_or_baseline",
            "role": "hard_fail",
            "field": "$.difference_attributable_to",
            "kind": "avoid",
            "expected": [
              "apc_engagement",
              "tumor_cell_engagement",
              "clonotype_intrinsic_baseline"
            ]
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "engagement_induces_exhausted_state",
              "pre_exhausted_cells_select_into_apc_conjugates"
            ]
          },
          {
            "name": "cf_causal_fabrication",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "resolved_by_pseudotemporal_ordering",
              "resolved_by_perturbation_experiment"
            ]
          },
          {
            "name": "cf_functional_overclaim",
            "role": "hard_fail",
            "field": "$.functional_interpretation",
            "kind": "avoid",
            "expected": [
              "costimulation_licensed_reinvigoration_competent",
              "costimulation_licensed_checkpoint_responsive",
              "tumor_driven_terminal_exhaustion"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-7-claude-code",
      "modelName": "claude-opus-4-7",
      "harness": "claude-code",
      "task": "10x_5p_melanoma_tcr_matched_apc_vs_tumor_exhaustion",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 794,
      "cost": 1.2548030000000003,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "state_discriminating_genes",
            "kind": "list",
            "matchKey": "gene",
            "k": 3,
            "expected": [
              {
                "key": "CTLA4",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TOX",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "ENTPD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "HAVCR2",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TIGIT",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "PDCD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "difference_attributable_to",
            "role": "gate",
            "field": "difference_attributable_to",
            "kind": "value",
            "expected": "covaries_with_apc_conjugate_state"
          },
          {
            "name": "causal_status",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "direction_unresolved_selection_confound"
          },
          {
            "name": "functional_interpretation",
            "role": "gate",
            "field": "functional_interpretation",
            "kind": "value",
            "expected": "costimulation_signature_function_unresolved"
          },
          {
            "name": "cf_contamination_gene_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "MLANA",
              "PMEL",
              "DCT",
              "TYR",
              "TYRP1",
              "SOX10",
              "MITF",
              "MAGEA4"
            ]
          },
          {
            "name": "cf_apc_partner_contamination_gene",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "HLA-DRA",
              "CD68",
              "LYZ",
              "CD14",
              "C1QA",
              "ITGAX",
              "CD1C"
            ]
          },
          {
            "name": "cf_ligand_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "CD274",
              "PDCD1LG2",
              "CD80",
              "CD86"
            ]
          },
          {
            "name": "cf_attribution_causal_or_baseline",
            "role": "hard_fail",
            "field": "$.difference_attributable_to",
            "kind": "avoid",
            "expected": [
              "apc_engagement",
              "tumor_cell_engagement",
              "clonotype_intrinsic_baseline"
            ]
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "engagement_induces_exhausted_state",
              "pre_exhausted_cells_select_into_apc_conjugates"
            ]
          },
          {
            "name": "cf_causal_fabrication",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "resolved_by_pseudotemporal_ordering",
              "resolved_by_perturbation_experiment"
            ]
          },
          {
            "name": "cf_functional_overclaim",
            "role": "hard_fail",
            "field": "$.functional_interpretation",
            "kind": "avoid",
            "expected": [
              "costimulation_licensed_reinvigoration_competent",
              "costimulation_licensed_checkpoint_responsive",
              "tumor_driven_terminal_exhaustion"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-7-claude-code",
      "modelName": "claude-opus-4-7",
      "harness": "claude-code",
      "task": "10x_5p_melanoma_tcr_matched_apc_vs_tumor_exhaustion",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 458,
      "cost": 1.2474557499999999,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "state_discriminating_genes",
            "kind": "list",
            "matchKey": "gene",
            "k": 3,
            "expected": [
              {
                "key": "CTLA4",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TOX",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "ENTPD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "HAVCR2",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TIGIT",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "PDCD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "difference_attributable_to",
            "role": "gate",
            "field": "difference_attributable_to",
            "kind": "value",
            "expected": "covaries_with_apc_conjugate_state"
          },
          {
            "name": "causal_status",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "direction_unresolved_selection_confound"
          },
          {
            "name": "functional_interpretation",
            "role": "gate",
            "field": "functional_interpretation",
            "kind": "value",
            "expected": "costimulation_signature_function_unresolved"
          },
          {
            "name": "cf_contamination_gene_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "MLANA",
              "PMEL",
              "DCT",
              "TYR",
              "TYRP1",
              "SOX10",
              "MITF",
              "MAGEA4"
            ]
          },
          {
            "name": "cf_apc_partner_contamination_gene",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "HLA-DRA",
              "CD68",
              "LYZ",
              "CD14",
              "C1QA",
              "ITGAX",
              "CD1C"
            ]
          },
          {
            "name": "cf_ligand_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "CD274",
              "PDCD1LG2",
              "CD80",
              "CD86"
            ]
          },
          {
            "name": "cf_attribution_causal_or_baseline",
            "role": "hard_fail",
            "field": "$.difference_attributable_to",
            "kind": "avoid",
            "expected": [
              "apc_engagement",
              "tumor_cell_engagement",
              "clonotype_intrinsic_baseline"
            ]
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "engagement_induces_exhausted_state",
              "pre_exhausted_cells_select_into_apc_conjugates"
            ]
          },
          {
            "name": "cf_causal_fabrication",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "resolved_by_pseudotemporal_ordering",
              "resolved_by_perturbation_experiment"
            ]
          },
          {
            "name": "cf_functional_overclaim",
            "role": "hard_fail",
            "field": "$.functional_interpretation",
            "kind": "avoid",
            "expected": [
              "costimulation_licensed_reinvigoration_competent",
              "costimulation_licensed_checkpoint_responsive",
              "tumor_driven_terminal_exhaustion"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-7-claude-code",
      "modelName": "claude-opus-4-7",
      "harness": "claude-code",
      "task": "10x_5p_melanoma_tcr_matched_apc_vs_tumor_exhaustion",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 418,
      "cost": 1.50434025,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "state_discriminating_genes",
            "kind": "list",
            "matchKey": "gene",
            "k": 3,
            "expected": [
              {
                "key": "CTLA4",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TOX",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "ENTPD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "HAVCR2",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TIGIT",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "PDCD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "difference_attributable_to",
            "role": "gate",
            "field": "difference_attributable_to",
            "kind": "value",
            "expected": "covaries_with_apc_conjugate_state"
          },
          {
            "name": "causal_status",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "direction_unresolved_selection_confound"
          },
          {
            "name": "functional_interpretation",
            "role": "gate",
            "field": "functional_interpretation",
            "kind": "value",
            "expected": "costimulation_signature_function_unresolved"
          },
          {
            "name": "cf_contamination_gene_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "MLANA",
              "PMEL",
              "DCT",
              "TYR",
              "TYRP1",
              "SOX10",
              "MITF",
              "MAGEA4"
            ]
          },
          {
            "name": "cf_apc_partner_contamination_gene",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "HLA-DRA",
              "CD68",
              "LYZ",
              "CD14",
              "C1QA",
              "ITGAX",
              "CD1C"
            ]
          },
          {
            "name": "cf_ligand_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "CD274",
              "PDCD1LG2",
              "CD80",
              "CD86"
            ]
          },
          {
            "name": "cf_attribution_causal_or_baseline",
            "role": "hard_fail",
            "field": "$.difference_attributable_to",
            "kind": "avoid",
            "expected": [
              "apc_engagement",
              "tumor_cell_engagement",
              "clonotype_intrinsic_baseline"
            ]
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "engagement_induces_exhausted_state",
              "pre_exhausted_cells_select_into_apc_conjugates"
            ]
          },
          {
            "name": "cf_causal_fabrication",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "resolved_by_pseudotemporal_ordering",
              "resolved_by_perturbation_experiment"
            ]
          },
          {
            "name": "cf_functional_overclaim",
            "role": "hard_fail",
            "field": "$.functional_interpretation",
            "kind": "avoid",
            "expected": [
              "costimulation_licensed_reinvigoration_competent",
              "costimulation_licensed_checkpoint_responsive",
              "tumor_driven_terminal_exhaustion"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-7-pi",
      "modelName": "claude-opus-4-7",
      "harness": "pi",
      "task": "10x_5p_melanoma_tcr_matched_apc_vs_tumor_exhaustion",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 553,
      "cost": 0.50937425,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "state_discriminating_genes",
            "kind": "list",
            "matchKey": "gene",
            "k": 3,
            "expected": [
              {
                "key": "CTLA4",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TOX",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "ENTPD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "HAVCR2",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TIGIT",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "PDCD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "difference_attributable_to",
            "role": "gate",
            "field": "difference_attributable_to",
            "kind": "value",
            "expected": "covaries_with_apc_conjugate_state"
          },
          {
            "name": "causal_status",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "direction_unresolved_selection_confound"
          },
          {
            "name": "functional_interpretation",
            "role": "gate",
            "field": "functional_interpretation",
            "kind": "value",
            "expected": "costimulation_signature_function_unresolved"
          },
          {
            "name": "cf_contamination_gene_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "MLANA",
              "PMEL",
              "DCT",
              "TYR",
              "TYRP1",
              "SOX10",
              "MITF",
              "MAGEA4"
            ]
          },
          {
            "name": "cf_apc_partner_contamination_gene",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "HLA-DRA",
              "CD68",
              "LYZ",
              "CD14",
              "C1QA",
              "ITGAX",
              "CD1C"
            ]
          },
          {
            "name": "cf_ligand_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "CD274",
              "PDCD1LG2",
              "CD80",
              "CD86"
            ]
          },
          {
            "name": "cf_attribution_causal_or_baseline",
            "role": "hard_fail",
            "field": "$.difference_attributable_to",
            "kind": "avoid",
            "expected": [
              "apc_engagement",
              "tumor_cell_engagement",
              "clonotype_intrinsic_baseline"
            ]
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "engagement_induces_exhausted_state",
              "pre_exhausted_cells_select_into_apc_conjugates"
            ]
          },
          {
            "name": "cf_causal_fabrication",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "resolved_by_pseudotemporal_ordering",
              "resolved_by_perturbation_experiment"
            ]
          },
          {
            "name": "cf_functional_overclaim",
            "role": "hard_fail",
            "field": "$.functional_interpretation",
            "kind": "avoid",
            "expected": [
              "costimulation_licensed_reinvigoration_competent",
              "costimulation_licensed_checkpoint_responsive",
              "tumor_driven_terminal_exhaustion"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-7-pi",
      "modelName": "claude-opus-4-7",
      "harness": "pi",
      "task": "10x_5p_melanoma_tcr_matched_apc_vs_tumor_exhaustion",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 218,
      "cost": 0.42915325,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "state_discriminating_genes",
            "kind": "list",
            "matchKey": "gene",
            "k": 3,
            "expected": [
              {
                "key": "CTLA4",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TOX",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "ENTPD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "HAVCR2",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TIGIT",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "PDCD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "difference_attributable_to",
            "role": "gate",
            "field": "difference_attributable_to",
            "kind": "value",
            "expected": "covaries_with_apc_conjugate_state"
          },
          {
            "name": "causal_status",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "direction_unresolved_selection_confound"
          },
          {
            "name": "functional_interpretation",
            "role": "gate",
            "field": "functional_interpretation",
            "kind": "value",
            "expected": "costimulation_signature_function_unresolved"
          },
          {
            "name": "cf_contamination_gene_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "MLANA",
              "PMEL",
              "DCT",
              "TYR",
              "TYRP1",
              "SOX10",
              "MITF",
              "MAGEA4"
            ]
          },
          {
            "name": "cf_apc_partner_contamination_gene",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "HLA-DRA",
              "CD68",
              "LYZ",
              "CD14",
              "C1QA",
              "ITGAX",
              "CD1C"
            ]
          },
          {
            "name": "cf_ligand_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "CD274",
              "PDCD1LG2",
              "CD80",
              "CD86"
            ]
          },
          {
            "name": "cf_attribution_causal_or_baseline",
            "role": "hard_fail",
            "field": "$.difference_attributable_to",
            "kind": "avoid",
            "expected": [
              "apc_engagement",
              "tumor_cell_engagement",
              "clonotype_intrinsic_baseline"
            ]
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "engagement_induces_exhausted_state",
              "pre_exhausted_cells_select_into_apc_conjugates"
            ]
          },
          {
            "name": "cf_causal_fabrication",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "resolved_by_pseudotemporal_ordering",
              "resolved_by_perturbation_experiment"
            ]
          },
          {
            "name": "cf_functional_overclaim",
            "role": "hard_fail",
            "field": "$.functional_interpretation",
            "kind": "avoid",
            "expected": [
              "costimulation_licensed_reinvigoration_competent",
              "costimulation_licensed_checkpoint_responsive",
              "tumor_driven_terminal_exhaustion"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-7-pi",
      "modelName": "claude-opus-4-7",
      "harness": "pi",
      "task": "10x_5p_melanoma_tcr_matched_apc_vs_tumor_exhaustion",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 343,
      "cost": 0.309257,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "state_discriminating_genes",
            "kind": "list",
            "matchKey": "gene",
            "k": 3,
            "expected": [
              {
                "key": "CTLA4",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TOX",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "ENTPD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "HAVCR2",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TIGIT",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "PDCD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "difference_attributable_to",
            "role": "gate",
            "field": "difference_attributable_to",
            "kind": "value",
            "expected": "covaries_with_apc_conjugate_state"
          },
          {
            "name": "causal_status",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "direction_unresolved_selection_confound"
          },
          {
            "name": "functional_interpretation",
            "role": "gate",
            "field": "functional_interpretation",
            "kind": "value",
            "expected": "costimulation_signature_function_unresolved"
          },
          {
            "name": "cf_contamination_gene_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "MLANA",
              "PMEL",
              "DCT",
              "TYR",
              "TYRP1",
              "SOX10",
              "MITF",
              "MAGEA4"
            ]
          },
          {
            "name": "cf_apc_partner_contamination_gene",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "HLA-DRA",
              "CD68",
              "LYZ",
              "CD14",
              "C1QA",
              "ITGAX",
              "CD1C"
            ]
          },
          {
            "name": "cf_ligand_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "CD274",
              "PDCD1LG2",
              "CD80",
              "CD86"
            ]
          },
          {
            "name": "cf_attribution_causal_or_baseline",
            "role": "hard_fail",
            "field": "$.difference_attributable_to",
            "kind": "avoid",
            "expected": [
              "apc_engagement",
              "tumor_cell_engagement",
              "clonotype_intrinsic_baseline"
            ]
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "engagement_induces_exhausted_state",
              "pre_exhausted_cells_select_into_apc_conjugates"
            ]
          },
          {
            "name": "cf_causal_fabrication",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "resolved_by_pseudotemporal_ordering",
              "resolved_by_perturbation_experiment"
            ]
          },
          {
            "name": "cf_functional_overclaim",
            "role": "hard_fail",
            "field": "$.functional_interpretation",
            "kind": "avoid",
            "expected": [
              "costimulation_licensed_reinvigoration_competent",
              "costimulation_licensed_checkpoint_responsive",
              "tumor_driven_terminal_exhaustion"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-8-claude-code",
      "modelName": "claude-opus-4-8",
      "harness": "claude-code",
      "task": "10x_5p_melanoma_tcr_matched_apc_vs_tumor_exhaustion",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 1,
      "passed": true,
      "score": null,
      "durationS": 963,
      "cost": 3.513105750000001,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "state_discriminating_genes",
            "kind": "list",
            "matchKey": "gene",
            "k": 3,
            "expected": [
              {
                "key": "CTLA4",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TOX",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "ENTPD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "HAVCR2",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TIGIT",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "PDCD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "difference_attributable_to",
            "role": "gate",
            "field": "difference_attributable_to",
            "kind": "value",
            "expected": "covaries_with_apc_conjugate_state"
          },
          {
            "name": "causal_status",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "direction_unresolved_selection_confound"
          },
          {
            "name": "functional_interpretation",
            "role": "gate",
            "field": "functional_interpretation",
            "kind": "value",
            "expected": "costimulation_signature_function_unresolved"
          },
          {
            "name": "cf_contamination_gene_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "MLANA",
              "PMEL",
              "DCT",
              "TYR",
              "TYRP1",
              "SOX10",
              "MITF",
              "MAGEA4"
            ]
          },
          {
            "name": "cf_apc_partner_contamination_gene",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "HLA-DRA",
              "CD68",
              "LYZ",
              "CD14",
              "C1QA",
              "ITGAX",
              "CD1C"
            ]
          },
          {
            "name": "cf_ligand_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "CD274",
              "PDCD1LG2",
              "CD80",
              "CD86"
            ]
          },
          {
            "name": "cf_attribution_causal_or_baseline",
            "role": "hard_fail",
            "field": "$.difference_attributable_to",
            "kind": "avoid",
            "expected": [
              "apc_engagement",
              "tumor_cell_engagement",
              "clonotype_intrinsic_baseline"
            ]
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "engagement_induces_exhausted_state",
              "pre_exhausted_cells_select_into_apc_conjugates"
            ]
          },
          {
            "name": "cf_causal_fabrication",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "resolved_by_pseudotemporal_ordering",
              "resolved_by_perturbation_experiment"
            ]
          },
          {
            "name": "cf_functional_overclaim",
            "role": "hard_fail",
            "field": "$.functional_interpretation",
            "kind": "avoid",
            "expected": [
              "costimulation_licensed_reinvigoration_competent",
              "costimulation_licensed_checkpoint_responsive",
              "tumor_driven_terminal_exhaustion"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-8-claude-code",
      "modelName": "claude-opus-4-8",
      "harness": "claude-code",
      "task": "10x_5p_melanoma_tcr_matched_apc_vs_tumor_exhaustion",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 2,
      "passed": true,
      "score": null,
      "durationS": 973,
      "cost": 2.9595469999999993,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "state_discriminating_genes",
            "kind": "list",
            "matchKey": "gene",
            "k": 3,
            "expected": [
              {
                "key": "CTLA4",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TOX",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "ENTPD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "HAVCR2",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TIGIT",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "PDCD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "difference_attributable_to",
            "role": "gate",
            "field": "difference_attributable_to",
            "kind": "value",
            "expected": "covaries_with_apc_conjugate_state"
          },
          {
            "name": "causal_status",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "direction_unresolved_selection_confound"
          },
          {
            "name": "functional_interpretation",
            "role": "gate",
            "field": "functional_interpretation",
            "kind": "value",
            "expected": "costimulation_signature_function_unresolved"
          },
          {
            "name": "cf_contamination_gene_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "MLANA",
              "PMEL",
              "DCT",
              "TYR",
              "TYRP1",
              "SOX10",
              "MITF",
              "MAGEA4"
            ]
          },
          {
            "name": "cf_apc_partner_contamination_gene",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "HLA-DRA",
              "CD68",
              "LYZ",
              "CD14",
              "C1QA",
              "ITGAX",
              "CD1C"
            ]
          },
          {
            "name": "cf_ligand_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "CD274",
              "PDCD1LG2",
              "CD80",
              "CD86"
            ]
          },
          {
            "name": "cf_attribution_causal_or_baseline",
            "role": "hard_fail",
            "field": "$.difference_attributable_to",
            "kind": "avoid",
            "expected": [
              "apc_engagement",
              "tumor_cell_engagement",
              "clonotype_intrinsic_baseline"
            ]
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "engagement_induces_exhausted_state",
              "pre_exhausted_cells_select_into_apc_conjugates"
            ]
          },
          {
            "name": "cf_causal_fabrication",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "resolved_by_pseudotemporal_ordering",
              "resolved_by_perturbation_experiment"
            ]
          },
          {
            "name": "cf_functional_overclaim",
            "role": "hard_fail",
            "field": "$.functional_interpretation",
            "kind": "avoid",
            "expected": [
              "costimulation_licensed_reinvigoration_competent",
              "costimulation_licensed_checkpoint_responsive",
              "tumor_driven_terminal_exhaustion"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-8-claude-code",
      "modelName": "claude-opus-4-8",
      "harness": "claude-code",
      "task": "10x_5p_melanoma_tcr_matched_apc_vs_tumor_exhaustion",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": true,
      "score": null,
      "durationS": 1090,
      "cost": 3.7239584999999997,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "state_discriminating_genes",
            "kind": "list",
            "matchKey": "gene",
            "k": 3,
            "expected": [
              {
                "key": "CTLA4",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TOX",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "ENTPD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "HAVCR2",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TIGIT",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "PDCD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "difference_attributable_to",
            "role": "gate",
            "field": "difference_attributable_to",
            "kind": "value",
            "expected": "covaries_with_apc_conjugate_state"
          },
          {
            "name": "causal_status",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "direction_unresolved_selection_confound"
          },
          {
            "name": "functional_interpretation",
            "role": "gate",
            "field": "functional_interpretation",
            "kind": "value",
            "expected": "costimulation_signature_function_unresolved"
          },
          {
            "name": "cf_contamination_gene_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "MLANA",
              "PMEL",
              "DCT",
              "TYR",
              "TYRP1",
              "SOX10",
              "MITF",
              "MAGEA4"
            ]
          },
          {
            "name": "cf_apc_partner_contamination_gene",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "HLA-DRA",
              "CD68",
              "LYZ",
              "CD14",
              "C1QA",
              "ITGAX",
              "CD1C"
            ]
          },
          {
            "name": "cf_ligand_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "CD274",
              "PDCD1LG2",
              "CD80",
              "CD86"
            ]
          },
          {
            "name": "cf_attribution_causal_or_baseline",
            "role": "hard_fail",
            "field": "$.difference_attributable_to",
            "kind": "avoid",
            "expected": [
              "apc_engagement",
              "tumor_cell_engagement",
              "clonotype_intrinsic_baseline"
            ]
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "engagement_induces_exhausted_state",
              "pre_exhausted_cells_select_into_apc_conjugates"
            ]
          },
          {
            "name": "cf_causal_fabrication",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "resolved_by_pseudotemporal_ordering",
              "resolved_by_perturbation_experiment"
            ]
          },
          {
            "name": "cf_functional_overclaim",
            "role": "hard_fail",
            "field": "$.functional_interpretation",
            "kind": "avoid",
            "expected": [
              "costimulation_licensed_reinvigoration_competent",
              "costimulation_licensed_checkpoint_responsive",
              "tumor_driven_terminal_exhaustion"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-8-pi",
      "modelName": "claude-opus-4-8",
      "harness": "pi",
      "task": "10x_5p_melanoma_tcr_matched_apc_vs_tumor_exhaustion",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 677,
      "cost": 0.9970902500000001,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "state_discriminating_genes",
            "kind": "list",
            "matchKey": "gene",
            "k": 3,
            "expected": [
              {
                "key": "CTLA4",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TOX",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "ENTPD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "HAVCR2",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TIGIT",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "PDCD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "difference_attributable_to",
            "role": "gate",
            "field": "difference_attributable_to",
            "kind": "value",
            "expected": "covaries_with_apc_conjugate_state"
          },
          {
            "name": "causal_status",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "direction_unresolved_selection_confound"
          },
          {
            "name": "functional_interpretation",
            "role": "gate",
            "field": "functional_interpretation",
            "kind": "value",
            "expected": "costimulation_signature_function_unresolved"
          },
          {
            "name": "cf_contamination_gene_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "MLANA",
              "PMEL",
              "DCT",
              "TYR",
              "TYRP1",
              "SOX10",
              "MITF",
              "MAGEA4"
            ]
          },
          {
            "name": "cf_apc_partner_contamination_gene",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "HLA-DRA",
              "CD68",
              "LYZ",
              "CD14",
              "C1QA",
              "ITGAX",
              "CD1C"
            ]
          },
          {
            "name": "cf_ligand_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "CD274",
              "PDCD1LG2",
              "CD80",
              "CD86"
            ]
          },
          {
            "name": "cf_attribution_causal_or_baseline",
            "role": "hard_fail",
            "field": "$.difference_attributable_to",
            "kind": "avoid",
            "expected": [
              "apc_engagement",
              "tumor_cell_engagement",
              "clonotype_intrinsic_baseline"
            ]
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "engagement_induces_exhausted_state",
              "pre_exhausted_cells_select_into_apc_conjugates"
            ]
          },
          {
            "name": "cf_causal_fabrication",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "resolved_by_pseudotemporal_ordering",
              "resolved_by_perturbation_experiment"
            ]
          },
          {
            "name": "cf_functional_overclaim",
            "role": "hard_fail",
            "field": "$.functional_interpretation",
            "kind": "avoid",
            "expected": [
              "costimulation_licensed_reinvigoration_competent",
              "costimulation_licensed_checkpoint_responsive",
              "tumor_driven_terminal_exhaustion"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-8-pi",
      "modelName": "claude-opus-4-8",
      "harness": "pi",
      "task": "10x_5p_melanoma_tcr_matched_apc_vs_tumor_exhaustion",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 2,
      "passed": true,
      "score": null,
      "durationS": 807,
      "cost": 2.05892425,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "state_discriminating_genes",
            "kind": "list",
            "matchKey": "gene",
            "k": 3,
            "expected": [
              {
                "key": "CTLA4",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TOX",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "ENTPD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "HAVCR2",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TIGIT",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "PDCD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "difference_attributable_to",
            "role": "gate",
            "field": "difference_attributable_to",
            "kind": "value",
            "expected": "covaries_with_apc_conjugate_state"
          },
          {
            "name": "causal_status",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "direction_unresolved_selection_confound"
          },
          {
            "name": "functional_interpretation",
            "role": "gate",
            "field": "functional_interpretation",
            "kind": "value",
            "expected": "costimulation_signature_function_unresolved"
          },
          {
            "name": "cf_contamination_gene_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "MLANA",
              "PMEL",
              "DCT",
              "TYR",
              "TYRP1",
              "SOX10",
              "MITF",
              "MAGEA4"
            ]
          },
          {
            "name": "cf_apc_partner_contamination_gene",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "HLA-DRA",
              "CD68",
              "LYZ",
              "CD14",
              "C1QA",
              "ITGAX",
              "CD1C"
            ]
          },
          {
            "name": "cf_ligand_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "CD274",
              "PDCD1LG2",
              "CD80",
              "CD86"
            ]
          },
          {
            "name": "cf_attribution_causal_or_baseline",
            "role": "hard_fail",
            "field": "$.difference_attributable_to",
            "kind": "avoid",
            "expected": [
              "apc_engagement",
              "tumor_cell_engagement",
              "clonotype_intrinsic_baseline"
            ]
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "engagement_induces_exhausted_state",
              "pre_exhausted_cells_select_into_apc_conjugates"
            ]
          },
          {
            "name": "cf_causal_fabrication",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "resolved_by_pseudotemporal_ordering",
              "resolved_by_perturbation_experiment"
            ]
          },
          {
            "name": "cf_functional_overclaim",
            "role": "hard_fail",
            "field": "$.functional_interpretation",
            "kind": "avoid",
            "expected": [
              "costimulation_licensed_reinvigoration_competent",
              "costimulation_licensed_checkpoint_responsive",
              "tumor_driven_terminal_exhaustion"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-8-pi",
      "modelName": "claude-opus-4-8",
      "harness": "pi",
      "task": "10x_5p_melanoma_tcr_matched_apc_vs_tumor_exhaustion",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 430,
      "cost": 0.9356025000000001,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "state_discriminating_genes",
            "kind": "list",
            "matchKey": "gene",
            "k": 3,
            "expected": [
              {
                "key": "CTLA4",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TOX",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "ENTPD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "HAVCR2",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TIGIT",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "PDCD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "difference_attributable_to",
            "role": "gate",
            "field": "difference_attributable_to",
            "kind": "value",
            "expected": "covaries_with_apc_conjugate_state"
          },
          {
            "name": "causal_status",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "direction_unresolved_selection_confound"
          },
          {
            "name": "functional_interpretation",
            "role": "gate",
            "field": "functional_interpretation",
            "kind": "value",
            "expected": "costimulation_signature_function_unresolved"
          },
          {
            "name": "cf_contamination_gene_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "MLANA",
              "PMEL",
              "DCT",
              "TYR",
              "TYRP1",
              "SOX10",
              "MITF",
              "MAGEA4"
            ]
          },
          {
            "name": "cf_apc_partner_contamination_gene",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "HLA-DRA",
              "CD68",
              "LYZ",
              "CD14",
              "C1QA",
              "ITGAX",
              "CD1C"
            ]
          },
          {
            "name": "cf_ligand_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "CD274",
              "PDCD1LG2",
              "CD80",
              "CD86"
            ]
          },
          {
            "name": "cf_attribution_causal_or_baseline",
            "role": "hard_fail",
            "field": "$.difference_attributable_to",
            "kind": "avoid",
            "expected": [
              "apc_engagement",
              "tumor_cell_engagement",
              "clonotype_intrinsic_baseline"
            ]
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "engagement_induces_exhausted_state",
              "pre_exhausted_cells_select_into_apc_conjugates"
            ]
          },
          {
            "name": "cf_causal_fabrication",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "resolved_by_pseudotemporal_ordering",
              "resolved_by_perturbation_experiment"
            ]
          },
          {
            "name": "cf_functional_overclaim",
            "role": "hard_fail",
            "field": "$.functional_interpretation",
            "kind": "avoid",
            "expected": [
              "costimulation_licensed_reinvigoration_competent",
              "costimulation_licensed_checkpoint_responsive",
              "tumor_driven_terminal_exhaustion"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-4-6-pi",
      "modelName": "claude-sonnet-4-6",
      "harness": "pi",
      "task": "10x_5p_melanoma_tcr_matched_apc_vs_tumor_exhaustion",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 1018,
      "cost": 0.9078576,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "state_discriminating_genes",
            "kind": "list",
            "matchKey": "gene",
            "k": 3,
            "expected": [
              {
                "key": "CTLA4",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TOX",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "ENTPD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "HAVCR2",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TIGIT",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "PDCD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "difference_attributable_to",
            "role": "gate",
            "field": "difference_attributable_to",
            "kind": "value",
            "expected": "covaries_with_apc_conjugate_state"
          },
          {
            "name": "causal_status",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "direction_unresolved_selection_confound"
          },
          {
            "name": "functional_interpretation",
            "role": "gate",
            "field": "functional_interpretation",
            "kind": "value",
            "expected": "costimulation_signature_function_unresolved"
          },
          {
            "name": "cf_contamination_gene_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "MLANA",
              "PMEL",
              "DCT",
              "TYR",
              "TYRP1",
              "SOX10",
              "MITF",
              "MAGEA4"
            ]
          },
          {
            "name": "cf_apc_partner_contamination_gene",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "HLA-DRA",
              "CD68",
              "LYZ",
              "CD14",
              "C1QA",
              "ITGAX",
              "CD1C"
            ]
          },
          {
            "name": "cf_ligand_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "CD274",
              "PDCD1LG2",
              "CD80",
              "CD86"
            ]
          },
          {
            "name": "cf_attribution_causal_or_baseline",
            "role": "hard_fail",
            "field": "$.difference_attributable_to",
            "kind": "avoid",
            "expected": [
              "apc_engagement",
              "tumor_cell_engagement",
              "clonotype_intrinsic_baseline"
            ]
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "engagement_induces_exhausted_state",
              "pre_exhausted_cells_select_into_apc_conjugates"
            ]
          },
          {
            "name": "cf_causal_fabrication",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "resolved_by_pseudotemporal_ordering",
              "resolved_by_perturbation_experiment"
            ]
          },
          {
            "name": "cf_functional_overclaim",
            "role": "hard_fail",
            "field": "$.functional_interpretation",
            "kind": "avoid",
            "expected": [
              "costimulation_licensed_reinvigoration_competent",
              "costimulation_licensed_checkpoint_responsive",
              "tumor_driven_terminal_exhaustion"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-4-6-pi",
      "modelName": "claude-sonnet-4-6",
      "harness": "pi",
      "task": "10x_5p_melanoma_tcr_matched_apc_vs_tumor_exhaustion",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 1546,
      "cost": 0.8074305,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "state_discriminating_genes",
            "kind": "list",
            "matchKey": "gene",
            "k": 3,
            "expected": [
              {
                "key": "CTLA4",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TOX",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "ENTPD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "HAVCR2",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TIGIT",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "PDCD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "difference_attributable_to",
            "role": "gate",
            "field": "difference_attributable_to",
            "kind": "value",
            "expected": "covaries_with_apc_conjugate_state"
          },
          {
            "name": "causal_status",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "direction_unresolved_selection_confound"
          },
          {
            "name": "functional_interpretation",
            "role": "gate",
            "field": "functional_interpretation",
            "kind": "value",
            "expected": "costimulation_signature_function_unresolved"
          },
          {
            "name": "cf_contamination_gene_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "MLANA",
              "PMEL",
              "DCT",
              "TYR",
              "TYRP1",
              "SOX10",
              "MITF",
              "MAGEA4"
            ]
          },
          {
            "name": "cf_apc_partner_contamination_gene",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "HLA-DRA",
              "CD68",
              "LYZ",
              "CD14",
              "C1QA",
              "ITGAX",
              "CD1C"
            ]
          },
          {
            "name": "cf_ligand_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "CD274",
              "PDCD1LG2",
              "CD80",
              "CD86"
            ]
          },
          {
            "name": "cf_attribution_causal_or_baseline",
            "role": "hard_fail",
            "field": "$.difference_attributable_to",
            "kind": "avoid",
            "expected": [
              "apc_engagement",
              "tumor_cell_engagement",
              "clonotype_intrinsic_baseline"
            ]
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "engagement_induces_exhausted_state",
              "pre_exhausted_cells_select_into_apc_conjugates"
            ]
          },
          {
            "name": "cf_causal_fabrication",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "resolved_by_pseudotemporal_ordering",
              "resolved_by_perturbation_experiment"
            ]
          },
          {
            "name": "cf_functional_overclaim",
            "role": "hard_fail",
            "field": "$.functional_interpretation",
            "kind": "avoid",
            "expected": [
              "costimulation_licensed_reinvigoration_competent",
              "costimulation_licensed_checkpoint_responsive",
              "tumor_driven_terminal_exhaustion"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-4-6-pi",
      "modelName": "claude-sonnet-4-6",
      "harness": "pi",
      "task": "10x_5p_melanoma_tcr_matched_apc_vs_tumor_exhaustion",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1815,
      "cost": 1.338945,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "state_discriminating_genes",
            "kind": "list",
            "matchKey": "gene",
            "k": 3,
            "expected": [
              {
                "key": "CTLA4",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TOX",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "ENTPD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "HAVCR2",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TIGIT",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "PDCD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "difference_attributable_to",
            "role": "gate",
            "field": "difference_attributable_to",
            "kind": "value",
            "expected": "covaries_with_apc_conjugate_state"
          },
          {
            "name": "causal_status",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "direction_unresolved_selection_confound"
          },
          {
            "name": "functional_interpretation",
            "role": "gate",
            "field": "functional_interpretation",
            "kind": "value",
            "expected": "costimulation_signature_function_unresolved"
          },
          {
            "name": "cf_contamination_gene_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "MLANA",
              "PMEL",
              "DCT",
              "TYR",
              "TYRP1",
              "SOX10",
              "MITF",
              "MAGEA4"
            ]
          },
          {
            "name": "cf_apc_partner_contamination_gene",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "HLA-DRA",
              "CD68",
              "LYZ",
              "CD14",
              "C1QA",
              "ITGAX",
              "CD1C"
            ]
          },
          {
            "name": "cf_ligand_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "CD274",
              "PDCD1LG2",
              "CD80",
              "CD86"
            ]
          },
          {
            "name": "cf_attribution_causal_or_baseline",
            "role": "hard_fail",
            "field": "$.difference_attributable_to",
            "kind": "avoid",
            "expected": [
              "apc_engagement",
              "tumor_cell_engagement",
              "clonotype_intrinsic_baseline"
            ]
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "engagement_induces_exhausted_state",
              "pre_exhausted_cells_select_into_apc_conjugates"
            ]
          },
          {
            "name": "cf_causal_fabrication",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "resolved_by_pseudotemporal_ordering",
              "resolved_by_perturbation_experiment"
            ]
          },
          {
            "name": "cf_functional_overclaim",
            "role": "hard_fail",
            "field": "$.functional_interpretation",
            "kind": "avoid",
            "expected": [
              "costimulation_licensed_reinvigoration_competent",
              "costimulation_licensed_checkpoint_responsive",
              "tumor_driven_terminal_exhaustion"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gemini-3-1-pro-preview-pi",
      "modelName": "gemini-3.1-pro-preview",
      "harness": "pi",
      "task": "10x_5p_melanoma_tcr_matched_apc_vs_tumor_exhaustion",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 737,
      "cost": 0.7217953999999998,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "state_discriminating_genes",
            "kind": "list",
            "matchKey": "gene",
            "k": 3,
            "expected": [
              {
                "key": "CTLA4",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TOX",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "ENTPD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "HAVCR2",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TIGIT",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "PDCD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "difference_attributable_to",
            "role": "gate",
            "field": "difference_attributable_to",
            "kind": "value",
            "expected": "covaries_with_apc_conjugate_state"
          },
          {
            "name": "causal_status",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "direction_unresolved_selection_confound"
          },
          {
            "name": "functional_interpretation",
            "role": "gate",
            "field": "functional_interpretation",
            "kind": "value",
            "expected": "costimulation_signature_function_unresolved"
          },
          {
            "name": "cf_contamination_gene_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "MLANA",
              "PMEL",
              "DCT",
              "TYR",
              "TYRP1",
              "SOX10",
              "MITF",
              "MAGEA4"
            ]
          },
          {
            "name": "cf_apc_partner_contamination_gene",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "HLA-DRA",
              "CD68",
              "LYZ",
              "CD14",
              "C1QA",
              "ITGAX",
              "CD1C"
            ]
          },
          {
            "name": "cf_ligand_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "CD274",
              "PDCD1LG2",
              "CD80",
              "CD86"
            ]
          },
          {
            "name": "cf_attribution_causal_or_baseline",
            "role": "hard_fail",
            "field": "$.difference_attributable_to",
            "kind": "avoid",
            "expected": [
              "apc_engagement",
              "tumor_cell_engagement",
              "clonotype_intrinsic_baseline"
            ]
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "engagement_induces_exhausted_state",
              "pre_exhausted_cells_select_into_apc_conjugates"
            ]
          },
          {
            "name": "cf_causal_fabrication",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "resolved_by_pseudotemporal_ordering",
              "resolved_by_perturbation_experiment"
            ]
          },
          {
            "name": "cf_functional_overclaim",
            "role": "hard_fail",
            "field": "$.functional_interpretation",
            "kind": "avoid",
            "expected": [
              "costimulation_licensed_reinvigoration_competent",
              "costimulation_licensed_checkpoint_responsive",
              "tumor_driven_terminal_exhaustion"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gemini-3-1-pro-preview-pi",
      "modelName": "gemini-3.1-pro-preview",
      "harness": "pi",
      "task": "10x_5p_melanoma_tcr_matched_apc_vs_tumor_exhaustion",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 1077,
      "cost": 1.0157889999999998,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "state_discriminating_genes",
            "kind": "list",
            "matchKey": "gene",
            "k": 3,
            "expected": [
              {
                "key": "CTLA4",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TOX",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "ENTPD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "HAVCR2",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TIGIT",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "PDCD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "difference_attributable_to",
            "role": "gate",
            "field": "difference_attributable_to",
            "kind": "value",
            "expected": "covaries_with_apc_conjugate_state"
          },
          {
            "name": "causal_status",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "direction_unresolved_selection_confound"
          },
          {
            "name": "functional_interpretation",
            "role": "gate",
            "field": "functional_interpretation",
            "kind": "value",
            "expected": "costimulation_signature_function_unresolved"
          },
          {
            "name": "cf_contamination_gene_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "MLANA",
              "PMEL",
              "DCT",
              "TYR",
              "TYRP1",
              "SOX10",
              "MITF",
              "MAGEA4"
            ]
          },
          {
            "name": "cf_apc_partner_contamination_gene",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "HLA-DRA",
              "CD68",
              "LYZ",
              "CD14",
              "C1QA",
              "ITGAX",
              "CD1C"
            ]
          },
          {
            "name": "cf_ligand_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "CD274",
              "PDCD1LG2",
              "CD80",
              "CD86"
            ]
          },
          {
            "name": "cf_attribution_causal_or_baseline",
            "role": "hard_fail",
            "field": "$.difference_attributable_to",
            "kind": "avoid",
            "expected": [
              "apc_engagement",
              "tumor_cell_engagement",
              "clonotype_intrinsic_baseline"
            ]
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "engagement_induces_exhausted_state",
              "pre_exhausted_cells_select_into_apc_conjugates"
            ]
          },
          {
            "name": "cf_causal_fabrication",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "resolved_by_pseudotemporal_ordering",
              "resolved_by_perturbation_experiment"
            ]
          },
          {
            "name": "cf_functional_overclaim",
            "role": "hard_fail",
            "field": "$.functional_interpretation",
            "kind": "avoid",
            "expected": [
              "costimulation_licensed_reinvigoration_competent",
              "costimulation_licensed_checkpoint_responsive",
              "tumor_driven_terminal_exhaustion"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gemini-3-1-pro-preview-pi",
      "modelName": "gemini-3.1-pro-preview",
      "harness": "pi",
      "task": "10x_5p_melanoma_tcr_matched_apc_vs_tumor_exhaustion",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": true,
      "score": null,
      "durationS": 525,
      "cost": 0.6099092,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "state_discriminating_genes",
            "kind": "list",
            "matchKey": "gene",
            "k": 3,
            "expected": [
              {
                "key": "CTLA4",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TOX",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "ENTPD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "HAVCR2",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TIGIT",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "PDCD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "difference_attributable_to",
            "role": "gate",
            "field": "difference_attributable_to",
            "kind": "value",
            "expected": "covaries_with_apc_conjugate_state"
          },
          {
            "name": "causal_status",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "direction_unresolved_selection_confound"
          },
          {
            "name": "functional_interpretation",
            "role": "gate",
            "field": "functional_interpretation",
            "kind": "value",
            "expected": "costimulation_signature_function_unresolved"
          },
          {
            "name": "cf_contamination_gene_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "MLANA",
              "PMEL",
              "DCT",
              "TYR",
              "TYRP1",
              "SOX10",
              "MITF",
              "MAGEA4"
            ]
          },
          {
            "name": "cf_apc_partner_contamination_gene",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "HLA-DRA",
              "CD68",
              "LYZ",
              "CD14",
              "C1QA",
              "ITGAX",
              "CD1C"
            ]
          },
          {
            "name": "cf_ligand_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "CD274",
              "PDCD1LG2",
              "CD80",
              "CD86"
            ]
          },
          {
            "name": "cf_attribution_causal_or_baseline",
            "role": "hard_fail",
            "field": "$.difference_attributable_to",
            "kind": "avoid",
            "expected": [
              "apc_engagement",
              "tumor_cell_engagement",
              "clonotype_intrinsic_baseline"
            ]
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "engagement_induces_exhausted_state",
              "pre_exhausted_cells_select_into_apc_conjugates"
            ]
          },
          {
            "name": "cf_causal_fabrication",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "resolved_by_pseudotemporal_ordering",
              "resolved_by_perturbation_experiment"
            ]
          },
          {
            "name": "cf_functional_overclaim",
            "role": "hard_fail",
            "field": "$.functional_interpretation",
            "kind": "avoid",
            "expected": [
              "costimulation_licensed_reinvigoration_competent",
              "costimulation_licensed_checkpoint_responsive",
              "tumor_driven_terminal_exhaustion"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gemini-3-5-flash-pi",
      "modelName": "gemini-3.5-flash",
      "harness": "pi",
      "task": "10x_5p_melanoma_tcr_matched_apc_vs_tumor_exhaustion",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 641,
      "cost": 1.3251384,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "state_discriminating_genes",
            "kind": "list",
            "matchKey": "gene",
            "k": 3,
            "expected": [
              {
                "key": "CTLA4",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TOX",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "ENTPD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "HAVCR2",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TIGIT",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "PDCD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "difference_attributable_to",
            "role": "gate",
            "field": "difference_attributable_to",
            "kind": "value",
            "expected": "covaries_with_apc_conjugate_state"
          },
          {
            "name": "causal_status",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "direction_unresolved_selection_confound"
          },
          {
            "name": "functional_interpretation",
            "role": "gate",
            "field": "functional_interpretation",
            "kind": "value",
            "expected": "costimulation_signature_function_unresolved"
          },
          {
            "name": "cf_contamination_gene_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "MLANA",
              "PMEL",
              "DCT",
              "TYR",
              "TYRP1",
              "SOX10",
              "MITF",
              "MAGEA4"
            ]
          },
          {
            "name": "cf_apc_partner_contamination_gene",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "HLA-DRA",
              "CD68",
              "LYZ",
              "CD14",
              "C1QA",
              "ITGAX",
              "CD1C"
            ]
          },
          {
            "name": "cf_ligand_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "CD274",
              "PDCD1LG2",
              "CD80",
              "CD86"
            ]
          },
          {
            "name": "cf_attribution_causal_or_baseline",
            "role": "hard_fail",
            "field": "$.difference_attributable_to",
            "kind": "avoid",
            "expected": [
              "apc_engagement",
              "tumor_cell_engagement",
              "clonotype_intrinsic_baseline"
            ]
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "engagement_induces_exhausted_state",
              "pre_exhausted_cells_select_into_apc_conjugates"
            ]
          },
          {
            "name": "cf_causal_fabrication",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "resolved_by_pseudotemporal_ordering",
              "resolved_by_perturbation_experiment"
            ]
          },
          {
            "name": "cf_functional_overclaim",
            "role": "hard_fail",
            "field": "$.functional_interpretation",
            "kind": "avoid",
            "expected": [
              "costimulation_licensed_reinvigoration_competent",
              "costimulation_licensed_checkpoint_responsive",
              "tumor_driven_terminal_exhaustion"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gemini-3-5-flash-pi",
      "modelName": "gemini-3.5-flash",
      "harness": "pi",
      "task": "10x_5p_melanoma_tcr_matched_apc_vs_tumor_exhaustion",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 1511,
      "cost": 3.2861027999999997,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "state_discriminating_genes",
            "kind": "list",
            "matchKey": "gene",
            "k": 3,
            "expected": [
              {
                "key": "CTLA4",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TOX",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "ENTPD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "HAVCR2",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TIGIT",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "PDCD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "difference_attributable_to",
            "role": "gate",
            "field": "difference_attributable_to",
            "kind": "value",
            "expected": "covaries_with_apc_conjugate_state"
          },
          {
            "name": "causal_status",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "direction_unresolved_selection_confound"
          },
          {
            "name": "functional_interpretation",
            "role": "gate",
            "field": "functional_interpretation",
            "kind": "value",
            "expected": "costimulation_signature_function_unresolved"
          },
          {
            "name": "cf_contamination_gene_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "MLANA",
              "PMEL",
              "DCT",
              "TYR",
              "TYRP1",
              "SOX10",
              "MITF",
              "MAGEA4"
            ]
          },
          {
            "name": "cf_apc_partner_contamination_gene",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "HLA-DRA",
              "CD68",
              "LYZ",
              "CD14",
              "C1QA",
              "ITGAX",
              "CD1C"
            ]
          },
          {
            "name": "cf_ligand_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "CD274",
              "PDCD1LG2",
              "CD80",
              "CD86"
            ]
          },
          {
            "name": "cf_attribution_causal_or_baseline",
            "role": "hard_fail",
            "field": "$.difference_attributable_to",
            "kind": "avoid",
            "expected": [
              "apc_engagement",
              "tumor_cell_engagement",
              "clonotype_intrinsic_baseline"
            ]
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "engagement_induces_exhausted_state",
              "pre_exhausted_cells_select_into_apc_conjugates"
            ]
          },
          {
            "name": "cf_causal_fabrication",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "resolved_by_pseudotemporal_ordering",
              "resolved_by_perturbation_experiment"
            ]
          },
          {
            "name": "cf_functional_overclaim",
            "role": "hard_fail",
            "field": "$.functional_interpretation",
            "kind": "avoid",
            "expected": [
              "costimulation_licensed_reinvigoration_competent",
              "costimulation_licensed_checkpoint_responsive",
              "tumor_driven_terminal_exhaustion"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gemini-3-5-flash-pi",
      "modelName": "gemini-3.5-flash",
      "harness": "pi",
      "task": "10x_5p_melanoma_tcr_matched_apc_vs_tumor_exhaustion",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1343,
      "cost": 1.22903055,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "state_discriminating_genes",
            "kind": "list",
            "matchKey": "gene",
            "k": 3,
            "expected": [
              {
                "key": "CTLA4",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TOX",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "ENTPD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "HAVCR2",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TIGIT",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "PDCD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "difference_attributable_to",
            "role": "gate",
            "field": "difference_attributable_to",
            "kind": "value",
            "expected": "covaries_with_apc_conjugate_state"
          },
          {
            "name": "causal_status",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "direction_unresolved_selection_confound"
          },
          {
            "name": "functional_interpretation",
            "role": "gate",
            "field": "functional_interpretation",
            "kind": "value",
            "expected": "costimulation_signature_function_unresolved"
          },
          {
            "name": "cf_contamination_gene_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "MLANA",
              "PMEL",
              "DCT",
              "TYR",
              "TYRP1",
              "SOX10",
              "MITF",
              "MAGEA4"
            ]
          },
          {
            "name": "cf_apc_partner_contamination_gene",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "HLA-DRA",
              "CD68",
              "LYZ",
              "CD14",
              "C1QA",
              "ITGAX",
              "CD1C"
            ]
          },
          {
            "name": "cf_ligand_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "CD274",
              "PDCD1LG2",
              "CD80",
              "CD86"
            ]
          },
          {
            "name": "cf_attribution_causal_or_baseline",
            "role": "hard_fail",
            "field": "$.difference_attributable_to",
            "kind": "avoid",
            "expected": [
              "apc_engagement",
              "tumor_cell_engagement",
              "clonotype_intrinsic_baseline"
            ]
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "engagement_induces_exhausted_state",
              "pre_exhausted_cells_select_into_apc_conjugates"
            ]
          },
          {
            "name": "cf_causal_fabrication",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "resolved_by_pseudotemporal_ordering",
              "resolved_by_perturbation_experiment"
            ]
          },
          {
            "name": "cf_functional_overclaim",
            "role": "hard_fail",
            "field": "$.functional_interpretation",
            "kind": "avoid",
            "expected": [
              "costimulation_licensed_reinvigoration_competent",
              "costimulation_licensed_checkpoint_responsive",
              "tumor_driven_terminal_exhaustion"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-glm-5p2-pi",
      "modelName": "glm-5p2",
      "harness": "pi",
      "task": "10x_5p_melanoma_tcr_matched_apc_vs_tumor_exhaustion",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 1768,
      "cost": 0.87625296,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "state_discriminating_genes",
            "kind": "list",
            "matchKey": "gene",
            "k": 3,
            "expected": [
              {
                "key": "CTLA4",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TOX",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "ENTPD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "HAVCR2",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TIGIT",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "PDCD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "difference_attributable_to",
            "role": "gate",
            "field": "difference_attributable_to",
            "kind": "value",
            "expected": "covaries_with_apc_conjugate_state"
          },
          {
            "name": "causal_status",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "direction_unresolved_selection_confound"
          },
          {
            "name": "functional_interpretation",
            "role": "gate",
            "field": "functional_interpretation",
            "kind": "value",
            "expected": "costimulation_signature_function_unresolved"
          },
          {
            "name": "cf_contamination_gene_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "MLANA",
              "PMEL",
              "DCT",
              "TYR",
              "TYRP1",
              "SOX10",
              "MITF",
              "MAGEA4"
            ]
          },
          {
            "name": "cf_apc_partner_contamination_gene",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "HLA-DRA",
              "CD68",
              "LYZ",
              "CD14",
              "C1QA",
              "ITGAX",
              "CD1C"
            ]
          },
          {
            "name": "cf_ligand_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "CD274",
              "PDCD1LG2",
              "CD80",
              "CD86"
            ]
          },
          {
            "name": "cf_attribution_causal_or_baseline",
            "role": "hard_fail",
            "field": "$.difference_attributable_to",
            "kind": "avoid",
            "expected": [
              "apc_engagement",
              "tumor_cell_engagement",
              "clonotype_intrinsic_baseline"
            ]
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "engagement_induces_exhausted_state",
              "pre_exhausted_cells_select_into_apc_conjugates"
            ]
          },
          {
            "name": "cf_causal_fabrication",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "resolved_by_pseudotemporal_ordering",
              "resolved_by_perturbation_experiment"
            ]
          },
          {
            "name": "cf_functional_overclaim",
            "role": "hard_fail",
            "field": "$.functional_interpretation",
            "kind": "avoid",
            "expected": [
              "costimulation_licensed_reinvigoration_competent",
              "costimulation_licensed_checkpoint_responsive",
              "tumor_driven_terminal_exhaustion"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-glm-5p2-pi",
      "modelName": "glm-5p2",
      "harness": "pi",
      "task": "10x_5p_melanoma_tcr_matched_apc_vs_tumor_exhaustion",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 1433,
      "cost": 0.58029436,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "state_discriminating_genes",
            "kind": "list",
            "matchKey": "gene",
            "k": 3,
            "expected": [
              {
                "key": "CTLA4",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TOX",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "ENTPD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "HAVCR2",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TIGIT",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "PDCD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "difference_attributable_to",
            "role": "gate",
            "field": "difference_attributable_to",
            "kind": "value",
            "expected": "covaries_with_apc_conjugate_state"
          },
          {
            "name": "causal_status",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "direction_unresolved_selection_confound"
          },
          {
            "name": "functional_interpretation",
            "role": "gate",
            "field": "functional_interpretation",
            "kind": "value",
            "expected": "costimulation_signature_function_unresolved"
          },
          {
            "name": "cf_contamination_gene_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "MLANA",
              "PMEL",
              "DCT",
              "TYR",
              "TYRP1",
              "SOX10",
              "MITF",
              "MAGEA4"
            ]
          },
          {
            "name": "cf_apc_partner_contamination_gene",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "HLA-DRA",
              "CD68",
              "LYZ",
              "CD14",
              "C1QA",
              "ITGAX",
              "CD1C"
            ]
          },
          {
            "name": "cf_ligand_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "CD274",
              "PDCD1LG2",
              "CD80",
              "CD86"
            ]
          },
          {
            "name": "cf_attribution_causal_or_baseline",
            "role": "hard_fail",
            "field": "$.difference_attributable_to",
            "kind": "avoid",
            "expected": [
              "apc_engagement",
              "tumor_cell_engagement",
              "clonotype_intrinsic_baseline"
            ]
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "engagement_induces_exhausted_state",
              "pre_exhausted_cells_select_into_apc_conjugates"
            ]
          },
          {
            "name": "cf_causal_fabrication",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "resolved_by_pseudotemporal_ordering",
              "resolved_by_perturbation_experiment"
            ]
          },
          {
            "name": "cf_functional_overclaim",
            "role": "hard_fail",
            "field": "$.functional_interpretation",
            "kind": "avoid",
            "expected": [
              "costimulation_licensed_reinvigoration_competent",
              "costimulation_licensed_checkpoint_responsive",
              "tumor_driven_terminal_exhaustion"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-glm-5p2-pi",
      "modelName": "glm-5p2",
      "harness": "pi",
      "task": "10x_5p_melanoma_tcr_matched_apc_vs_tumor_exhaustion",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1817,
      "cost": 1.2900087,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "state_discriminating_genes",
            "kind": "list",
            "matchKey": "gene",
            "k": 3,
            "expected": [
              {
                "key": "CTLA4",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TOX",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "ENTPD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "HAVCR2",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TIGIT",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "PDCD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "difference_attributable_to",
            "role": "gate",
            "field": "difference_attributable_to",
            "kind": "value",
            "expected": "covaries_with_apc_conjugate_state"
          },
          {
            "name": "causal_status",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "direction_unresolved_selection_confound"
          },
          {
            "name": "functional_interpretation",
            "role": "gate",
            "field": "functional_interpretation",
            "kind": "value",
            "expected": "costimulation_signature_function_unresolved"
          },
          {
            "name": "cf_contamination_gene_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "MLANA",
              "PMEL",
              "DCT",
              "TYR",
              "TYRP1",
              "SOX10",
              "MITF",
              "MAGEA4"
            ]
          },
          {
            "name": "cf_apc_partner_contamination_gene",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "HLA-DRA",
              "CD68",
              "LYZ",
              "CD14",
              "C1QA",
              "ITGAX",
              "CD1C"
            ]
          },
          {
            "name": "cf_ligand_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "CD274",
              "PDCD1LG2",
              "CD80",
              "CD86"
            ]
          },
          {
            "name": "cf_attribution_causal_or_baseline",
            "role": "hard_fail",
            "field": "$.difference_attributable_to",
            "kind": "avoid",
            "expected": [
              "apc_engagement",
              "tumor_cell_engagement",
              "clonotype_intrinsic_baseline"
            ]
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "engagement_induces_exhausted_state",
              "pre_exhausted_cells_select_into_apc_conjugates"
            ]
          },
          {
            "name": "cf_causal_fabrication",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "resolved_by_pseudotemporal_ordering",
              "resolved_by_perturbation_experiment"
            ]
          },
          {
            "name": "cf_functional_overclaim",
            "role": "hard_fail",
            "field": "$.functional_interpretation",
            "kind": "avoid",
            "expected": [
              "costimulation_licensed_reinvigoration_competent",
              "costimulation_licensed_checkpoint_responsive",
              "tumor_driven_terminal_exhaustion"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-4-codex",
      "modelName": "gpt-5.4",
      "harness": "openai-codex",
      "task": "10x_5p_melanoma_tcr_matched_apc_vs_tumor_exhaustion",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 1,
      "passed": true,
      "score": null,
      "durationS": 361,
      "cost": 0.9262535000000001,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "state_discriminating_genes",
            "kind": "list",
            "matchKey": "gene",
            "k": 3,
            "expected": [
              {
                "key": "CTLA4",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TOX",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "ENTPD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "HAVCR2",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TIGIT",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "PDCD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "difference_attributable_to",
            "role": "gate",
            "field": "difference_attributable_to",
            "kind": "value",
            "expected": "covaries_with_apc_conjugate_state"
          },
          {
            "name": "causal_status",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "direction_unresolved_selection_confound"
          },
          {
            "name": "functional_interpretation",
            "role": "gate",
            "field": "functional_interpretation",
            "kind": "value",
            "expected": "costimulation_signature_function_unresolved"
          },
          {
            "name": "cf_contamination_gene_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "MLANA",
              "PMEL",
              "DCT",
              "TYR",
              "TYRP1",
              "SOX10",
              "MITF",
              "MAGEA4"
            ]
          },
          {
            "name": "cf_apc_partner_contamination_gene",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "HLA-DRA",
              "CD68",
              "LYZ",
              "CD14",
              "C1QA",
              "ITGAX",
              "CD1C"
            ]
          },
          {
            "name": "cf_ligand_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "CD274",
              "PDCD1LG2",
              "CD80",
              "CD86"
            ]
          },
          {
            "name": "cf_attribution_causal_or_baseline",
            "role": "hard_fail",
            "field": "$.difference_attributable_to",
            "kind": "avoid",
            "expected": [
              "apc_engagement",
              "tumor_cell_engagement",
              "clonotype_intrinsic_baseline"
            ]
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "engagement_induces_exhausted_state",
              "pre_exhausted_cells_select_into_apc_conjugates"
            ]
          },
          {
            "name": "cf_causal_fabrication",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "resolved_by_pseudotemporal_ordering",
              "resolved_by_perturbation_experiment"
            ]
          },
          {
            "name": "cf_functional_overclaim",
            "role": "hard_fail",
            "field": "$.functional_interpretation",
            "kind": "avoid",
            "expected": [
              "costimulation_licensed_reinvigoration_competent",
              "costimulation_licensed_checkpoint_responsive",
              "tumor_driven_terminal_exhaustion"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-4-codex",
      "modelName": "gpt-5.4",
      "harness": "openai-codex",
      "task": "10x_5p_melanoma_tcr_matched_apc_vs_tumor_exhaustion",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 443,
      "cost": 1.2316785000000001,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "state_discriminating_genes",
            "kind": "list",
            "matchKey": "gene",
            "k": 3,
            "expected": [
              {
                "key": "CTLA4",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TOX",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "ENTPD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "HAVCR2",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TIGIT",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "PDCD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "difference_attributable_to",
            "role": "gate",
            "field": "difference_attributable_to",
            "kind": "value",
            "expected": "covaries_with_apc_conjugate_state"
          },
          {
            "name": "causal_status",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "direction_unresolved_selection_confound"
          },
          {
            "name": "functional_interpretation",
            "role": "gate",
            "field": "functional_interpretation",
            "kind": "value",
            "expected": "costimulation_signature_function_unresolved"
          },
          {
            "name": "cf_contamination_gene_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "MLANA",
              "PMEL",
              "DCT",
              "TYR",
              "TYRP1",
              "SOX10",
              "MITF",
              "MAGEA4"
            ]
          },
          {
            "name": "cf_apc_partner_contamination_gene",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "HLA-DRA",
              "CD68",
              "LYZ",
              "CD14",
              "C1QA",
              "ITGAX",
              "CD1C"
            ]
          },
          {
            "name": "cf_ligand_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "CD274",
              "PDCD1LG2",
              "CD80",
              "CD86"
            ]
          },
          {
            "name": "cf_attribution_causal_or_baseline",
            "role": "hard_fail",
            "field": "$.difference_attributable_to",
            "kind": "avoid",
            "expected": [
              "apc_engagement",
              "tumor_cell_engagement",
              "clonotype_intrinsic_baseline"
            ]
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "engagement_induces_exhausted_state",
              "pre_exhausted_cells_select_into_apc_conjugates"
            ]
          },
          {
            "name": "cf_causal_fabrication",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "resolved_by_pseudotemporal_ordering",
              "resolved_by_perturbation_experiment"
            ]
          },
          {
            "name": "cf_functional_overclaim",
            "role": "hard_fail",
            "field": "$.functional_interpretation",
            "kind": "avoid",
            "expected": [
              "costimulation_licensed_reinvigoration_competent",
              "costimulation_licensed_checkpoint_responsive",
              "tumor_driven_terminal_exhaustion"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-4-codex",
      "modelName": "gpt-5.4",
      "harness": "openai-codex",
      "task": "10x_5p_melanoma_tcr_matched_apc_vs_tumor_exhaustion",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": true,
      "score": null,
      "durationS": 391,
      "cost": 1.1899250000000001,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "state_discriminating_genes",
            "kind": "list",
            "matchKey": "gene",
            "k": 3,
            "expected": [
              {
                "key": "CTLA4",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TOX",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "ENTPD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "HAVCR2",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TIGIT",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "PDCD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "difference_attributable_to",
            "role": "gate",
            "field": "difference_attributable_to",
            "kind": "value",
            "expected": "covaries_with_apc_conjugate_state"
          },
          {
            "name": "causal_status",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "direction_unresolved_selection_confound"
          },
          {
            "name": "functional_interpretation",
            "role": "gate",
            "field": "functional_interpretation",
            "kind": "value",
            "expected": "costimulation_signature_function_unresolved"
          },
          {
            "name": "cf_contamination_gene_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "MLANA",
              "PMEL",
              "DCT",
              "TYR",
              "TYRP1",
              "SOX10",
              "MITF",
              "MAGEA4"
            ]
          },
          {
            "name": "cf_apc_partner_contamination_gene",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "HLA-DRA",
              "CD68",
              "LYZ",
              "CD14",
              "C1QA",
              "ITGAX",
              "CD1C"
            ]
          },
          {
            "name": "cf_ligand_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "CD274",
              "PDCD1LG2",
              "CD80",
              "CD86"
            ]
          },
          {
            "name": "cf_attribution_causal_or_baseline",
            "role": "hard_fail",
            "field": "$.difference_attributable_to",
            "kind": "avoid",
            "expected": [
              "apc_engagement",
              "tumor_cell_engagement",
              "clonotype_intrinsic_baseline"
            ]
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "engagement_induces_exhausted_state",
              "pre_exhausted_cells_select_into_apc_conjugates"
            ]
          },
          {
            "name": "cf_causal_fabrication",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "resolved_by_pseudotemporal_ordering",
              "resolved_by_perturbation_experiment"
            ]
          },
          {
            "name": "cf_functional_overclaim",
            "role": "hard_fail",
            "field": "$.functional_interpretation",
            "kind": "avoid",
            "expected": [
              "costimulation_licensed_reinvigoration_competent",
              "costimulation_licensed_checkpoint_responsive",
              "tumor_driven_terminal_exhaustion"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-4-pi",
      "modelName": "gpt-5.4",
      "harness": "pi",
      "task": "10x_5p_melanoma_tcr_matched_apc_vs_tumor_exhaustion",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 960,
      "cost": 0.8246555,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "state_discriminating_genes",
            "kind": "list",
            "matchKey": "gene",
            "k": 3,
            "expected": [
              {
                "key": "CTLA4",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TOX",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "ENTPD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "HAVCR2",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TIGIT",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "PDCD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "difference_attributable_to",
            "role": "gate",
            "field": "difference_attributable_to",
            "kind": "value",
            "expected": "covaries_with_apc_conjugate_state"
          },
          {
            "name": "causal_status",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "direction_unresolved_selection_confound"
          },
          {
            "name": "functional_interpretation",
            "role": "gate",
            "field": "functional_interpretation",
            "kind": "value",
            "expected": "costimulation_signature_function_unresolved"
          },
          {
            "name": "cf_contamination_gene_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "MLANA",
              "PMEL",
              "DCT",
              "TYR",
              "TYRP1",
              "SOX10",
              "MITF",
              "MAGEA4"
            ]
          },
          {
            "name": "cf_apc_partner_contamination_gene",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "HLA-DRA",
              "CD68",
              "LYZ",
              "CD14",
              "C1QA",
              "ITGAX",
              "CD1C"
            ]
          },
          {
            "name": "cf_ligand_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "CD274",
              "PDCD1LG2",
              "CD80",
              "CD86"
            ]
          },
          {
            "name": "cf_attribution_causal_or_baseline",
            "role": "hard_fail",
            "field": "$.difference_attributable_to",
            "kind": "avoid",
            "expected": [
              "apc_engagement",
              "tumor_cell_engagement",
              "clonotype_intrinsic_baseline"
            ]
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "engagement_induces_exhausted_state",
              "pre_exhausted_cells_select_into_apc_conjugates"
            ]
          },
          {
            "name": "cf_causal_fabrication",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "resolved_by_pseudotemporal_ordering",
              "resolved_by_perturbation_experiment"
            ]
          },
          {
            "name": "cf_functional_overclaim",
            "role": "hard_fail",
            "field": "$.functional_interpretation",
            "kind": "avoid",
            "expected": [
              "costimulation_licensed_reinvigoration_competent",
              "costimulation_licensed_checkpoint_responsive",
              "tumor_driven_terminal_exhaustion"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-4-pi",
      "modelName": "gpt-5.4",
      "harness": "pi",
      "task": "10x_5p_melanoma_tcr_matched_apc_vs_tumor_exhaustion",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 2,
      "passed": true,
      "score": null,
      "durationS": 939,
      "cost": 0.9140590000000001,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "state_discriminating_genes",
            "kind": "list",
            "matchKey": "gene",
            "k": 3,
            "expected": [
              {
                "key": "CTLA4",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TOX",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "ENTPD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "HAVCR2",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TIGIT",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "PDCD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "difference_attributable_to",
            "role": "gate",
            "field": "difference_attributable_to",
            "kind": "value",
            "expected": "covaries_with_apc_conjugate_state"
          },
          {
            "name": "causal_status",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "direction_unresolved_selection_confound"
          },
          {
            "name": "functional_interpretation",
            "role": "gate",
            "field": "functional_interpretation",
            "kind": "value",
            "expected": "costimulation_signature_function_unresolved"
          },
          {
            "name": "cf_contamination_gene_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "MLANA",
              "PMEL",
              "DCT",
              "TYR",
              "TYRP1",
              "SOX10",
              "MITF",
              "MAGEA4"
            ]
          },
          {
            "name": "cf_apc_partner_contamination_gene",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "HLA-DRA",
              "CD68",
              "LYZ",
              "CD14",
              "C1QA",
              "ITGAX",
              "CD1C"
            ]
          },
          {
            "name": "cf_ligand_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "CD274",
              "PDCD1LG2",
              "CD80",
              "CD86"
            ]
          },
          {
            "name": "cf_attribution_causal_or_baseline",
            "role": "hard_fail",
            "field": "$.difference_attributable_to",
            "kind": "avoid",
            "expected": [
              "apc_engagement",
              "tumor_cell_engagement",
              "clonotype_intrinsic_baseline"
            ]
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "engagement_induces_exhausted_state",
              "pre_exhausted_cells_select_into_apc_conjugates"
            ]
          },
          {
            "name": "cf_causal_fabrication",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "resolved_by_pseudotemporal_ordering",
              "resolved_by_perturbation_experiment"
            ]
          },
          {
            "name": "cf_functional_overclaim",
            "role": "hard_fail",
            "field": "$.functional_interpretation",
            "kind": "avoid",
            "expected": [
              "costimulation_licensed_reinvigoration_competent",
              "costimulation_licensed_checkpoint_responsive",
              "tumor_driven_terminal_exhaustion"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-4-pi",
      "modelName": "gpt-5.4",
      "harness": "pi",
      "task": "10x_5p_melanoma_tcr_matched_apc_vs_tumor_exhaustion",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 673,
      "cost": 0.8454854999999998,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "state_discriminating_genes",
            "kind": "list",
            "matchKey": "gene",
            "k": 3,
            "expected": [
              {
                "key": "CTLA4",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TOX",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "ENTPD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "HAVCR2",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TIGIT",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "PDCD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "difference_attributable_to",
            "role": "gate",
            "field": "difference_attributable_to",
            "kind": "value",
            "expected": "covaries_with_apc_conjugate_state"
          },
          {
            "name": "causal_status",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "direction_unresolved_selection_confound"
          },
          {
            "name": "functional_interpretation",
            "role": "gate",
            "field": "functional_interpretation",
            "kind": "value",
            "expected": "costimulation_signature_function_unresolved"
          },
          {
            "name": "cf_contamination_gene_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "MLANA",
              "PMEL",
              "DCT",
              "TYR",
              "TYRP1",
              "SOX10",
              "MITF",
              "MAGEA4"
            ]
          },
          {
            "name": "cf_apc_partner_contamination_gene",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "HLA-DRA",
              "CD68",
              "LYZ",
              "CD14",
              "C1QA",
              "ITGAX",
              "CD1C"
            ]
          },
          {
            "name": "cf_ligand_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "CD274",
              "PDCD1LG2",
              "CD80",
              "CD86"
            ]
          },
          {
            "name": "cf_attribution_causal_or_baseline",
            "role": "hard_fail",
            "field": "$.difference_attributable_to",
            "kind": "avoid",
            "expected": [
              "apc_engagement",
              "tumor_cell_engagement",
              "clonotype_intrinsic_baseline"
            ]
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "engagement_induces_exhausted_state",
              "pre_exhausted_cells_select_into_apc_conjugates"
            ]
          },
          {
            "name": "cf_causal_fabrication",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "resolved_by_pseudotemporal_ordering",
              "resolved_by_perturbation_experiment"
            ]
          },
          {
            "name": "cf_functional_overclaim",
            "role": "hard_fail",
            "field": "$.functional_interpretation",
            "kind": "avoid",
            "expected": [
              "costimulation_licensed_reinvigoration_competent",
              "costimulation_licensed_checkpoint_responsive",
              "tumor_driven_terminal_exhaustion"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-5-codex",
      "modelName": "gpt-5.5",
      "harness": "openai-codex",
      "task": "10x_5p_melanoma_tcr_matched_apc_vs_tumor_exhaustion",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 1,
      "passed": true,
      "score": null,
      "durationS": 465,
      "cost": 1.60117,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "state_discriminating_genes",
            "kind": "list",
            "matchKey": "gene",
            "k": 3,
            "expected": [
              {
                "key": "CTLA4",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TOX",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "ENTPD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "HAVCR2",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TIGIT",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "PDCD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "difference_attributable_to",
            "role": "gate",
            "field": "difference_attributable_to",
            "kind": "value",
            "expected": "covaries_with_apc_conjugate_state"
          },
          {
            "name": "causal_status",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "direction_unresolved_selection_confound"
          },
          {
            "name": "functional_interpretation",
            "role": "gate",
            "field": "functional_interpretation",
            "kind": "value",
            "expected": "costimulation_signature_function_unresolved"
          },
          {
            "name": "cf_contamination_gene_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "MLANA",
              "PMEL",
              "DCT",
              "TYR",
              "TYRP1",
              "SOX10",
              "MITF",
              "MAGEA4"
            ]
          },
          {
            "name": "cf_apc_partner_contamination_gene",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "HLA-DRA",
              "CD68",
              "LYZ",
              "CD14",
              "C1QA",
              "ITGAX",
              "CD1C"
            ]
          },
          {
            "name": "cf_ligand_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "CD274",
              "PDCD1LG2",
              "CD80",
              "CD86"
            ]
          },
          {
            "name": "cf_attribution_causal_or_baseline",
            "role": "hard_fail",
            "field": "$.difference_attributable_to",
            "kind": "avoid",
            "expected": [
              "apc_engagement",
              "tumor_cell_engagement",
              "clonotype_intrinsic_baseline"
            ]
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "engagement_induces_exhausted_state",
              "pre_exhausted_cells_select_into_apc_conjugates"
            ]
          },
          {
            "name": "cf_causal_fabrication",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "resolved_by_pseudotemporal_ordering",
              "resolved_by_perturbation_experiment"
            ]
          },
          {
            "name": "cf_functional_overclaim",
            "role": "hard_fail",
            "field": "$.functional_interpretation",
            "kind": "avoid",
            "expected": [
              "costimulation_licensed_reinvigoration_competent",
              "costimulation_licensed_checkpoint_responsive",
              "tumor_driven_terminal_exhaustion"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-5-codex",
      "modelName": "gpt-5.5",
      "harness": "openai-codex",
      "task": "10x_5p_melanoma_tcr_matched_apc_vs_tumor_exhaustion",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 2,
      "passed": true,
      "score": null,
      "durationS": 441,
      "cost": 1.516931,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "state_discriminating_genes",
            "kind": "list",
            "matchKey": "gene",
            "k": 3,
            "expected": [
              {
                "key": "CTLA4",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TOX",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "ENTPD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "HAVCR2",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TIGIT",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "PDCD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "difference_attributable_to",
            "role": "gate",
            "field": "difference_attributable_to",
            "kind": "value",
            "expected": "covaries_with_apc_conjugate_state"
          },
          {
            "name": "causal_status",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "direction_unresolved_selection_confound"
          },
          {
            "name": "functional_interpretation",
            "role": "gate",
            "field": "functional_interpretation",
            "kind": "value",
            "expected": "costimulation_signature_function_unresolved"
          },
          {
            "name": "cf_contamination_gene_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "MLANA",
              "PMEL",
              "DCT",
              "TYR",
              "TYRP1",
              "SOX10",
              "MITF",
              "MAGEA4"
            ]
          },
          {
            "name": "cf_apc_partner_contamination_gene",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "HLA-DRA",
              "CD68",
              "LYZ",
              "CD14",
              "C1QA",
              "ITGAX",
              "CD1C"
            ]
          },
          {
            "name": "cf_ligand_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "CD274",
              "PDCD1LG2",
              "CD80",
              "CD86"
            ]
          },
          {
            "name": "cf_attribution_causal_or_baseline",
            "role": "hard_fail",
            "field": "$.difference_attributable_to",
            "kind": "avoid",
            "expected": [
              "apc_engagement",
              "tumor_cell_engagement",
              "clonotype_intrinsic_baseline"
            ]
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "engagement_induces_exhausted_state",
              "pre_exhausted_cells_select_into_apc_conjugates"
            ]
          },
          {
            "name": "cf_causal_fabrication",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "resolved_by_pseudotemporal_ordering",
              "resolved_by_perturbation_experiment"
            ]
          },
          {
            "name": "cf_functional_overclaim",
            "role": "hard_fail",
            "field": "$.functional_interpretation",
            "kind": "avoid",
            "expected": [
              "costimulation_licensed_reinvigoration_competent",
              "costimulation_licensed_checkpoint_responsive",
              "tumor_driven_terminal_exhaustion"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-5-codex",
      "modelName": "gpt-5.5",
      "harness": "openai-codex",
      "task": "10x_5p_melanoma_tcr_matched_apc_vs_tumor_exhaustion",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": true,
      "score": null,
      "durationS": 429,
      "cost": 1.119285,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "state_discriminating_genes",
            "kind": "list",
            "matchKey": "gene",
            "k": 3,
            "expected": [
              {
                "key": "CTLA4",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TOX",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "ENTPD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "HAVCR2",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TIGIT",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "PDCD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "difference_attributable_to",
            "role": "gate",
            "field": "difference_attributable_to",
            "kind": "value",
            "expected": "covaries_with_apc_conjugate_state"
          },
          {
            "name": "causal_status",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "direction_unresolved_selection_confound"
          },
          {
            "name": "functional_interpretation",
            "role": "gate",
            "field": "functional_interpretation",
            "kind": "value",
            "expected": "costimulation_signature_function_unresolved"
          },
          {
            "name": "cf_contamination_gene_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "MLANA",
              "PMEL",
              "DCT",
              "TYR",
              "TYRP1",
              "SOX10",
              "MITF",
              "MAGEA4"
            ]
          },
          {
            "name": "cf_apc_partner_contamination_gene",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "HLA-DRA",
              "CD68",
              "LYZ",
              "CD14",
              "C1QA",
              "ITGAX",
              "CD1C"
            ]
          },
          {
            "name": "cf_ligand_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "CD274",
              "PDCD1LG2",
              "CD80",
              "CD86"
            ]
          },
          {
            "name": "cf_attribution_causal_or_baseline",
            "role": "hard_fail",
            "field": "$.difference_attributable_to",
            "kind": "avoid",
            "expected": [
              "apc_engagement",
              "tumor_cell_engagement",
              "clonotype_intrinsic_baseline"
            ]
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "engagement_induces_exhausted_state",
              "pre_exhausted_cells_select_into_apc_conjugates"
            ]
          },
          {
            "name": "cf_causal_fabrication",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "resolved_by_pseudotemporal_ordering",
              "resolved_by_perturbation_experiment"
            ]
          },
          {
            "name": "cf_functional_overclaim",
            "role": "hard_fail",
            "field": "$.functional_interpretation",
            "kind": "avoid",
            "expected": [
              "costimulation_licensed_reinvigoration_competent",
              "costimulation_licensed_checkpoint_responsive",
              "tumor_driven_terminal_exhaustion"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-5-pi",
      "modelName": "gpt-5.5",
      "harness": "pi",
      "task": "10x_5p_melanoma_tcr_matched_apc_vs_tumor_exhaustion",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 1,
      "passed": true,
      "score": null,
      "durationS": 890,
      "cost": 1.635544,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "state_discriminating_genes",
            "kind": "list",
            "matchKey": "gene",
            "k": 3,
            "expected": [
              {
                "key": "CTLA4",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TOX",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "ENTPD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "HAVCR2",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TIGIT",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "PDCD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "difference_attributable_to",
            "role": "gate",
            "field": "difference_attributable_to",
            "kind": "value",
            "expected": "covaries_with_apc_conjugate_state"
          },
          {
            "name": "causal_status",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "direction_unresolved_selection_confound"
          },
          {
            "name": "functional_interpretation",
            "role": "gate",
            "field": "functional_interpretation",
            "kind": "value",
            "expected": "costimulation_signature_function_unresolved"
          },
          {
            "name": "cf_contamination_gene_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "MLANA",
              "PMEL",
              "DCT",
              "TYR",
              "TYRP1",
              "SOX10",
              "MITF",
              "MAGEA4"
            ]
          },
          {
            "name": "cf_apc_partner_contamination_gene",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "HLA-DRA",
              "CD68",
              "LYZ",
              "CD14",
              "C1QA",
              "ITGAX",
              "CD1C"
            ]
          },
          {
            "name": "cf_ligand_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "CD274",
              "PDCD1LG2",
              "CD80",
              "CD86"
            ]
          },
          {
            "name": "cf_attribution_causal_or_baseline",
            "role": "hard_fail",
            "field": "$.difference_attributable_to",
            "kind": "avoid",
            "expected": [
              "apc_engagement",
              "tumor_cell_engagement",
              "clonotype_intrinsic_baseline"
            ]
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "engagement_induces_exhausted_state",
              "pre_exhausted_cells_select_into_apc_conjugates"
            ]
          },
          {
            "name": "cf_causal_fabrication",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "resolved_by_pseudotemporal_ordering",
              "resolved_by_perturbation_experiment"
            ]
          },
          {
            "name": "cf_functional_overclaim",
            "role": "hard_fail",
            "field": "$.functional_interpretation",
            "kind": "avoid",
            "expected": [
              "costimulation_licensed_reinvigoration_competent",
              "costimulation_licensed_checkpoint_responsive",
              "tumor_driven_terminal_exhaustion"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-5-pi",
      "modelName": "gpt-5.5",
      "harness": "pi",
      "task": "10x_5p_melanoma_tcr_matched_apc_vs_tumor_exhaustion",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 2,
      "passed": true,
      "score": null,
      "durationS": 630,
      "cost": 1.422584,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "state_discriminating_genes",
            "kind": "list",
            "matchKey": "gene",
            "k": 3,
            "expected": [
              {
                "key": "CTLA4",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TOX",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "ENTPD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "HAVCR2",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TIGIT",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "PDCD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "difference_attributable_to",
            "role": "gate",
            "field": "difference_attributable_to",
            "kind": "value",
            "expected": "covaries_with_apc_conjugate_state"
          },
          {
            "name": "causal_status",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "direction_unresolved_selection_confound"
          },
          {
            "name": "functional_interpretation",
            "role": "gate",
            "field": "functional_interpretation",
            "kind": "value",
            "expected": "costimulation_signature_function_unresolved"
          },
          {
            "name": "cf_contamination_gene_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "MLANA",
              "PMEL",
              "DCT",
              "TYR",
              "TYRP1",
              "SOX10",
              "MITF",
              "MAGEA4"
            ]
          },
          {
            "name": "cf_apc_partner_contamination_gene",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "HLA-DRA",
              "CD68",
              "LYZ",
              "CD14",
              "C1QA",
              "ITGAX",
              "CD1C"
            ]
          },
          {
            "name": "cf_ligand_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "CD274",
              "PDCD1LG2",
              "CD80",
              "CD86"
            ]
          },
          {
            "name": "cf_attribution_causal_or_baseline",
            "role": "hard_fail",
            "field": "$.difference_attributable_to",
            "kind": "avoid",
            "expected": [
              "apc_engagement",
              "tumor_cell_engagement",
              "clonotype_intrinsic_baseline"
            ]
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "engagement_induces_exhausted_state",
              "pre_exhausted_cells_select_into_apc_conjugates"
            ]
          },
          {
            "name": "cf_causal_fabrication",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "resolved_by_pseudotemporal_ordering",
              "resolved_by_perturbation_experiment"
            ]
          },
          {
            "name": "cf_functional_overclaim",
            "role": "hard_fail",
            "field": "$.functional_interpretation",
            "kind": "avoid",
            "expected": [
              "costimulation_licensed_reinvigoration_competent",
              "costimulation_licensed_checkpoint_responsive",
              "tumor_driven_terminal_exhaustion"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-5-pi",
      "modelName": "gpt-5.5",
      "harness": "pi",
      "task": "10x_5p_melanoma_tcr_matched_apc_vs_tumor_exhaustion",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": true,
      "score": null,
      "durationS": 371,
      "cost": 1.3202180000000003,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "state_discriminating_genes",
            "kind": "list",
            "matchKey": "gene",
            "k": 3,
            "expected": [
              {
                "key": "CTLA4",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TOX",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "ENTPD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "HAVCR2",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TIGIT",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "PDCD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "difference_attributable_to",
            "role": "gate",
            "field": "difference_attributable_to",
            "kind": "value",
            "expected": "covaries_with_apc_conjugate_state"
          },
          {
            "name": "causal_status",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "direction_unresolved_selection_confound"
          },
          {
            "name": "functional_interpretation",
            "role": "gate",
            "field": "functional_interpretation",
            "kind": "value",
            "expected": "costimulation_signature_function_unresolved"
          },
          {
            "name": "cf_contamination_gene_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "MLANA",
              "PMEL",
              "DCT",
              "TYR",
              "TYRP1",
              "SOX10",
              "MITF",
              "MAGEA4"
            ]
          },
          {
            "name": "cf_apc_partner_contamination_gene",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "HLA-DRA",
              "CD68",
              "LYZ",
              "CD14",
              "C1QA",
              "ITGAX",
              "CD1C"
            ]
          },
          {
            "name": "cf_ligand_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "CD274",
              "PDCD1LG2",
              "CD80",
              "CD86"
            ]
          },
          {
            "name": "cf_attribution_causal_or_baseline",
            "role": "hard_fail",
            "field": "$.difference_attributable_to",
            "kind": "avoid",
            "expected": [
              "apc_engagement",
              "tumor_cell_engagement",
              "clonotype_intrinsic_baseline"
            ]
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "engagement_induces_exhausted_state",
              "pre_exhausted_cells_select_into_apc_conjugates"
            ]
          },
          {
            "name": "cf_causal_fabrication",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "resolved_by_pseudotemporal_ordering",
              "resolved_by_perturbation_experiment"
            ]
          },
          {
            "name": "cf_functional_overclaim",
            "role": "hard_fail",
            "field": "$.functional_interpretation",
            "kind": "avoid",
            "expected": [
              "costimulation_licensed_reinvigoration_competent",
              "costimulation_licensed_checkpoint_responsive",
              "tumor_driven_terminal_exhaustion"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-20-beta-reasoning-pi",
      "modelName": "grok-4.20-beta-0309-reasoning",
      "harness": "pi",
      "task": "10x_5p_melanoma_tcr_matched_apc_vs_tumor_exhaustion",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 958,
      "cost": 0.05896725,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "state_discriminating_genes",
            "kind": "list",
            "matchKey": "gene",
            "k": 3,
            "expected": [
              {
                "key": "CTLA4",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TOX",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "ENTPD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "HAVCR2",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TIGIT",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "PDCD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "difference_attributable_to",
            "role": "gate",
            "field": "difference_attributable_to",
            "kind": "value",
            "expected": "covaries_with_apc_conjugate_state"
          },
          {
            "name": "causal_status",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "direction_unresolved_selection_confound"
          },
          {
            "name": "functional_interpretation",
            "role": "gate",
            "field": "functional_interpretation",
            "kind": "value",
            "expected": "costimulation_signature_function_unresolved"
          },
          {
            "name": "cf_contamination_gene_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "MLANA",
              "PMEL",
              "DCT",
              "TYR",
              "TYRP1",
              "SOX10",
              "MITF",
              "MAGEA4"
            ]
          },
          {
            "name": "cf_apc_partner_contamination_gene",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "HLA-DRA",
              "CD68",
              "LYZ",
              "CD14",
              "C1QA",
              "ITGAX",
              "CD1C"
            ]
          },
          {
            "name": "cf_ligand_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "CD274",
              "PDCD1LG2",
              "CD80",
              "CD86"
            ]
          },
          {
            "name": "cf_attribution_causal_or_baseline",
            "role": "hard_fail",
            "field": "$.difference_attributable_to",
            "kind": "avoid",
            "expected": [
              "apc_engagement",
              "tumor_cell_engagement",
              "clonotype_intrinsic_baseline"
            ]
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "engagement_induces_exhausted_state",
              "pre_exhausted_cells_select_into_apc_conjugates"
            ]
          },
          {
            "name": "cf_causal_fabrication",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "resolved_by_pseudotemporal_ordering",
              "resolved_by_perturbation_experiment"
            ]
          },
          {
            "name": "cf_functional_overclaim",
            "role": "hard_fail",
            "field": "$.functional_interpretation",
            "kind": "avoid",
            "expected": [
              "costimulation_licensed_reinvigoration_competent",
              "costimulation_licensed_checkpoint_responsive",
              "tumor_driven_terminal_exhaustion"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-20-beta-reasoning-pi",
      "modelName": "grok-4.20-beta-0309-reasoning",
      "harness": "pi",
      "task": "10x_5p_melanoma_tcr_matched_apc_vs_tumor_exhaustion",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 480,
      "cost": 0.05436425,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "state_discriminating_genes",
            "kind": "list",
            "matchKey": "gene",
            "k": 3,
            "expected": [
              {
                "key": "CTLA4",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TOX",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "ENTPD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "HAVCR2",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TIGIT",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "PDCD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "difference_attributable_to",
            "role": "gate",
            "field": "difference_attributable_to",
            "kind": "value",
            "expected": "covaries_with_apc_conjugate_state"
          },
          {
            "name": "causal_status",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "direction_unresolved_selection_confound"
          },
          {
            "name": "functional_interpretation",
            "role": "gate",
            "field": "functional_interpretation",
            "kind": "value",
            "expected": "costimulation_signature_function_unresolved"
          },
          {
            "name": "cf_contamination_gene_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "MLANA",
              "PMEL",
              "DCT",
              "TYR",
              "TYRP1",
              "SOX10",
              "MITF",
              "MAGEA4"
            ]
          },
          {
            "name": "cf_apc_partner_contamination_gene",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "HLA-DRA",
              "CD68",
              "LYZ",
              "CD14",
              "C1QA",
              "ITGAX",
              "CD1C"
            ]
          },
          {
            "name": "cf_ligand_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "CD274",
              "PDCD1LG2",
              "CD80",
              "CD86"
            ]
          },
          {
            "name": "cf_attribution_causal_or_baseline",
            "role": "hard_fail",
            "field": "$.difference_attributable_to",
            "kind": "avoid",
            "expected": [
              "apc_engagement",
              "tumor_cell_engagement",
              "clonotype_intrinsic_baseline"
            ]
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "engagement_induces_exhausted_state",
              "pre_exhausted_cells_select_into_apc_conjugates"
            ]
          },
          {
            "name": "cf_causal_fabrication",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "resolved_by_pseudotemporal_ordering",
              "resolved_by_perturbation_experiment"
            ]
          },
          {
            "name": "cf_functional_overclaim",
            "role": "hard_fail",
            "field": "$.functional_interpretation",
            "kind": "avoid",
            "expected": [
              "costimulation_licensed_reinvigoration_competent",
              "costimulation_licensed_checkpoint_responsive",
              "tumor_driven_terminal_exhaustion"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-20-beta-reasoning-pi",
      "modelName": "grok-4.20-beta-0309-reasoning",
      "harness": "pi",
      "task": "10x_5p_melanoma_tcr_matched_apc_vs_tumor_exhaustion",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1112,
      "cost": 0.06064594999999999,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "state_discriminating_genes",
            "kind": "list",
            "matchKey": "gene",
            "k": 3,
            "expected": [
              {
                "key": "CTLA4",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TOX",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "ENTPD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "HAVCR2",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TIGIT",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "PDCD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "difference_attributable_to",
            "role": "gate",
            "field": "difference_attributable_to",
            "kind": "value",
            "expected": "covaries_with_apc_conjugate_state"
          },
          {
            "name": "causal_status",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "direction_unresolved_selection_confound"
          },
          {
            "name": "functional_interpretation",
            "role": "gate",
            "field": "functional_interpretation",
            "kind": "value",
            "expected": "costimulation_signature_function_unresolved"
          },
          {
            "name": "cf_contamination_gene_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "MLANA",
              "PMEL",
              "DCT",
              "TYR",
              "TYRP1",
              "SOX10",
              "MITF",
              "MAGEA4"
            ]
          },
          {
            "name": "cf_apc_partner_contamination_gene",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "HLA-DRA",
              "CD68",
              "LYZ",
              "CD14",
              "C1QA",
              "ITGAX",
              "CD1C"
            ]
          },
          {
            "name": "cf_ligand_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "CD274",
              "PDCD1LG2",
              "CD80",
              "CD86"
            ]
          },
          {
            "name": "cf_attribution_causal_or_baseline",
            "role": "hard_fail",
            "field": "$.difference_attributable_to",
            "kind": "avoid",
            "expected": [
              "apc_engagement",
              "tumor_cell_engagement",
              "clonotype_intrinsic_baseline"
            ]
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "engagement_induces_exhausted_state",
              "pre_exhausted_cells_select_into_apc_conjugates"
            ]
          },
          {
            "name": "cf_causal_fabrication",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "resolved_by_pseudotemporal_ordering",
              "resolved_by_perturbation_experiment"
            ]
          },
          {
            "name": "cf_functional_overclaim",
            "role": "hard_fail",
            "field": "$.functional_interpretation",
            "kind": "avoid",
            "expected": [
              "costimulation_licensed_reinvigoration_competent",
              "costimulation_licensed_checkpoint_responsive",
              "tumor_driven_terminal_exhaustion"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-3-pi",
      "modelName": "grok-4.3",
      "harness": "pi",
      "task": "10x_5p_melanoma_tcr_matched_apc_vs_tumor_exhaustion",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 239,
      "cost": 0.01686355,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "state_discriminating_genes",
            "kind": "list",
            "matchKey": "gene",
            "k": 3,
            "expected": [
              {
                "key": "CTLA4",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TOX",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "ENTPD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "HAVCR2",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TIGIT",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "PDCD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "difference_attributable_to",
            "role": "gate",
            "field": "difference_attributable_to",
            "kind": "value",
            "expected": "covaries_with_apc_conjugate_state"
          },
          {
            "name": "causal_status",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "direction_unresolved_selection_confound"
          },
          {
            "name": "functional_interpretation",
            "role": "gate",
            "field": "functional_interpretation",
            "kind": "value",
            "expected": "costimulation_signature_function_unresolved"
          },
          {
            "name": "cf_contamination_gene_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "MLANA",
              "PMEL",
              "DCT",
              "TYR",
              "TYRP1",
              "SOX10",
              "MITF",
              "MAGEA4"
            ]
          },
          {
            "name": "cf_apc_partner_contamination_gene",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "HLA-DRA",
              "CD68",
              "LYZ",
              "CD14",
              "C1QA",
              "ITGAX",
              "CD1C"
            ]
          },
          {
            "name": "cf_ligand_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "CD274",
              "PDCD1LG2",
              "CD80",
              "CD86"
            ]
          },
          {
            "name": "cf_attribution_causal_or_baseline",
            "role": "hard_fail",
            "field": "$.difference_attributable_to",
            "kind": "avoid",
            "expected": [
              "apc_engagement",
              "tumor_cell_engagement",
              "clonotype_intrinsic_baseline"
            ]
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "engagement_induces_exhausted_state",
              "pre_exhausted_cells_select_into_apc_conjugates"
            ]
          },
          {
            "name": "cf_causal_fabrication",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "resolved_by_pseudotemporal_ordering",
              "resolved_by_perturbation_experiment"
            ]
          },
          {
            "name": "cf_functional_overclaim",
            "role": "hard_fail",
            "field": "$.functional_interpretation",
            "kind": "avoid",
            "expected": [
              "costimulation_licensed_reinvigoration_competent",
              "costimulation_licensed_checkpoint_responsive",
              "tumor_driven_terminal_exhaustion"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-3-pi",
      "modelName": "grok-4.3",
      "harness": "pi",
      "task": "10x_5p_melanoma_tcr_matched_apc_vs_tumor_exhaustion",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 192,
      "cost": 0.02413635,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "state_discriminating_genes",
            "kind": "list",
            "matchKey": "gene",
            "k": 3,
            "expected": [
              {
                "key": "CTLA4",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TOX",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "ENTPD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "HAVCR2",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TIGIT",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "PDCD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "difference_attributable_to",
            "role": "gate",
            "field": "difference_attributable_to",
            "kind": "value",
            "expected": "covaries_with_apc_conjugate_state"
          },
          {
            "name": "causal_status",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "direction_unresolved_selection_confound"
          },
          {
            "name": "functional_interpretation",
            "role": "gate",
            "field": "functional_interpretation",
            "kind": "value",
            "expected": "costimulation_signature_function_unresolved"
          },
          {
            "name": "cf_contamination_gene_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "MLANA",
              "PMEL",
              "DCT",
              "TYR",
              "TYRP1",
              "SOX10",
              "MITF",
              "MAGEA4"
            ]
          },
          {
            "name": "cf_apc_partner_contamination_gene",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "HLA-DRA",
              "CD68",
              "LYZ",
              "CD14",
              "C1QA",
              "ITGAX",
              "CD1C"
            ]
          },
          {
            "name": "cf_ligand_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "CD274",
              "PDCD1LG2",
              "CD80",
              "CD86"
            ]
          },
          {
            "name": "cf_attribution_causal_or_baseline",
            "role": "hard_fail",
            "field": "$.difference_attributable_to",
            "kind": "avoid",
            "expected": [
              "apc_engagement",
              "tumor_cell_engagement",
              "clonotype_intrinsic_baseline"
            ]
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "engagement_induces_exhausted_state",
              "pre_exhausted_cells_select_into_apc_conjugates"
            ]
          },
          {
            "name": "cf_causal_fabrication",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "resolved_by_pseudotemporal_ordering",
              "resolved_by_perturbation_experiment"
            ]
          },
          {
            "name": "cf_functional_overclaim",
            "role": "hard_fail",
            "field": "$.functional_interpretation",
            "kind": "avoid",
            "expected": [
              "costimulation_licensed_reinvigoration_competent",
              "costimulation_licensed_checkpoint_responsive",
              "tumor_driven_terminal_exhaustion"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-3-pi",
      "modelName": "grok-4.3",
      "harness": "pi",
      "task": "10x_5p_melanoma_tcr_matched_apc_vs_tumor_exhaustion",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 110,
      "cost": 0.013153750000000002,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "state_discriminating_genes",
            "kind": "list",
            "matchKey": "gene",
            "k": 3,
            "expected": [
              {
                "key": "CTLA4",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TOX",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "ENTPD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "HAVCR2",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TIGIT",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "PDCD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "difference_attributable_to",
            "role": "gate",
            "field": "difference_attributable_to",
            "kind": "value",
            "expected": "covaries_with_apc_conjugate_state"
          },
          {
            "name": "causal_status",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "direction_unresolved_selection_confound"
          },
          {
            "name": "functional_interpretation",
            "role": "gate",
            "field": "functional_interpretation",
            "kind": "value",
            "expected": "costimulation_signature_function_unresolved"
          },
          {
            "name": "cf_contamination_gene_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "MLANA",
              "PMEL",
              "DCT",
              "TYR",
              "TYRP1",
              "SOX10",
              "MITF",
              "MAGEA4"
            ]
          },
          {
            "name": "cf_apc_partner_contamination_gene",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "HLA-DRA",
              "CD68",
              "LYZ",
              "CD14",
              "C1QA",
              "ITGAX",
              "CD1C"
            ]
          },
          {
            "name": "cf_ligand_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "CD274",
              "PDCD1LG2",
              "CD80",
              "CD86"
            ]
          },
          {
            "name": "cf_attribution_causal_or_baseline",
            "role": "hard_fail",
            "field": "$.difference_attributable_to",
            "kind": "avoid",
            "expected": [
              "apc_engagement",
              "tumor_cell_engagement",
              "clonotype_intrinsic_baseline"
            ]
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "engagement_induces_exhausted_state",
              "pre_exhausted_cells_select_into_apc_conjugates"
            ]
          },
          {
            "name": "cf_causal_fabrication",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "resolved_by_pseudotemporal_ordering",
              "resolved_by_perturbation_experiment"
            ]
          },
          {
            "name": "cf_functional_overclaim",
            "role": "hard_fail",
            "field": "$.functional_interpretation",
            "kind": "avoid",
            "expected": [
              "costimulation_licensed_reinvigoration_competent",
              "costimulation_licensed_checkpoint_responsive",
              "tumor_driven_terminal_exhaustion"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-kimi-k2p6-pi",
      "modelName": "kimi-k2p6",
      "harness": "pi",
      "task": "10x_5p_melanoma_tcr_matched_apc_vs_tumor_exhaustion",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 1472,
      "cost": 0.62257432,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "state_discriminating_genes",
            "kind": "list",
            "matchKey": "gene",
            "k": 3,
            "expected": [
              {
                "key": "CTLA4",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TOX",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "ENTPD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "HAVCR2",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TIGIT",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "PDCD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "difference_attributable_to",
            "role": "gate",
            "field": "difference_attributable_to",
            "kind": "value",
            "expected": "covaries_with_apc_conjugate_state"
          },
          {
            "name": "causal_status",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "direction_unresolved_selection_confound"
          },
          {
            "name": "functional_interpretation",
            "role": "gate",
            "field": "functional_interpretation",
            "kind": "value",
            "expected": "costimulation_signature_function_unresolved"
          },
          {
            "name": "cf_contamination_gene_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "MLANA",
              "PMEL",
              "DCT",
              "TYR",
              "TYRP1",
              "SOX10",
              "MITF",
              "MAGEA4"
            ]
          },
          {
            "name": "cf_apc_partner_contamination_gene",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "HLA-DRA",
              "CD68",
              "LYZ",
              "CD14",
              "C1QA",
              "ITGAX",
              "CD1C"
            ]
          },
          {
            "name": "cf_ligand_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "CD274",
              "PDCD1LG2",
              "CD80",
              "CD86"
            ]
          },
          {
            "name": "cf_attribution_causal_or_baseline",
            "role": "hard_fail",
            "field": "$.difference_attributable_to",
            "kind": "avoid",
            "expected": [
              "apc_engagement",
              "tumor_cell_engagement",
              "clonotype_intrinsic_baseline"
            ]
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "engagement_induces_exhausted_state",
              "pre_exhausted_cells_select_into_apc_conjugates"
            ]
          },
          {
            "name": "cf_causal_fabrication",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "resolved_by_pseudotemporal_ordering",
              "resolved_by_perturbation_experiment"
            ]
          },
          {
            "name": "cf_functional_overclaim",
            "role": "hard_fail",
            "field": "$.functional_interpretation",
            "kind": "avoid",
            "expected": [
              "costimulation_licensed_reinvigoration_competent",
              "costimulation_licensed_checkpoint_responsive",
              "tumor_driven_terminal_exhaustion"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-kimi-k2p6-pi",
      "modelName": "kimi-k2p6",
      "harness": "pi",
      "task": "10x_5p_melanoma_tcr_matched_apc_vs_tumor_exhaustion",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 1319,
      "cost": 0.7224584099999999,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "state_discriminating_genes",
            "kind": "list",
            "matchKey": "gene",
            "k": 3,
            "expected": [
              {
                "key": "CTLA4",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TOX",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "ENTPD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "HAVCR2",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TIGIT",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "PDCD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "difference_attributable_to",
            "role": "gate",
            "field": "difference_attributable_to",
            "kind": "value",
            "expected": "covaries_with_apc_conjugate_state"
          },
          {
            "name": "causal_status",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "direction_unresolved_selection_confound"
          },
          {
            "name": "functional_interpretation",
            "role": "gate",
            "field": "functional_interpretation",
            "kind": "value",
            "expected": "costimulation_signature_function_unresolved"
          },
          {
            "name": "cf_contamination_gene_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "MLANA",
              "PMEL",
              "DCT",
              "TYR",
              "TYRP1",
              "SOX10",
              "MITF",
              "MAGEA4"
            ]
          },
          {
            "name": "cf_apc_partner_contamination_gene",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "HLA-DRA",
              "CD68",
              "LYZ",
              "CD14",
              "C1QA",
              "ITGAX",
              "CD1C"
            ]
          },
          {
            "name": "cf_ligand_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "CD274",
              "PDCD1LG2",
              "CD80",
              "CD86"
            ]
          },
          {
            "name": "cf_attribution_causal_or_baseline",
            "role": "hard_fail",
            "field": "$.difference_attributable_to",
            "kind": "avoid",
            "expected": [
              "apc_engagement",
              "tumor_cell_engagement",
              "clonotype_intrinsic_baseline"
            ]
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "engagement_induces_exhausted_state",
              "pre_exhausted_cells_select_into_apc_conjugates"
            ]
          },
          {
            "name": "cf_causal_fabrication",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "resolved_by_pseudotemporal_ordering",
              "resolved_by_perturbation_experiment"
            ]
          },
          {
            "name": "cf_functional_overclaim",
            "role": "hard_fail",
            "field": "$.functional_interpretation",
            "kind": "avoid",
            "expected": [
              "costimulation_licensed_reinvigoration_competent",
              "costimulation_licensed_checkpoint_responsive",
              "tumor_driven_terminal_exhaustion"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-kimi-k2p6-pi",
      "modelName": "kimi-k2p6",
      "harness": "pi",
      "task": "10x_5p_melanoma_tcr_matched_apc_vs_tumor_exhaustion",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1504,
      "cost": 0.528839,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "state_discriminating_genes",
            "kind": "list",
            "matchKey": "gene",
            "k": 3,
            "expected": [
              {
                "key": "CTLA4",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TOX",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "ENTPD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "HAVCR2",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TIGIT",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "PDCD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "difference_attributable_to",
            "role": "gate",
            "field": "difference_attributable_to",
            "kind": "value",
            "expected": "covaries_with_apc_conjugate_state"
          },
          {
            "name": "causal_status",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "direction_unresolved_selection_confound"
          },
          {
            "name": "functional_interpretation",
            "role": "gate",
            "field": "functional_interpretation",
            "kind": "value",
            "expected": "costimulation_signature_function_unresolved"
          },
          {
            "name": "cf_contamination_gene_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "MLANA",
              "PMEL",
              "DCT",
              "TYR",
              "TYRP1",
              "SOX10",
              "MITF",
              "MAGEA4"
            ]
          },
          {
            "name": "cf_apc_partner_contamination_gene",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "HLA-DRA",
              "CD68",
              "LYZ",
              "CD14",
              "C1QA",
              "ITGAX",
              "CD1C"
            ]
          },
          {
            "name": "cf_ligand_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "CD274",
              "PDCD1LG2",
              "CD80",
              "CD86"
            ]
          },
          {
            "name": "cf_attribution_causal_or_baseline",
            "role": "hard_fail",
            "field": "$.difference_attributable_to",
            "kind": "avoid",
            "expected": [
              "apc_engagement",
              "tumor_cell_engagement",
              "clonotype_intrinsic_baseline"
            ]
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "engagement_induces_exhausted_state",
              "pre_exhausted_cells_select_into_apc_conjugates"
            ]
          },
          {
            "name": "cf_causal_fabrication",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "resolved_by_pseudotemporal_ordering",
              "resolved_by_perturbation_experiment"
            ]
          },
          {
            "name": "cf_functional_overclaim",
            "role": "hard_fail",
            "field": "$.functional_interpretation",
            "kind": "avoid",
            "expected": [
              "costimulation_licensed_reinvigoration_competent",
              "costimulation_licensed_checkpoint_responsive",
              "tumor_driven_terminal_exhaustion"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-6-claude-code",
      "modelName": "claude-opus-4-6",
      "harness": "claude-code",
      "task": "green_2_tme_context_dependence",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 1,
      "passed": false,
      "score": 8,
      "durationS": 2529.61,
      "cost": 4.088736,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "program_map",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "sig_includes_itgae",
            "role": "gate",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Itgae"
            ]
          },
          {
            "name": "declared_a_known_confounder",
            "role": "gate",
            "field": "$.excluded_confounders[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "cf_signature_gut_contaminant",
            "role": "hard_fail",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "generic_signature_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd103_program_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "<"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "per_model_regulatory_state",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "poised_unfired"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "late_activation"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "closing_committed"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_tuples",
            "kind": "list",
            "matchKey": "regulator",
            "k": 1,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "role": {
                    "op": "equals",
                    "values": [
                      "repressor"
                    ]
                  },
                  "gating_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  },
                  "evidence_basis": {
                    "op": "equals",
                    "values": [
                      "target_program_behavior"
                    ]
                  },
                  "expr_vs_activity": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "locus_differential_gate",
            "role": "gate",
            "field": "locus_differential",
            "kind": "value",
            "expected": "uniform_across_models"
          },
          {
            "name": "formation_limited_by_gate",
            "role": "gate",
            "field": "formation_limited_by",
            "kind": "value",
            "expected": "absent_induction"
          },
          {
            "name": "locus_competent_in_naive_gate",
            "role": "gate",
            "field": "locus_competent_in_naive",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "ready_silent_fraction_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 3,
            "expected": [
              {
                "key": "B16_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_KPC_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "activator_chromatin_footprint_gate",
            "role": "gate",
            "field": "activator_chromatin_footprint",
            "kind": "value",
            "expected": "inconsistent_across_formers"
          },
          {
            "name": "exhaustion_comparable_gate",
            "role": "gate",
            "field": "exhaustion_comparable_across_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_nondeterminant_gate",
            "role": "gate",
            "field": "exhaustion_explains_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_separable_program_gate",
            "role": "gate",
            "field": "exhaustion_separable_program",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "exhaustion_counterexample_gate",
            "role": "gate",
            "field": "exhaustion_counterexample_model",
            "kind": "value",
            "expected": "KPC_GP"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "exhaustion_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "cf_inconsistent_present_without_activator",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "program",
            "entity": "present",
            "valueField": "activator_state",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_no_block_but_absent",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "none",
            "valueField": "program",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_late_block_without_poised_locus",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "late_activation",
            "valueField": "locus_state",
            "expected": [
              "closing_committed",
              "active"
            ]
          },
          {
            "name": "cf_inconsistent_early_block_with_brake_released",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "early_release",
            "valueField": "brake_state",
            "expected": [
              "released"
            ]
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: FAIL\n  x list_match: list_match (answer_field='program_map'): FAIL  (score=0.0)\n  + sig_includes_itgae: passed=True, score=1.0\n  x declared_a_known_confounder: passed=False, score=0.0\n  x list_match: list_match (answer_field='generic_signature_relations'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='cd103_program_relations'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='per_model_regulatory_state'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='regulator_tuples'): FAIL  (score=0.0)\n  + locus_differential_gate: passed=True, score=1.0\n  + formation_limited_by_gate: passed=True, score=1.0\n  + locus_competent_in_naive_gate: passed=True, score=1.0\n  x list_match: list_match (answer_field='ready_silent_fraction_relations'): FAIL  (score=0.0)\n  x activator_chromatin_footprint_gate: passed=False, score=0.0\n  + exhaustion_comparable_gate: passed=True, score=1.0\n  + exhaustion_nondeterminant_gate: passed=True, score=1.0\n  + exhaustion_separable_program_gate: passed=True, score=1.0\n  + exhaustion_counterexample_gate: passed=True, score=1.0\n  x list_match: list_match (answer_field='exhaustion_relations'): FAIL  (score=0.0)\n  x hard_fail 'cf_signature_gut_contaminant': TRIGGERED\n  + hard_fail 'cf_inconsistent_present_without_activator': not triggered\n  + hard_fail 'cf_inconsistent_no_block_but_absent': not triggered\n  + hard_fail 'cf_inconsistent_late_block_without_poised_locus': not triggered\n  + hard_fail 'cf_inconsistent_early_block_with_brake_released': not triggered",
      "metrics": {
        "pass_rule": "all",
        "scoring_count": 17,
        "scoring_passed": 8,
        "scoring_total_score": 8,
        "hard_fail_triggered": [
          "cf_signature_gut_contaminant"
        ]
      },
      "fieldScores": {
        "list_match": 0,
        "sig_includes_itgae": 1,
        "declared_a_known_confounder": 0,
        "locus_differential_gate": 1,
        "formation_limited_by_gate": 1,
        "locus_competent_in_naive_gate": 1,
        "activator_chromatin_footprint_gate": 0,
        "exhaustion_comparable_gate": 1,
        "exhaustion_nondeterminant_gate": 1,
        "exhaustion_separable_program_gate": 1,
        "exhaustion_counterexample_gate": 1
      },
      "graderChecks": [
        {
          "name": "program_map",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "sig_includes_itgae",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "declared_a_known_confounder",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "generic_signature_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "cd103_program_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "per_model_regulatory_state",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "regulator_tuples",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "locus_differential_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "formation_limited_by_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "locus_competent_in_naive_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "ready_silent_fraction_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "activator_chromatin_footprint_gate",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "exhaustion_comparable_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "exhaustion_nondeterminant_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "exhaustion_separable_program_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "exhaustion_counterexample_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "exhaustion_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "cf_signature_gut_contaminant",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "cf_inconsistent_present_without_activator",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_no_block_but_absent",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_late_block_without_poised_locus",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_early_block_with_brake_released",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "activator_chromatin_footprint": "consistent_in_formers",
        "cd103_program_relations": [
          {
            "pair": "B16_GP_vs_KPC_GP",
            "relation": "<"
          },
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": "~"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": "<"
          },
          {
            "pair": "KPC_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "B_ALL_GP_vs_mWnt_GP",
            "relation": "<"
          }
        ],
        "core_residency_signature": [
          "Itgae",
          "Itga1",
          "P2rx7",
          "Cxcr6",
          "Cd69"
        ],
        "excluded_confounders": [
          "Prdm1",
          "Bhlhe40",
          "Nr4a2"
        ],
        "exhaustion_comparable_across_split": "no",
        "exhaustion_counterexample_model": "KPC_GP",
        "exhaustion_explains_split": "no",
        "exhaustion_relations": [
          {
            "pair": "B16_GP_vs_KPC_GP",
            "relation": "~"
          },
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": "<"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_B_ALL_GP",
            "relation": "<"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "B_ALL_GP_vs_mWnt_GP",
            "relation": ">"
          }
        ],
        "exhaustion_separable_program": "yes",
        "formation_limited_by": "absent_induction",
        "generic_signature_relations": [
          {
            "pair": "B16_GP_vs_KPC_GP",
            "relation": "<"
          },
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": "<"
          },
          {
            "pair": "KPC_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": "~"
          },
          {
            "pair": "B_ALL_GP_vs_mWnt_GP",
            "relation": "<"
          }
        ],
        "locus_competent_in_naive": "yes",
        "locus_differential": "uniform_across_models",
        "per_model_regulatory_state": [
          {
            "activator_state": "partial",
            "blocking_step": "late_activation",
            "brake_state": "released",
            "locus_state": "poised_unfired",
            "model": "B16_GP",
            "program": "absent"
          },
          {
            "activator_state": "present",
            "blocking_step": "none",
            "brake_state": "released",
            "locus_state": "active",
            "model": "KPC_GP",
            "program": "present"
          },
          {
            "activator_state": "present",
            "blocking_step": "both_steps",
            "brake_state": "engaged",
            "locus_state": "poised_unfired",
            "model": "B_ALL_GP",
            "program": "absent"
          },
          {
            "activator_state": "present",
            "blocking_step": "early_release",
            "brake_state": "partial",
            "locus_state": "active",
            "model": "mWnt_GP",
            "program": "partial"
          }
        ],
        "program_map": [
          {
            "call": "absent",
            "model": "B16_GP"
          },
          {
            "call": "present",
            "model": "KPC_GP"
          },
          {
            "call": "absent",
            "model": "B_ALL_GP"
          },
          {
            "call": "partial",
            "model": "mWnt_GP"
          }
        ],
        "ready_silent_fraction_relations": [
          {
            "pair": "B16_GP_vs_KPC_GP",
            "relation": ">"
          },
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_B_ALL_GP",
            "relation": "<"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": "<"
          },
          {
            "pair": "B_ALL_GP_vs_mWnt_GP",
            "relation": "~"
          }
        ],
        "regulator_tuples": [
          {
            "evidence_basis": "target_program_behavior",
            "expr_vs_activity": "discordant",
            "gating_step": "late_activation",
            "regulator": "Runx3",
            "role": "activator"
          },
          {
            "evidence_basis": "own_abundance",
            "expr_vs_activity": "concordant",
            "gating_step": "early_release",
            "regulator": "Klf2",
            "role": "repressor"
          }
        ]
      },
      "nTurns": 41,
      "checkpointSignals": {
        "clustering": false,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": false,
        "nerveJointAnchor": false,
        "multiomeAssembly": true,
        "depthNormalizedAtac": false,
        "regulonInference": true,
        "cd8SubsetIntegration": true,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": false,
        "orthologProjection": false,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": false,
        "mousePseudobulkDesign": false,
        "cancerEpithelialDenominator": false,
        "geneLevelDe": false,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "trajectoryFile": "scbench-long/trajectories/green_2_tme_context_dependence__claude-opus-4-6__claude-code__r1.json",
      "runId": "claude-opus-4-6__claude-code__green_2_tme_context_dependence__r1",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-opus-4-6-claude-code",
      "modelName": "claude-opus-4-6",
      "harness": "claude-code",
      "task": "green_2_tme_context_dependence",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 2,
      "passed": false,
      "score": 7,
      "durationS": 2546.42,
      "cost": 3.4682304999999998,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "program_map",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "sig_includes_itgae",
            "role": "gate",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Itgae"
            ]
          },
          {
            "name": "declared_a_known_confounder",
            "role": "gate",
            "field": "$.excluded_confounders[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "cf_signature_gut_contaminant",
            "role": "hard_fail",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "generic_signature_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd103_program_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "<"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "per_model_regulatory_state",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "poised_unfired"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "late_activation"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "closing_committed"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_tuples",
            "kind": "list",
            "matchKey": "regulator",
            "k": 1,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "role": {
                    "op": "equals",
                    "values": [
                      "repressor"
                    ]
                  },
                  "gating_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  },
                  "evidence_basis": {
                    "op": "equals",
                    "values": [
                      "target_program_behavior"
                    ]
                  },
                  "expr_vs_activity": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "locus_differential_gate",
            "role": "gate",
            "field": "locus_differential",
            "kind": "value",
            "expected": "uniform_across_models"
          },
          {
            "name": "formation_limited_by_gate",
            "role": "gate",
            "field": "formation_limited_by",
            "kind": "value",
            "expected": "absent_induction"
          },
          {
            "name": "locus_competent_in_naive_gate",
            "role": "gate",
            "field": "locus_competent_in_naive",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "ready_silent_fraction_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 3,
            "expected": [
              {
                "key": "B16_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_KPC_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "activator_chromatin_footprint_gate",
            "role": "gate",
            "field": "activator_chromatin_footprint",
            "kind": "value",
            "expected": "inconsistent_across_formers"
          },
          {
            "name": "exhaustion_comparable_gate",
            "role": "gate",
            "field": "exhaustion_comparable_across_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_nondeterminant_gate",
            "role": "gate",
            "field": "exhaustion_explains_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_separable_program_gate",
            "role": "gate",
            "field": "exhaustion_separable_program",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "exhaustion_counterexample_gate",
            "role": "gate",
            "field": "exhaustion_counterexample_model",
            "kind": "value",
            "expected": "KPC_GP"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "exhaustion_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "cf_inconsistent_present_without_activator",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "program",
            "entity": "present",
            "valueField": "activator_state",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_no_block_but_absent",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "none",
            "valueField": "program",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_late_block_without_poised_locus",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "late_activation",
            "valueField": "locus_state",
            "expected": [
              "closing_committed",
              "active"
            ]
          },
          {
            "name": "cf_inconsistent_early_block_with_brake_released",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "early_release",
            "valueField": "brake_state",
            "expected": [
              "released"
            ]
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: FAIL\n  x list_match: list_match (answer_field='program_map'): FAIL  (score=0.0)\n  + sig_includes_itgae: passed=True, score=1.0\n  x declared_a_known_confounder: passed=False, score=0.0\n  x list_match: list_match (answer_field='generic_signature_relations'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='cd103_program_relations'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='per_model_regulatory_state'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='regulator_tuples'): FAIL  (score=0.0)\n  + locus_differential_gate: passed=True, score=1.0\n  + formation_limited_by_gate: passed=True, score=1.0\n  + locus_competent_in_naive_gate: passed=True, score=1.0\n  x list_match: list_match (answer_field='ready_silent_fraction_relations'): FAIL  (score=0.0)\n  x activator_chromatin_footprint_gate: passed=False, score=0.0\n  x exhaustion_comparable_gate: passed=False, score=0.0\n  + exhaustion_nondeterminant_gate: passed=True, score=1.0\n  + exhaustion_separable_program_gate: passed=True, score=1.0\n  + exhaustion_counterexample_gate: passed=True, score=1.0\n  x list_match: list_match (answer_field='exhaustion_relations'): FAIL  (score=0.0)\n  x hard_fail 'cf_signature_gut_contaminant': TRIGGERED\n  + hard_fail 'cf_inconsistent_present_without_activator': not triggered\n  + hard_fail 'cf_inconsistent_no_block_but_absent': not triggered\n  x hard_fail 'cf_inconsistent_late_block_without_poised_locus': TRIGGERED\n  + hard_fail 'cf_inconsistent_early_block_with_brake_released': not triggered",
      "metrics": {
        "pass_rule": "all",
        "scoring_count": 17,
        "scoring_passed": 7,
        "scoring_total_score": 7,
        "hard_fail_triggered": [
          "cf_signature_gut_contaminant",
          "cf_inconsistent_late_block_without_poised_locus"
        ]
      },
      "fieldScores": {
        "list_match": 0,
        "sig_includes_itgae": 1,
        "declared_a_known_confounder": 0,
        "locus_differential_gate": 1,
        "formation_limited_by_gate": 1,
        "locus_competent_in_naive_gate": 1,
        "activator_chromatin_footprint_gate": 0,
        "exhaustion_comparable_gate": 0,
        "exhaustion_nondeterminant_gate": 1,
        "exhaustion_separable_program_gate": 1,
        "exhaustion_counterexample_gate": 1
      },
      "graderChecks": [
        {
          "name": "program_map",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "sig_includes_itgae",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "declared_a_known_confounder",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "generic_signature_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "cd103_program_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "per_model_regulatory_state",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "regulator_tuples",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "locus_differential_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "formation_limited_by_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "locus_competent_in_naive_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "ready_silent_fraction_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "activator_chromatin_footprint_gate",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "exhaustion_comparable_gate",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "exhaustion_nondeterminant_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "exhaustion_separable_program_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "exhaustion_counterexample_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "exhaustion_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "cf_signature_gut_contaminant",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "cf_inconsistent_present_without_activator",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_no_block_but_absent",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_late_block_without_poised_locus",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "cf_inconsistent_early_block_with_brake_released",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "activator_chromatin_footprint": "consistent_in_formers",
        "cd103_program_relations": [
          {
            "pair": "B16_GP_vs_KPC_GP",
            "relation": "<"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": "<"
          },
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "mWnt_GP_vs_B_ALL_GP",
            "relation": ">"
          }
        ],
        "core_residency_signature": [
          "Itgae",
          "Itga1",
          "Cd69",
          "P2rx7",
          "Cxcr6",
          "Xcl1",
          "Nr4a1",
          "Ahr"
        ],
        "excluded_confounders": [
          "Eomes",
          "Klrg1",
          "Cx3cr1",
          "S1pr1",
          "Klf2"
        ],
        "exhaustion_comparable_across_split": "yes",
        "exhaustion_counterexample_model": "KPC_GP",
        "exhaustion_explains_split": "no",
        "exhaustion_relations": [
          {
            "pair": "B16_GP_vs_KPC_GP",
            "relation": "<"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": "~"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "mWnt_GP_vs_B_ALL_GP",
            "relation": "<"
          }
        ],
        "exhaustion_separable_program": "yes",
        "formation_limited_by": "absent_induction",
        "generic_signature_relations": [
          {
            "pair": "B16_GP_vs_KPC_GP",
            "relation": "~"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_B_ALL_GP",
            "relation": "~"
          },
          {
            "pair": "mWnt_GP_vs_B_ALL_GP",
            "relation": "<"
          }
        ],
        "locus_competent_in_naive": "yes",
        "locus_differential": "uniform_across_models",
        "per_model_regulatory_state": [
          {
            "activator_state": "present",
            "blocking_step": "none",
            "brake_state": "released",
            "locus_state": "active",
            "model": "KPC_GP",
            "program": "present"
          },
          {
            "activator_state": "present",
            "blocking_step": "late_activation",
            "brake_state": "released",
            "locus_state": "active",
            "model": "mWnt_GP",
            "program": "partial"
          },
          {
            "activator_state": "partial",
            "blocking_step": "late_activation",
            "brake_state": "released",
            "locus_state": "poised_unfired",
            "model": "B16_GP",
            "program": "absent"
          },
          {
            "activator_state": "present",
            "blocking_step": "early_release",
            "brake_state": "engaged",
            "locus_state": "poised_unfired",
            "model": "B_ALL_GP",
            "program": "absent"
          }
        ],
        "program_map": [
          {
            "call": "absent",
            "model": "B16_GP"
          },
          {
            "call": "present",
            "model": "KPC_GP"
          },
          {
            "call": "partial",
            "model": "mWnt_GP"
          },
          {
            "call": "absent",
            "model": "B_ALL_GP"
          }
        ],
        "ready_silent_fraction_relations": [
          {
            "pair": "B16_GP_vs_KPC_GP",
            "relation": ">"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": "~"
          },
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": "<"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": "<"
          },
          {
            "pair": "KPC_GP_vs_B_ALL_GP",
            "relation": "<"
          },
          {
            "pair": "mWnt_GP_vs_B_ALL_GP",
            "relation": "<"
          }
        ],
        "regulator_tuples": [
          {
            "evidence_basis": "target_program_behavior",
            "expr_vs_activity": "discordant",
            "gating_step": "late_activation",
            "regulator": "Runx3",
            "role": "activator"
          },
          {
            "evidence_basis": "own_abundance",
            "expr_vs_activity": "concordant",
            "gating_step": "early_release",
            "regulator": "Eomes",
            "role": "repressor"
          },
          {
            "evidence_basis": "own_abundance",
            "expr_vs_activity": "concordant",
            "gating_step": "late_activation",
            "regulator": "Smad3",
            "role": "activator"
          }
        ]
      },
      "nTurns": 26,
      "checkpointSignals": {
        "clustering": false,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": false,
        "nerveJointAnchor": false,
        "multiomeAssembly": true,
        "depthNormalizedAtac": false,
        "regulonInference": true,
        "cd8SubsetIntegration": true,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": false,
        "orthologProjection": false,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": false,
        "mousePseudobulkDesign": false,
        "cancerEpithelialDenominator": false,
        "geneLevelDe": false,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "trajectoryFile": "scbench-long/trajectories/green_2_tme_context_dependence__claude-opus-4-6__claude-code__r2.json",
      "runId": "claude-opus-4-6__claude-code__green_2_tme_context_dependence__r2",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-opus-4-6-claude-code",
      "modelName": "claude-opus-4-6",
      "harness": "claude-code",
      "task": "green_2_tme_context_dependence",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 3,
      "passed": false,
      "score": 7,
      "durationS": 2641.41,
      "cost": 2.8787872500000002,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "program_map",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "sig_includes_itgae",
            "role": "gate",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Itgae"
            ]
          },
          {
            "name": "declared_a_known_confounder",
            "role": "gate",
            "field": "$.excluded_confounders[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "cf_signature_gut_contaminant",
            "role": "hard_fail",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "generic_signature_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd103_program_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "<"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "per_model_regulatory_state",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "poised_unfired"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "late_activation"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "closing_committed"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_tuples",
            "kind": "list",
            "matchKey": "regulator",
            "k": 1,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "role": {
                    "op": "equals",
                    "values": [
                      "repressor"
                    ]
                  },
                  "gating_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  },
                  "evidence_basis": {
                    "op": "equals",
                    "values": [
                      "target_program_behavior"
                    ]
                  },
                  "expr_vs_activity": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "locus_differential_gate",
            "role": "gate",
            "field": "locus_differential",
            "kind": "value",
            "expected": "uniform_across_models"
          },
          {
            "name": "formation_limited_by_gate",
            "role": "gate",
            "field": "formation_limited_by",
            "kind": "value",
            "expected": "absent_induction"
          },
          {
            "name": "locus_competent_in_naive_gate",
            "role": "gate",
            "field": "locus_competent_in_naive",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "ready_silent_fraction_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 3,
            "expected": [
              {
                "key": "B16_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_KPC_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "activator_chromatin_footprint_gate",
            "role": "gate",
            "field": "activator_chromatin_footprint",
            "kind": "value",
            "expected": "inconsistent_across_formers"
          },
          {
            "name": "exhaustion_comparable_gate",
            "role": "gate",
            "field": "exhaustion_comparable_across_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_nondeterminant_gate",
            "role": "gate",
            "field": "exhaustion_explains_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_separable_program_gate",
            "role": "gate",
            "field": "exhaustion_separable_program",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "exhaustion_counterexample_gate",
            "role": "gate",
            "field": "exhaustion_counterexample_model",
            "kind": "value",
            "expected": "KPC_GP"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "exhaustion_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "cf_inconsistent_present_without_activator",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "program",
            "entity": "present",
            "valueField": "activator_state",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_no_block_but_absent",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "none",
            "valueField": "program",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_late_block_without_poised_locus",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "late_activation",
            "valueField": "locus_state",
            "expected": [
              "closing_committed",
              "active"
            ]
          },
          {
            "name": "cf_inconsistent_early_block_with_brake_released",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "early_release",
            "valueField": "brake_state",
            "expected": [
              "released"
            ]
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: FAIL\n  x list_match: list_match (answer_field='program_map'): FAIL  (score=0.0)\n  + sig_includes_itgae: passed=True, score=1.0\n  x declared_a_known_confounder: passed=False, score=0.0\n  x list_match: list_match (answer_field='generic_signature_relations'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='cd103_program_relations'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='per_model_regulatory_state'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='regulator_tuples'): FAIL  (score=0.0)\n  x locus_differential_gate: passed=False, score=0.0\n  + formation_limited_by_gate: passed=True, score=1.0\n  + locus_competent_in_naive_gate: passed=True, score=1.0\n  x list_match: list_match (answer_field='ready_silent_fraction_relations'): FAIL  (score=0.0)\n  x activator_chromatin_footprint_gate: passed=False, score=0.0\n  + exhaustion_comparable_gate: passed=True, score=1.0\n  + exhaustion_nondeterminant_gate: passed=True, score=1.0\n  + exhaustion_separable_program_gate: passed=True, score=1.0\n  + exhaustion_counterexample_gate: passed=True, score=1.0\n  x list_match: list_match (answer_field='exhaustion_relations'): FAIL  (score=0.0)\n  + hard_fail 'cf_signature_gut_contaminant': not triggered\n  + hard_fail 'cf_inconsistent_present_without_activator': not triggered\n  + hard_fail 'cf_inconsistent_no_block_but_absent': not triggered\n  + hard_fail 'cf_inconsistent_late_block_without_poised_locus': not triggered\n  + hard_fail 'cf_inconsistent_early_block_with_brake_released': not triggered",
      "metrics": {
        "pass_rule": "all",
        "scoring_count": 17,
        "scoring_passed": 7,
        "scoring_total_score": 7,
        "hard_fail_triggered": []
      },
      "fieldScores": {
        "list_match": 0,
        "sig_includes_itgae": 1,
        "declared_a_known_confounder": 0,
        "locus_differential_gate": 0,
        "formation_limited_by_gate": 1,
        "locus_competent_in_naive_gate": 1,
        "activator_chromatin_footprint_gate": 0,
        "exhaustion_comparable_gate": 1,
        "exhaustion_nondeterminant_gate": 1,
        "exhaustion_separable_program_gate": 1,
        "exhaustion_counterexample_gate": 1
      },
      "graderChecks": [
        {
          "name": "program_map",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "sig_includes_itgae",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "declared_a_known_confounder",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "generic_signature_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "cd103_program_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "per_model_regulatory_state",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "regulator_tuples",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "locus_differential_gate",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "formation_limited_by_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "locus_competent_in_naive_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "ready_silent_fraction_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "activator_chromatin_footprint_gate",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "exhaustion_comparable_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "exhaustion_nondeterminant_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "exhaustion_separable_program_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "exhaustion_counterexample_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "exhaustion_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "cf_signature_gut_contaminant",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_present_without_activator",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_no_block_but_absent",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_late_block_without_poised_locus",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_early_block_with_brake_released",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "activator_chromatin_footprint": "consistent_in_formers",
        "cd103_program_relations": [
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": "~"
          },
          {
            "pair": "B16_GP_vs_KPC_GP",
            "relation": "<"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": "<"
          },
          {
            "pair": "B_ALL_GP_vs_KPC_GP",
            "relation": "<"
          },
          {
            "pair": "B_ALL_GP_vs_mWnt_GP",
            "relation": "<"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": ">"
          }
        ],
        "core_residency_signature": [
          "Itgae",
          "Cd69",
          "Cxcr6",
          "Runx3",
          "Il7r"
        ],
        "excluded_confounders": [
          "Prdm1",
          "Tox",
          "Entpd1"
        ],
        "exhaustion_comparable_across_split": "no",
        "exhaustion_counterexample_model": "KPC_GP",
        "exhaustion_explains_split": "no",
        "exhaustion_relations": [
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": "<"
          },
          {
            "pair": "B16_GP_vs_KPC_GP",
            "relation": "~"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "B_ALL_GP_vs_KPC_GP",
            "relation": ">"
          },
          {
            "pair": "B_ALL_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": ">"
          }
        ],
        "exhaustion_separable_program": "yes",
        "formation_limited_by": "absent_induction",
        "generic_signature_relations": [
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": "~"
          },
          {
            "pair": "B16_GP_vs_KPC_GP",
            "relation": ">"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "B_ALL_GP_vs_KPC_GP",
            "relation": ">"
          },
          {
            "pair": "B_ALL_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": ">"
          }
        ],
        "locus_competent_in_naive": "yes",
        "locus_differential": "model_specific_element",
        "per_model_regulatory_state": [
          {
            "activator_state": "absent",
            "blocking_step": "late_activation",
            "brake_state": "released",
            "locus_state": "repressed",
            "model": "B16_GP",
            "program": "absent"
          },
          {
            "activator_state": "partial",
            "blocking_step": "both_steps",
            "brake_state": "engaged",
            "locus_state": "poised_unfired",
            "model": "B_ALL_GP",
            "program": "absent"
          },
          {
            "activator_state": "present",
            "blocking_step": "none",
            "brake_state": "released",
            "locus_state": "active",
            "model": "KPC_GP",
            "program": "present"
          },
          {
            "activator_state": "present",
            "blocking_step": "early_release",
            "brake_state": "partial",
            "locus_state": "active",
            "model": "mWnt_GP",
            "program": "partial"
          }
        ],
        "program_map": [
          {
            "call": "absent",
            "model": "B16_GP"
          },
          {
            "call": "absent",
            "model": "B_ALL_GP"
          },
          {
            "call": "present",
            "model": "KPC_GP"
          },
          {
            "call": "partial",
            "model": "mWnt_GP"
          }
        ],
        "ready_silent_fraction_relations": [
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "B16_GP_vs_KPC_GP",
            "relation": ">"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "B_ALL_GP_vs_KPC_GP",
            "relation": ">"
          },
          {
            "pair": "B_ALL_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": "<"
          }
        ],
        "regulator_tuples": [
          {
            "evidence_basis": "target_program_behavior",
            "expr_vs_activity": "discordant",
            "gating_step": "late_activation",
            "regulator": "Runx3",
            "role": "activator"
          },
          {
            "evidence_basis": "own_abundance",
            "expr_vs_activity": "concordant",
            "gating_step": "early_release",
            "regulator": "Klf2",
            "role": "repressor"
          },
          {
            "evidence_basis": "target_program_behavior",
            "expr_vs_activity": "discordant",
            "gating_step": "late_activation",
            "regulator": "Prdm1",
            "role": "dual"
          }
        ]
      },
      "nTurns": 25,
      "checkpointSignals": {
        "clustering": false,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": false,
        "nerveJointAnchor": false,
        "multiomeAssembly": true,
        "depthNormalizedAtac": true,
        "regulonInference": true,
        "cd8SubsetIntegration": true,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": false,
        "orthologProjection": false,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": false,
        "mousePseudobulkDesign": true,
        "cancerEpithelialDenominator": false,
        "geneLevelDe": true,
        "agingSignatureControl": true,
        "senescencePanelScoring": false
      },
      "trajectoryFile": "scbench-long/trajectories/green_2_tme_context_dependence__claude-opus-4-6__claude-code__r3.json",
      "runId": "claude-opus-4-6__claude-code__green_2_tme_context_dependence__r3",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-opus-4-6-pi",
      "modelName": "claude-opus-4-6",
      "harness": "pi",
      "task": "green_2_tme_context_dependence",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 1,
      "passed": false,
      "score": 6,
      "durationS": 2299.43,
      "cost": 2.9098037500000005,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "program_map",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "sig_includes_itgae",
            "role": "gate",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Itgae"
            ]
          },
          {
            "name": "declared_a_known_confounder",
            "role": "gate",
            "field": "$.excluded_confounders[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "cf_signature_gut_contaminant",
            "role": "hard_fail",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "generic_signature_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd103_program_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "<"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "per_model_regulatory_state",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "poised_unfired"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "late_activation"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "closing_committed"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_tuples",
            "kind": "list",
            "matchKey": "regulator",
            "k": 1,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "role": {
                    "op": "equals",
                    "values": [
                      "repressor"
                    ]
                  },
                  "gating_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  },
                  "evidence_basis": {
                    "op": "equals",
                    "values": [
                      "target_program_behavior"
                    ]
                  },
                  "expr_vs_activity": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "locus_differential_gate",
            "role": "gate",
            "field": "locus_differential",
            "kind": "value",
            "expected": "uniform_across_models"
          },
          {
            "name": "formation_limited_by_gate",
            "role": "gate",
            "field": "formation_limited_by",
            "kind": "value",
            "expected": "absent_induction"
          },
          {
            "name": "locus_competent_in_naive_gate",
            "role": "gate",
            "field": "locus_competent_in_naive",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "ready_silent_fraction_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 3,
            "expected": [
              {
                "key": "B16_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_KPC_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "activator_chromatin_footprint_gate",
            "role": "gate",
            "field": "activator_chromatin_footprint",
            "kind": "value",
            "expected": "inconsistent_across_formers"
          },
          {
            "name": "exhaustion_comparable_gate",
            "role": "gate",
            "field": "exhaustion_comparable_across_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_nondeterminant_gate",
            "role": "gate",
            "field": "exhaustion_explains_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_separable_program_gate",
            "role": "gate",
            "field": "exhaustion_separable_program",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "exhaustion_counterexample_gate",
            "role": "gate",
            "field": "exhaustion_counterexample_model",
            "kind": "value",
            "expected": "KPC_GP"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "exhaustion_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "cf_inconsistent_present_without_activator",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "program",
            "entity": "present",
            "valueField": "activator_state",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_no_block_but_absent",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "none",
            "valueField": "program",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_late_block_without_poised_locus",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "late_activation",
            "valueField": "locus_state",
            "expected": [
              "closing_committed",
              "active"
            ]
          },
          {
            "name": "cf_inconsistent_early_block_with_brake_released",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "early_release",
            "valueField": "brake_state",
            "expected": [
              "released"
            ]
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: FAIL\n  x list_match: list_match (answer_field='program_map'): FAIL  (score=0.0)\n  + sig_includes_itgae: passed=True, score=1.0\n  x declared_a_known_confounder: passed=False, score=0.0\n  x list_match: list_match (answer_field='generic_signature_relations'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='cd103_program_relations'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='per_model_regulatory_state'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='regulator_tuples'): FAIL  (score=0.0)\n  + locus_differential_gate: passed=True, score=1.0\n  + formation_limited_by_gate: passed=True, score=1.0\n  + locus_competent_in_naive_gate: passed=True, score=1.0\n  x list_match: list_match (answer_field='ready_silent_fraction_relations'): FAIL  (score=0.0)\n  x activator_chromatin_footprint_gate: passed=False, score=0.0\n  x exhaustion_comparable_gate: passed=False, score=0.0\n  + exhaustion_nondeterminant_gate: passed=True, score=1.0\n  + exhaustion_separable_program_gate: passed=True, score=1.0\n  x exhaustion_counterexample_gate: passed=False, score=0.0\n  x list_match: list_match (answer_field='exhaustion_relations'): FAIL  (score=0.0)\n  x hard_fail 'cf_signature_gut_contaminant': TRIGGERED\n  + hard_fail 'cf_inconsistent_present_without_activator': not triggered\n  + hard_fail 'cf_inconsistent_no_block_but_absent': not triggered\n  + hard_fail 'cf_inconsistent_late_block_without_poised_locus': not triggered\n  + hard_fail 'cf_inconsistent_early_block_with_brake_released': not triggered",
      "metrics": {
        "pass_rule": "all",
        "scoring_count": 17,
        "scoring_passed": 6,
        "scoring_total_score": 6,
        "hard_fail_triggered": [
          "cf_signature_gut_contaminant"
        ]
      },
      "fieldScores": {
        "list_match": 0,
        "sig_includes_itgae": 1,
        "declared_a_known_confounder": 0,
        "locus_differential_gate": 1,
        "formation_limited_by_gate": 1,
        "locus_competent_in_naive_gate": 1,
        "activator_chromatin_footprint_gate": 0,
        "exhaustion_comparable_gate": 0,
        "exhaustion_nondeterminant_gate": 1,
        "exhaustion_separable_program_gate": 1,
        "exhaustion_counterexample_gate": 0
      },
      "graderChecks": [
        {
          "name": "program_map",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "sig_includes_itgae",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "declared_a_known_confounder",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "generic_signature_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "cd103_program_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "per_model_regulatory_state",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "regulator_tuples",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "locus_differential_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "formation_limited_by_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "locus_competent_in_naive_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "ready_silent_fraction_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "activator_chromatin_footprint_gate",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "exhaustion_comparable_gate",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "exhaustion_nondeterminant_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "exhaustion_separable_program_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "exhaustion_counterexample_gate",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "exhaustion_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "cf_signature_gut_contaminant",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "cf_inconsistent_present_without_activator",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_no_block_but_absent",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_late_block_without_poised_locus",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_early_block_with_brake_released",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "activator_chromatin_footprint": "consistent_in_formers",
        "cd103_program_relations": [
          {
            "pair": "B16_GP_vs_KPC_GP",
            "relation": "<"
          },
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": "~"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": "<"
          },
          {
            "pair": "KPC_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "B_ALL_GP_vs_mWnt_GP",
            "relation": "<"
          }
        ],
        "core_residency_signature": [
          "Itgae",
          "Itga1",
          "Cd69"
        ],
        "excluded_confounders": [
          "Xcl1",
          "Cxcr6"
        ],
        "exhaustion_comparable_across_split": "yes",
        "exhaustion_counterexample_model": "mWnt_GP",
        "exhaustion_explains_split": "no",
        "exhaustion_relations": [
          {
            "pair": "B16_GP_vs_KPC_GP",
            "relation": "~"
          },
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": "<"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_B_ALL_GP",
            "relation": "<"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "B_ALL_GP_vs_mWnt_GP",
            "relation": ">"
          }
        ],
        "exhaustion_separable_program": "yes",
        "formation_limited_by": "absent_induction",
        "generic_signature_relations": [
          {
            "pair": "B16_GP_vs_KPC_GP",
            "relation": "<"
          },
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": "<"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_B_ALL_GP",
            "relation": "~"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "B_ALL_GP_vs_mWnt_GP",
            "relation": ">"
          }
        ],
        "locus_competent_in_naive": "yes",
        "locus_differential": "uniform_across_models",
        "per_model_regulatory_state": [
          {
            "activator_state": "absent",
            "blocking_step": "late_activation",
            "brake_state": "released",
            "locus_state": "poised_unfired",
            "model": "B16_GP",
            "program": "absent"
          },
          {
            "activator_state": "present",
            "blocking_step": "none",
            "brake_state": "released",
            "locus_state": "active",
            "model": "KPC_GP",
            "program": "present"
          },
          {
            "activator_state": "absent",
            "blocking_step": "both_steps",
            "brake_state": "partial",
            "locus_state": "poised_unfired",
            "model": "B_ALL_GP",
            "program": "absent"
          },
          {
            "activator_state": "present",
            "blocking_step": "early_release",
            "brake_state": "partial",
            "locus_state": "poised_unfired",
            "model": "mWnt_GP",
            "program": "partial"
          }
        ],
        "program_map": [
          {
            "call": "absent",
            "model": "B16_GP"
          },
          {
            "call": "present",
            "model": "KPC_GP"
          },
          {
            "call": "absent",
            "model": "B_ALL_GP"
          },
          {
            "call": "partial",
            "model": "mWnt_GP"
          }
        ],
        "ready_silent_fraction_relations": [
          {
            "pair": "B16_GP_vs_KPC_GP",
            "relation": ">"
          },
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_B_ALL_GP",
            "relation": "~"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": "~"
          },
          {
            "pair": "B_ALL_GP_vs_mWnt_GP",
            "relation": "~"
          }
        ],
        "regulator_tuples": [
          {
            "evidence_basis": "own_abundance",
            "expr_vs_activity": "discordant",
            "gating_step": "late_activation",
            "regulator": "Runx3",
            "role": "activator"
          },
          {
            "evidence_basis": "own_abundance",
            "expr_vs_activity": "concordant",
            "gating_step": "early_release",
            "regulator": "S1pr1",
            "role": "repressor"
          },
          {
            "evidence_basis": "own_abundance",
            "expr_vs_activity": "concordant",
            "gating_step": "early_release",
            "regulator": "Klf2",
            "role": "repressor"
          }
        ]
      },
      "nTurns": 31,
      "checkpointSignals": {
        "clustering": false,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": false,
        "nerveJointAnchor": false,
        "multiomeAssembly": true,
        "depthNormalizedAtac": true,
        "regulonInference": true,
        "cd8SubsetIntegration": true,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": true,
        "orthologProjection": false,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": false,
        "mousePseudobulkDesign": false,
        "cancerEpithelialDenominator": false,
        "geneLevelDe": true,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "trajectoryFile": "scbench-long/trajectories/green_2_tme_context_dependence__claude-opus-4-6__pi__r1.json",
      "runId": "claude-opus-4-6__pi__green_2_tme_context_dependence__r1",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-opus-4-6-pi",
      "modelName": "claude-opus-4-6",
      "harness": "pi",
      "task": "green_2_tme_context_dependence",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 2,
      "passed": false,
      "score": 7,
      "durationS": 4546.77,
      "cost": 2.995858,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "program_map",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "sig_includes_itgae",
            "role": "gate",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Itgae"
            ]
          },
          {
            "name": "declared_a_known_confounder",
            "role": "gate",
            "field": "$.excluded_confounders[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "cf_signature_gut_contaminant",
            "role": "hard_fail",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "generic_signature_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd103_program_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "<"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "per_model_regulatory_state",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "poised_unfired"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "late_activation"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "closing_committed"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_tuples",
            "kind": "list",
            "matchKey": "regulator",
            "k": 1,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "role": {
                    "op": "equals",
                    "values": [
                      "repressor"
                    ]
                  },
                  "gating_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  },
                  "evidence_basis": {
                    "op": "equals",
                    "values": [
                      "target_program_behavior"
                    ]
                  },
                  "expr_vs_activity": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "locus_differential_gate",
            "role": "gate",
            "field": "locus_differential",
            "kind": "value",
            "expected": "uniform_across_models"
          },
          {
            "name": "formation_limited_by_gate",
            "role": "gate",
            "field": "formation_limited_by",
            "kind": "value",
            "expected": "absent_induction"
          },
          {
            "name": "locus_competent_in_naive_gate",
            "role": "gate",
            "field": "locus_competent_in_naive",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "ready_silent_fraction_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 3,
            "expected": [
              {
                "key": "B16_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_KPC_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "activator_chromatin_footprint_gate",
            "role": "gate",
            "field": "activator_chromatin_footprint",
            "kind": "value",
            "expected": "inconsistent_across_formers"
          },
          {
            "name": "exhaustion_comparable_gate",
            "role": "gate",
            "field": "exhaustion_comparable_across_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_nondeterminant_gate",
            "role": "gate",
            "field": "exhaustion_explains_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_separable_program_gate",
            "role": "gate",
            "field": "exhaustion_separable_program",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "exhaustion_counterexample_gate",
            "role": "gate",
            "field": "exhaustion_counterexample_model",
            "kind": "value",
            "expected": "KPC_GP"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "exhaustion_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "cf_inconsistent_present_without_activator",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "program",
            "entity": "present",
            "valueField": "activator_state",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_no_block_but_absent",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "none",
            "valueField": "program",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_late_block_without_poised_locus",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "late_activation",
            "valueField": "locus_state",
            "expected": [
              "closing_committed",
              "active"
            ]
          },
          {
            "name": "cf_inconsistent_early_block_with_brake_released",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "early_release",
            "valueField": "brake_state",
            "expected": [
              "released"
            ]
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: FAIL\n  x list_match: list_match (answer_field='program_map'): FAIL  (score=0.0)\n  + sig_includes_itgae: passed=True, score=1.0\n  x declared_a_known_confounder: passed=False, score=0.0\n  x list_match: list_match (answer_field='generic_signature_relations'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='cd103_program_relations'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='per_model_regulatory_state'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='regulator_tuples'): FAIL  (score=0.0)\n  + locus_differential_gate: passed=True, score=1.0\n  + formation_limited_by_gate: passed=True, score=1.0\n  + locus_competent_in_naive_gate: passed=True, score=1.0\n  x list_match: list_match (answer_field='ready_silent_fraction_relations'): FAIL  (score=0.0)\n  x activator_chromatin_footprint_gate: passed=False, score=0.0\n  x exhaustion_comparable_gate: passed=False, score=0.0\n  + exhaustion_nondeterminant_gate: passed=True, score=1.0\n  + exhaustion_separable_program_gate: passed=True, score=1.0\n  + exhaustion_counterexample_gate: passed=True, score=1.0\n  x list_match: list_match (answer_field='exhaustion_relations'): FAIL  (score=0.0)\n  x hard_fail 'cf_signature_gut_contaminant': TRIGGERED\n  + hard_fail 'cf_inconsistent_present_without_activator': not triggered\n  + hard_fail 'cf_inconsistent_no_block_but_absent': not triggered\n  + hard_fail 'cf_inconsistent_late_block_without_poised_locus': not triggered\n  + hard_fail 'cf_inconsistent_early_block_with_brake_released': not triggered",
      "metrics": {
        "pass_rule": "all",
        "scoring_count": 17,
        "scoring_passed": 7,
        "scoring_total_score": 7,
        "hard_fail_triggered": [
          "cf_signature_gut_contaminant"
        ]
      },
      "fieldScores": {
        "list_match": 0,
        "sig_includes_itgae": 1,
        "declared_a_known_confounder": 0,
        "locus_differential_gate": 1,
        "formation_limited_by_gate": 1,
        "locus_competent_in_naive_gate": 1,
        "activator_chromatin_footprint_gate": 0,
        "exhaustion_comparable_gate": 0,
        "exhaustion_nondeterminant_gate": 1,
        "exhaustion_separable_program_gate": 1,
        "exhaustion_counterexample_gate": 1
      },
      "graderChecks": [
        {
          "name": "program_map",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "sig_includes_itgae",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "declared_a_known_confounder",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "generic_signature_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "cd103_program_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "per_model_regulatory_state",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "regulator_tuples",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "locus_differential_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "formation_limited_by_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "locus_competent_in_naive_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "ready_silent_fraction_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "activator_chromatin_footprint_gate",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "exhaustion_comparable_gate",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "exhaustion_nondeterminant_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "exhaustion_separable_program_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "exhaustion_counterexample_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "exhaustion_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "cf_signature_gut_contaminant",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "cf_inconsistent_present_without_activator",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_no_block_but_absent",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_late_block_without_poised_locus",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_early_block_with_brake_released",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "activator_chromatin_footprint": "consistent_in_formers",
        "cd103_program_relations": [
          {
            "pair": "B16_GP_vs_KPC_GP",
            "relation": "<"
          },
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": "<"
          },
          {
            "pair": "KPC_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "B_ALL_GP_vs_mWnt_GP",
            "relation": "<"
          }
        ],
        "core_residency_signature": [
          "Itgae",
          "Itga1",
          "Cd69",
          "Cxcr6",
          "P2rx7",
          "Ahr",
          "Cd101"
        ],
        "excluded_confounders": [
          "Tox",
          "Pdcd1",
          "Lag3",
          "Havcr2",
          "Tigit",
          "Ctla4",
          "Entpd1"
        ],
        "exhaustion_comparable_across_split": "yes",
        "exhaustion_counterexample_model": "KPC_GP",
        "exhaustion_explains_split": "no",
        "exhaustion_relations": [
          {
            "pair": "B16_GP_vs_KPC_GP",
            "relation": "~"
          },
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": "~"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "B_ALL_GP_vs_mWnt_GP",
            "relation": ">"
          }
        ],
        "exhaustion_separable_program": "yes",
        "formation_limited_by": "absent_induction",
        "generic_signature_relations": [
          {
            "pair": "B16_GP_vs_KPC_GP",
            "relation": "~"
          },
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": "~"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_B_ALL_GP",
            "relation": "~"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "B_ALL_GP_vs_mWnt_GP",
            "relation": ">"
          }
        ],
        "locus_competent_in_naive": "yes",
        "locus_differential": "uniform_across_models",
        "per_model_regulatory_state": [
          {
            "activator_state": "absent",
            "blocking_step": "late_activation",
            "brake_state": "released",
            "locus_state": "poised_unfired",
            "model": "B16_GP",
            "program": "absent"
          },
          {
            "activator_state": "present",
            "blocking_step": "none",
            "brake_state": "released",
            "locus_state": "active",
            "model": "KPC_GP",
            "program": "present"
          },
          {
            "activator_state": "absent",
            "blocking_step": "both_steps",
            "brake_state": "engaged",
            "locus_state": "poised_unfired",
            "model": "B_ALL_GP",
            "program": "absent"
          },
          {
            "activator_state": "present",
            "blocking_step": "none",
            "brake_state": "released",
            "locus_state": "active",
            "model": "mWnt_GP",
            "program": "partial"
          }
        ],
        "program_map": [
          {
            "call": "absent",
            "model": "B16_GP"
          },
          {
            "call": "present",
            "model": "KPC_GP"
          },
          {
            "call": "absent",
            "model": "B_ALL_GP"
          },
          {
            "call": "partial",
            "model": "mWnt_GP"
          }
        ],
        "ready_silent_fraction_relations": [
          {
            "pair": "B16_GP_vs_KPC_GP",
            "relation": ">"
          },
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": "~"
          },
          {
            "pair": "KPC_GP_vs_B_ALL_GP",
            "relation": "~"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": "<"
          },
          {
            "pair": "B_ALL_GP_vs_mWnt_GP",
            "relation": "<"
          }
        ],
        "regulator_tuples": [
          {
            "evidence_basis": "target_program_behavior",
            "expr_vs_activity": "concordant",
            "gating_step": "early_release",
            "regulator": "Eomes",
            "role": "repressor"
          },
          {
            "evidence_basis": "own_abundance",
            "expr_vs_activity": "discordant",
            "gating_step": "late_activation",
            "regulator": "Runx3",
            "role": "activator"
          }
        ]
      },
      "nTurns": 30,
      "checkpointSignals": {
        "clustering": false,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": false,
        "nerveJointAnchor": false,
        "multiomeAssembly": true,
        "depthNormalizedAtac": true,
        "regulonInference": true,
        "cd8SubsetIntegration": true,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": false,
        "orthologProjection": false,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": false,
        "mousePseudobulkDesign": false,
        "cancerEpithelialDenominator": false,
        "geneLevelDe": false,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "trajectoryFile": "scbench-long/trajectories/green_2_tme_context_dependence__claude-opus-4-6__pi__r2.json",
      "runId": "claude-opus-4-6__pi__green_2_tme_context_dependence__r2",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-opus-4-6-pi",
      "modelName": "claude-opus-4-6",
      "harness": "pi",
      "task": "green_2_tme_context_dependence",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 3,
      "passed": false,
      "score": 7,
      "durationS": 4013.68,
      "cost": 2.85633125,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "program_map",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "sig_includes_itgae",
            "role": "gate",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Itgae"
            ]
          },
          {
            "name": "declared_a_known_confounder",
            "role": "gate",
            "field": "$.excluded_confounders[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "cf_signature_gut_contaminant",
            "role": "hard_fail",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "generic_signature_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd103_program_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "<"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "per_model_regulatory_state",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "poised_unfired"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "late_activation"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "closing_committed"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_tuples",
            "kind": "list",
            "matchKey": "regulator",
            "k": 1,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "role": {
                    "op": "equals",
                    "values": [
                      "repressor"
                    ]
                  },
                  "gating_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  },
                  "evidence_basis": {
                    "op": "equals",
                    "values": [
                      "target_program_behavior"
                    ]
                  },
                  "expr_vs_activity": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "locus_differential_gate",
            "role": "gate",
            "field": "locus_differential",
            "kind": "value",
            "expected": "uniform_across_models"
          },
          {
            "name": "formation_limited_by_gate",
            "role": "gate",
            "field": "formation_limited_by",
            "kind": "value",
            "expected": "absent_induction"
          },
          {
            "name": "locus_competent_in_naive_gate",
            "role": "gate",
            "field": "locus_competent_in_naive",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "ready_silent_fraction_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 3,
            "expected": [
              {
                "key": "B16_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_KPC_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "activator_chromatin_footprint_gate",
            "role": "gate",
            "field": "activator_chromatin_footprint",
            "kind": "value",
            "expected": "inconsistent_across_formers"
          },
          {
            "name": "exhaustion_comparable_gate",
            "role": "gate",
            "field": "exhaustion_comparable_across_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_nondeterminant_gate",
            "role": "gate",
            "field": "exhaustion_explains_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_separable_program_gate",
            "role": "gate",
            "field": "exhaustion_separable_program",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "exhaustion_counterexample_gate",
            "role": "gate",
            "field": "exhaustion_counterexample_model",
            "kind": "value",
            "expected": "KPC_GP"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "exhaustion_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "cf_inconsistent_present_without_activator",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "program",
            "entity": "present",
            "valueField": "activator_state",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_no_block_but_absent",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "none",
            "valueField": "program",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_late_block_without_poised_locus",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "late_activation",
            "valueField": "locus_state",
            "expected": [
              "closing_committed",
              "active"
            ]
          },
          {
            "name": "cf_inconsistent_early_block_with_brake_released",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "early_release",
            "valueField": "brake_state",
            "expected": [
              "released"
            ]
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: FAIL\n  x list_match: list_match (answer_field='program_map'): FAIL  (score=0.0)\n  + sig_includes_itgae: passed=True, score=1.0\n  x declared_a_known_confounder: passed=False, score=0.0\n  x list_match: list_match (answer_field='generic_signature_relations'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='cd103_program_relations'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='per_model_regulatory_state'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='regulator_tuples'): FAIL  (score=0.0)\n  + locus_differential_gate: passed=True, score=1.0\n  + formation_limited_by_gate: passed=True, score=1.0\n  + locus_competent_in_naive_gate: passed=True, score=1.0\n  x list_match: list_match (answer_field='ready_silent_fraction_relations'): FAIL  (score=0.0)\n  x activator_chromatin_footprint_gate: passed=False, score=0.0\n  + exhaustion_comparable_gate: passed=True, score=1.0\n  + exhaustion_nondeterminant_gate: passed=True, score=1.0\n  + exhaustion_separable_program_gate: passed=True, score=1.0\n  x exhaustion_counterexample_gate: passed=False, score=0.0\n  x list_match: list_match (answer_field='exhaustion_relations'): FAIL  (score=0.0)\n  x hard_fail 'cf_signature_gut_contaminant': TRIGGERED\n  + hard_fail 'cf_inconsistent_present_without_activator': not triggered\n  + hard_fail 'cf_inconsistent_no_block_but_absent': not triggered\n  + hard_fail 'cf_inconsistent_late_block_without_poised_locus': not triggered\n  + hard_fail 'cf_inconsistent_early_block_with_brake_released': not triggered",
      "metrics": {
        "pass_rule": "all",
        "scoring_count": 17,
        "scoring_passed": 7,
        "scoring_total_score": 7,
        "hard_fail_triggered": [
          "cf_signature_gut_contaminant"
        ]
      },
      "fieldScores": {
        "list_match": 0,
        "sig_includes_itgae": 1,
        "declared_a_known_confounder": 0,
        "locus_differential_gate": 1,
        "formation_limited_by_gate": 1,
        "locus_competent_in_naive_gate": 1,
        "activator_chromatin_footprint_gate": 0,
        "exhaustion_comparable_gate": 1,
        "exhaustion_nondeterminant_gate": 1,
        "exhaustion_separable_program_gate": 1,
        "exhaustion_counterexample_gate": 0
      },
      "graderChecks": [
        {
          "name": "program_map",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "sig_includes_itgae",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "declared_a_known_confounder",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "generic_signature_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "cd103_program_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "per_model_regulatory_state",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "regulator_tuples",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "locus_differential_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "formation_limited_by_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "locus_competent_in_naive_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "ready_silent_fraction_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "activator_chromatin_footprint_gate",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "exhaustion_comparable_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "exhaustion_nondeterminant_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "exhaustion_separable_program_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "exhaustion_counterexample_gate",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "exhaustion_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "cf_signature_gut_contaminant",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "cf_inconsistent_present_without_activator",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_no_block_but_absent",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_late_block_without_poised_locus",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_early_block_with_brake_released",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "activator_chromatin_footprint": "consistent_in_formers",
        "cd103_program_relations": [
          {
            "pair": "B16_GP_vs_KPC_GP",
            "relation": "<"
          },
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": "<"
          },
          {
            "pair": "KPC_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "B_ALL_GP_vs_mWnt_GP",
            "relation": "<"
          }
        ],
        "core_residency_signature": [
          "Itgae",
          "Cd69",
          "Itga1",
          "Cxcr6",
          "P2rx7"
        ],
        "excluded_confounders": [
          "Pdcd1",
          "Havcr2",
          "Lag3",
          "Tox",
          "Entpd1"
        ],
        "exhaustion_comparable_across_split": "no",
        "exhaustion_counterexample_model": "B16_GP",
        "exhaustion_explains_split": "no",
        "exhaustion_relations": [
          {
            "pair": "B16_GP_vs_KPC_GP",
            "relation": "~"
          },
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": "<"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_B_ALL_GP",
            "relation": "~"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "B_ALL_GP_vs_mWnt_GP",
            "relation": ">"
          }
        ],
        "exhaustion_separable_program": "yes",
        "formation_limited_by": "absent_induction",
        "generic_signature_relations": [
          {
            "pair": "B16_GP_vs_KPC_GP",
            "relation": "<"
          },
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": "<"
          },
          {
            "pair": "KPC_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": "~"
          },
          {
            "pair": "B_ALL_GP_vs_mWnt_GP",
            "relation": "<"
          }
        ],
        "locus_competent_in_naive": "yes",
        "locus_differential": "uniform_across_models",
        "per_model_regulatory_state": [
          {
            "activator_state": "absent",
            "blocking_step": "late_activation",
            "brake_state": "released",
            "locus_state": "poised_unfired",
            "model": "B16_GP",
            "program": "absent"
          },
          {
            "activator_state": "present",
            "blocking_step": "none",
            "brake_state": "released",
            "locus_state": "active",
            "model": "KPC_GP",
            "program": "present"
          },
          {
            "activator_state": "absent",
            "blocking_step": "both_steps",
            "brake_state": "engaged",
            "locus_state": "poised_unfired",
            "model": "B_ALL_GP",
            "program": "absent"
          },
          {
            "activator_state": "present",
            "blocking_step": "early_release",
            "brake_state": "partial",
            "locus_state": "active",
            "model": "mWnt_GP",
            "program": "partial"
          }
        ],
        "program_map": [
          {
            "call": "absent",
            "model": "B16_GP"
          },
          {
            "call": "present",
            "model": "KPC_GP"
          },
          {
            "call": "absent",
            "model": "B_ALL_GP"
          },
          {
            "call": "partial",
            "model": "mWnt_GP"
          }
        ],
        "ready_silent_fraction_relations": [
          {
            "pair": "B16_GP_vs_KPC_GP",
            "relation": ">"
          },
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": "<"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": "~"
          },
          {
            "pair": "KPC_GP_vs_B_ALL_GP",
            "relation": "<"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": "<"
          },
          {
            "pair": "B_ALL_GP_vs_mWnt_GP",
            "relation": ">"
          }
        ],
        "regulator_tuples": [
          {
            "evidence_basis": "target_program_behavior",
            "expr_vs_activity": "discordant",
            "gating_step": "late_activation",
            "regulator": "Runx3",
            "role": "activator"
          },
          {
            "evidence_basis": "own_abundance",
            "expr_vs_activity": "concordant",
            "gating_step": "early_release",
            "regulator": "Klf2",
            "role": "repressor"
          }
        ]
      },
      "nTurns": 28,
      "checkpointSignals": {
        "clustering": false,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": false,
        "nerveJointAnchor": false,
        "multiomeAssembly": true,
        "depthNormalizedAtac": true,
        "regulonInference": true,
        "cd8SubsetIntegration": true,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": false,
        "orthologProjection": false,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": false,
        "mousePseudobulkDesign": false,
        "cancerEpithelialDenominator": false,
        "geneLevelDe": true,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "trajectoryFile": "scbench-long/trajectories/green_2_tme_context_dependence__claude-opus-4-6__pi__r3.json",
      "runId": "claude-opus-4-6__pi__green_2_tme_context_dependence__r3",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-opus-4-7-claude-code",
      "modelName": "claude-opus-4-7",
      "harness": "claude-code",
      "task": "green_2_tme_context_dependence",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 1,
      "passed": false,
      "score": 9,
      "durationS": 1409.37,
      "cost": 5.70238675,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "program_map",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "sig_includes_itgae",
            "role": "gate",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Itgae"
            ]
          },
          {
            "name": "declared_a_known_confounder",
            "role": "gate",
            "field": "$.excluded_confounders[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "cf_signature_gut_contaminant",
            "role": "hard_fail",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "generic_signature_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd103_program_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "<"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "per_model_regulatory_state",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "poised_unfired"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "late_activation"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "closing_committed"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_tuples",
            "kind": "list",
            "matchKey": "regulator",
            "k": 1,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "role": {
                    "op": "equals",
                    "values": [
                      "repressor"
                    ]
                  },
                  "gating_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  },
                  "evidence_basis": {
                    "op": "equals",
                    "values": [
                      "target_program_behavior"
                    ]
                  },
                  "expr_vs_activity": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "locus_differential_gate",
            "role": "gate",
            "field": "locus_differential",
            "kind": "value",
            "expected": "uniform_across_models"
          },
          {
            "name": "formation_limited_by_gate",
            "role": "gate",
            "field": "formation_limited_by",
            "kind": "value",
            "expected": "absent_induction"
          },
          {
            "name": "locus_competent_in_naive_gate",
            "role": "gate",
            "field": "locus_competent_in_naive",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "ready_silent_fraction_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 3,
            "expected": [
              {
                "key": "B16_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_KPC_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "activator_chromatin_footprint_gate",
            "role": "gate",
            "field": "activator_chromatin_footprint",
            "kind": "value",
            "expected": "inconsistent_across_formers"
          },
          {
            "name": "exhaustion_comparable_gate",
            "role": "gate",
            "field": "exhaustion_comparable_across_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_nondeterminant_gate",
            "role": "gate",
            "field": "exhaustion_explains_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_separable_program_gate",
            "role": "gate",
            "field": "exhaustion_separable_program",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "exhaustion_counterexample_gate",
            "role": "gate",
            "field": "exhaustion_counterexample_model",
            "kind": "value",
            "expected": "KPC_GP"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "exhaustion_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "cf_inconsistent_present_without_activator",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "program",
            "entity": "present",
            "valueField": "activator_state",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_no_block_but_absent",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "none",
            "valueField": "program",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_late_block_without_poised_locus",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "late_activation",
            "valueField": "locus_state",
            "expected": [
              "closing_committed",
              "active"
            ]
          },
          {
            "name": "cf_inconsistent_early_block_with_brake_released",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "early_release",
            "valueField": "brake_state",
            "expected": [
              "released"
            ]
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: FAIL\n  x list_match: list_match (answer_field='program_map'): FAIL  (score=0.0)\n  + sig_includes_itgae: passed=True, score=1.0\n  x declared_a_known_confounder: passed=False, score=0.0\n  x list_match: list_match (answer_field='generic_signature_relations'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='cd103_program_relations'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='per_model_regulatory_state'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='regulator_tuples'): FAIL  (score=0.0)\n  + locus_differential_gate: passed=True, score=1.0\n  + formation_limited_by_gate: passed=True, score=1.0\n  + locus_competent_in_naive_gate: passed=True, score=1.0\n  x list_match: list_match (answer_field='ready_silent_fraction_relations'): FAIL  (score=0.0)\n  + activator_chromatin_footprint_gate: passed=True, score=1.0\n  + exhaustion_comparable_gate: passed=True, score=1.0\n  + exhaustion_nondeterminant_gate: passed=True, score=1.0\n  + exhaustion_separable_program_gate: passed=True, score=1.0\n  + exhaustion_counterexample_gate: passed=True, score=1.0\n  + list_match: list_match (answer_field='exhaustion_relations'): PASS  (score=0.0)\n  x hard_fail 'cf_signature_gut_contaminant': TRIGGERED\n  + hard_fail 'cf_inconsistent_present_without_activator': not triggered\n  + hard_fail 'cf_inconsistent_no_block_but_absent': not triggered\n  + hard_fail 'cf_inconsistent_late_block_without_poised_locus': not triggered\n  + hard_fail 'cf_inconsistent_early_block_with_brake_released': not triggered",
      "metrics": {
        "pass_rule": "all",
        "scoring_count": 17,
        "scoring_passed": 10,
        "scoring_total_score": 9,
        "hard_fail_triggered": [
          "cf_signature_gut_contaminant"
        ]
      },
      "fieldScores": {
        "list_match": 0,
        "sig_includes_itgae": 1,
        "declared_a_known_confounder": 0,
        "locus_differential_gate": 1,
        "formation_limited_by_gate": 1,
        "locus_competent_in_naive_gate": 1,
        "activator_chromatin_footprint_gate": 1,
        "exhaustion_comparable_gate": 1,
        "exhaustion_nondeterminant_gate": 1,
        "exhaustion_separable_program_gate": 1,
        "exhaustion_counterexample_gate": 1
      },
      "graderChecks": [
        {
          "name": "program_map",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "sig_includes_itgae",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "declared_a_known_confounder",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "generic_signature_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "cd103_program_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "per_model_regulatory_state",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "regulator_tuples",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "locus_differential_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "formation_limited_by_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "locus_competent_in_naive_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "ready_silent_fraction_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "activator_chromatin_footprint_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "exhaustion_comparable_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "exhaustion_nondeterminant_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "exhaustion_separable_program_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "exhaustion_counterexample_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "exhaustion_relations",
          "kind": "list_match",
          "passed": true,
          "score": 0
        },
        {
          "name": "cf_signature_gut_contaminant",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "cf_inconsistent_present_without_activator",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_no_block_but_absent",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_late_block_without_poised_locus",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_early_block_with_brake_released",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "activator_chromatin_footprint": "inconsistent_across_formers",
        "cd103_program_relations": [
          {
            "pair": "KPC_GP_vs_B16_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": "~"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": "<"
          },
          {
            "pair": "B_ALL_GP_vs_mWnt_GP",
            "relation": "<"
          }
        ],
        "core_residency_signature": [
          "Itgae",
          "Itga1",
          "Cd69",
          "Cxcr6",
          "Rgs1",
          "Xcl1",
          "Cd160",
          "Runx3",
          "Bhlhe40",
          "Ahr",
          "Prdm1",
          "Litaf",
          "Inpp4b",
          "Nr4a2"
        ],
        "excluded_confounders": [
          "Klf2",
          "S1pr1",
          "Sell",
          "Ccr7",
          "S1pr5",
          "Klrg1",
          "Cx3cr1",
          "Ly6c2"
        ],
        "exhaustion_comparable_across_split": "no",
        "exhaustion_counterexample_model": "KPC_GP",
        "exhaustion_explains_split": "no",
        "exhaustion_relations": [
          {
            "pair": "KPC_GP_vs_B16_GP",
            "relation": "~"
          },
          {
            "pair": "KPC_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "B_ALL_GP_vs_mWnt_GP",
            "relation": ">"
          }
        ],
        "exhaustion_separable_program": "yes",
        "formation_limited_by": "absent_induction",
        "generic_signature_relations": [
          {
            "pair": "KPC_GP_vs_B16_GP",
            "relation": "~"
          },
          {
            "pair": "KPC_GP_vs_B_ALL_GP",
            "relation": "<"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": "<"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "B_ALL_GP_vs_mWnt_GP",
            "relation": ">"
          }
        ],
        "locus_competent_in_naive": "yes",
        "locus_differential": "uniform_across_models",
        "per_model_regulatory_state": [
          {
            "activator_state": "present",
            "blocking_step": "none",
            "brake_state": "released",
            "locus_state": "active",
            "model": "KPC_GP",
            "program": "present"
          },
          {
            "activator_state": "partial",
            "blocking_step": "late_activation",
            "brake_state": "released",
            "locus_state": "poised_unfired",
            "model": "B16_GP",
            "program": "absent"
          },
          {
            "activator_state": "partial",
            "blocking_step": "both_steps",
            "brake_state": "partial",
            "locus_state": "closing_committed",
            "model": "B_ALL_GP",
            "program": "absent"
          },
          {
            "activator_state": "present",
            "blocking_step": "early_release",
            "brake_state": "partial",
            "locus_state": "active",
            "model": "mWnt_GP",
            "program": "partial"
          }
        ],
        "program_map": [
          {
            "call": "present",
            "model": "KPC_GP"
          },
          {
            "call": "absent",
            "model": "B16_GP"
          },
          {
            "call": "absent",
            "model": "B_ALL_GP"
          },
          {
            "call": "partial",
            "model": "mWnt_GP"
          }
        ],
        "ready_silent_fraction_relations": [
          {
            "pair": "KPC_GP_vs_B16_GP",
            "relation": "<"
          },
          {
            "pair": "KPC_GP_vs_B_ALL_GP",
            "relation": "<"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": "<"
          },
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "B_ALL_GP_vs_mWnt_GP",
            "relation": "~"
          }
        ],
        "regulator_tuples": [
          {
            "evidence_basis": "own_abundance",
            "expr_vs_activity": "concordant",
            "gating_step": "early_release",
            "regulator": "Klf2",
            "role": "repressor"
          },
          {
            "evidence_basis": "own_abundance",
            "expr_vs_activity": "concordant",
            "gating_step": "early_release",
            "regulator": "S1pr1",
            "role": "repressor"
          },
          {
            "evidence_basis": "target_program_behavior",
            "expr_vs_activity": "discordant",
            "gating_step": "late_activation",
            "regulator": "Runx3",
            "role": "activator"
          },
          {
            "evidence_basis": "target_program_behavior",
            "expr_vs_activity": "discordant",
            "gating_step": "late_activation",
            "regulator": "Bhlhe40",
            "role": "activator"
          },
          {
            "evidence_basis": "own_abundance",
            "expr_vs_activity": "concordant",
            "gating_step": "late_activation",
            "regulator": "Ahr",
            "role": "activator"
          },
          {
            "evidence_basis": "target_program_behavior",
            "expr_vs_activity": "discordant",
            "gating_step": "late_activation",
            "regulator": "Prdm1",
            "role": "activator"
          },
          {
            "evidence_basis": "target_program_behavior",
            "expr_vs_activity": "indeterminate",
            "gating_step": "late_activation",
            "regulator": "Zfp683",
            "role": "activator"
          },
          {
            "evidence_basis": "target_program_behavior",
            "expr_vs_activity": "discordant",
            "gating_step": "indeterminate",
            "regulator": "Tox",
            "role": "dual"
          }
        ]
      },
      "nTurns": 74,
      "checkpointSignals": {
        "clustering": false,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": false,
        "nerveJointAnchor": false,
        "multiomeAssembly": true,
        "depthNormalizedAtac": true,
        "regulonInference": true,
        "cd8SubsetIntegration": true,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": false,
        "orthologProjection": false,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": false,
        "mousePseudobulkDesign": false,
        "cancerEpithelialDenominator": false,
        "geneLevelDe": false,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "trajectoryFile": "scbench-long/trajectories/green_2_tme_context_dependence__claude-opus-4-7__claude-code__r1.json",
      "runId": "claude-opus-4-7__claude-code__green_2_tme_context_dependence__r1",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-opus-4-7-claude-code",
      "modelName": "claude-opus-4-7",
      "harness": "claude-code",
      "task": "green_2_tme_context_dependence",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 2,
      "passed": false,
      "score": 7,
      "durationS": 1886.75,
      "cost": 5.16936,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "program_map",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "sig_includes_itgae",
            "role": "gate",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Itgae"
            ]
          },
          {
            "name": "declared_a_known_confounder",
            "role": "gate",
            "field": "$.excluded_confounders[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "cf_signature_gut_contaminant",
            "role": "hard_fail",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "generic_signature_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd103_program_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "<"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "per_model_regulatory_state",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "poised_unfired"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "late_activation"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "closing_committed"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_tuples",
            "kind": "list",
            "matchKey": "regulator",
            "k": 1,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "role": {
                    "op": "equals",
                    "values": [
                      "repressor"
                    ]
                  },
                  "gating_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  },
                  "evidence_basis": {
                    "op": "equals",
                    "values": [
                      "target_program_behavior"
                    ]
                  },
                  "expr_vs_activity": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "locus_differential_gate",
            "role": "gate",
            "field": "locus_differential",
            "kind": "value",
            "expected": "uniform_across_models"
          },
          {
            "name": "formation_limited_by_gate",
            "role": "gate",
            "field": "formation_limited_by",
            "kind": "value",
            "expected": "absent_induction"
          },
          {
            "name": "locus_competent_in_naive_gate",
            "role": "gate",
            "field": "locus_competent_in_naive",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "ready_silent_fraction_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 3,
            "expected": [
              {
                "key": "B16_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_KPC_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "activator_chromatin_footprint_gate",
            "role": "gate",
            "field": "activator_chromatin_footprint",
            "kind": "value",
            "expected": "inconsistent_across_formers"
          },
          {
            "name": "exhaustion_comparable_gate",
            "role": "gate",
            "field": "exhaustion_comparable_across_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_nondeterminant_gate",
            "role": "gate",
            "field": "exhaustion_explains_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_separable_program_gate",
            "role": "gate",
            "field": "exhaustion_separable_program",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "exhaustion_counterexample_gate",
            "role": "gate",
            "field": "exhaustion_counterexample_model",
            "kind": "value",
            "expected": "KPC_GP"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "exhaustion_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "cf_inconsistent_present_without_activator",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "program",
            "entity": "present",
            "valueField": "activator_state",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_no_block_but_absent",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "none",
            "valueField": "program",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_late_block_without_poised_locus",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "late_activation",
            "valueField": "locus_state",
            "expected": [
              "closing_committed",
              "active"
            ]
          },
          {
            "name": "cf_inconsistent_early_block_with_brake_released",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "early_release",
            "valueField": "brake_state",
            "expected": [
              "released"
            ]
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: FAIL\n  x list_match: list_match (answer_field='program_map'): FAIL  (score=0.0)\n  + sig_includes_itgae: passed=True, score=1.0\n  x declared_a_known_confounder: passed=False, score=0.0\n  x list_match: list_match (answer_field='generic_signature_relations'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='cd103_program_relations'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='per_model_regulatory_state'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='regulator_tuples'): FAIL  (score=0.0)\n  + locus_differential_gate: passed=True, score=1.0\n  + formation_limited_by_gate: passed=True, score=1.0\n  + locus_competent_in_naive_gate: passed=True, score=1.0\n  x list_match: list_match (answer_field='ready_silent_fraction_relations'): FAIL  (score=0.0)\n  x activator_chromatin_footprint_gate: passed=False, score=0.0\n  x exhaustion_comparable_gate: passed=False, score=0.0\n  + exhaustion_nondeterminant_gate: passed=True, score=1.0\n  + exhaustion_separable_program_gate: passed=True, score=1.0\n  + exhaustion_counterexample_gate: passed=True, score=1.0\n  x list_match: list_match (answer_field='exhaustion_relations'): FAIL  (score=0.0)\n  x hard_fail 'cf_signature_gut_contaminant': TRIGGERED\n  + hard_fail 'cf_inconsistent_present_without_activator': not triggered\n  + hard_fail 'cf_inconsistent_no_block_but_absent': not triggered\n  + hard_fail 'cf_inconsistent_late_block_without_poised_locus': not triggered\n  + hard_fail 'cf_inconsistent_early_block_with_brake_released': not triggered",
      "metrics": {
        "pass_rule": "all",
        "scoring_count": 17,
        "scoring_passed": 7,
        "scoring_total_score": 7,
        "hard_fail_triggered": [
          "cf_signature_gut_contaminant"
        ]
      },
      "fieldScores": {
        "list_match": 0,
        "sig_includes_itgae": 1,
        "declared_a_known_confounder": 0,
        "locus_differential_gate": 1,
        "formation_limited_by_gate": 1,
        "locus_competent_in_naive_gate": 1,
        "activator_chromatin_footprint_gate": 0,
        "exhaustion_comparable_gate": 0,
        "exhaustion_nondeterminant_gate": 1,
        "exhaustion_separable_program_gate": 1,
        "exhaustion_counterexample_gate": 1
      },
      "graderChecks": [
        {
          "name": "program_map",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "sig_includes_itgae",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "declared_a_known_confounder",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "generic_signature_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "cd103_program_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "per_model_regulatory_state",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "regulator_tuples",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "locus_differential_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "formation_limited_by_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "locus_competent_in_naive_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "ready_silent_fraction_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "activator_chromatin_footprint_gate",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "exhaustion_comparable_gate",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "exhaustion_nondeterminant_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "exhaustion_separable_program_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "exhaustion_counterexample_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "exhaustion_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "cf_signature_gut_contaminant",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "cf_inconsistent_present_without_activator",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_no_block_but_absent",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_late_block_without_poised_locus",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_early_block_with_brake_released",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "activator_chromatin_footprint": "consistent_in_formers",
        "cd103_program_relations": [
          {
            "pair": "B16_GP_vs_KPC_GP",
            "relation": "<"
          },
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": "<"
          },
          {
            "pair": "KPC_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": "~"
          },
          {
            "pair": "B_ALL_GP_vs_mWnt_GP",
            "relation": "<"
          }
        ],
        "core_residency_signature": [
          "Itgae",
          "Itga1",
          "Cxcr6",
          "Cd69",
          "Rgs1",
          "Crtam",
          "Xcl1",
          "Bhlhe40",
          "Runx3",
          "Prdm1",
          "Zfp683"
        ],
        "excluded_confounders": [
          "Cd160",
          "Atp8a2",
          "Chn2",
          "Itga4",
          "Itgb7",
          "Gzmb",
          "Aopep"
        ],
        "exhaustion_comparable_across_split": "yes",
        "exhaustion_counterexample_model": "KPC_GP",
        "exhaustion_explains_split": "no",
        "exhaustion_relations": [
          {
            "pair": "B16_GP_vs_KPC_GP",
            "relation": "~"
          },
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": "~"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_B_ALL_GP",
            "relation": "~"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "B_ALL_GP_vs_mWnt_GP",
            "relation": ">"
          }
        ],
        "exhaustion_separable_program": "yes",
        "formation_limited_by": "absent_induction",
        "generic_signature_relations": [
          {
            "pair": "B16_GP_vs_KPC_GP",
            "relation": ">"
          },
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_B_ALL_GP",
            "relation": "~"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "B_ALL_GP_vs_mWnt_GP",
            "relation": ">"
          }
        ],
        "locus_competent_in_naive": "yes",
        "locus_differential": "uniform_across_models",
        "per_model_regulatory_state": [
          {
            "activator_state": "present",
            "blocking_step": "none",
            "brake_state": "released",
            "locus_state": "active",
            "model": "KPC_GP",
            "program": "present"
          },
          {
            "activator_state": "partial",
            "blocking_step": "early_release",
            "brake_state": "partial",
            "locus_state": "active",
            "model": "mWnt_GP",
            "program": "partial"
          },
          {
            "activator_state": "present",
            "blocking_step": "late_activation",
            "brake_state": "released",
            "locus_state": "poised_unfired",
            "model": "B16_GP",
            "program": "absent"
          },
          {
            "activator_state": "partial",
            "blocking_step": "both_steps",
            "brake_state": "partial",
            "locus_state": "poised_unfired",
            "model": "B_ALL_GP",
            "program": "absent"
          }
        ],
        "program_map": [
          {
            "call": "present",
            "model": "KPC_GP"
          },
          {
            "call": "partial",
            "model": "mWnt_GP"
          },
          {
            "call": "absent",
            "model": "B16_GP"
          },
          {
            "call": "absent",
            "model": "B_ALL_GP"
          }
        ],
        "ready_silent_fraction_relations": [
          {
            "pair": "B16_GP_vs_KPC_GP",
            "relation": ">"
          },
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": "<"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": "<"
          },
          {
            "pair": "KPC_GP_vs_B_ALL_GP",
            "relation": "<"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": "<"
          },
          {
            "pair": "B_ALL_GP_vs_mWnt_GP",
            "relation": ">"
          }
        ],
        "regulator_tuples": [
          {
            "evidence_basis": "own_abundance",
            "expr_vs_activity": "concordant",
            "gating_step": "early_release",
            "regulator": "Klf2",
            "role": "repressor"
          },
          {
            "evidence_basis": "own_abundance",
            "expr_vs_activity": "concordant",
            "gating_step": "early_release",
            "regulator": "S1pr1",
            "role": "repressor"
          },
          {
            "evidence_basis": "target_program_behavior",
            "expr_vs_activity": "discordant",
            "gating_step": "late_activation",
            "regulator": "Runx3",
            "role": "activator"
          },
          {
            "evidence_basis": "target_program_behavior",
            "expr_vs_activity": "discordant",
            "gating_step": "late_activation",
            "regulator": "Bhlhe40",
            "role": "activator"
          },
          {
            "evidence_basis": "target_program_behavior",
            "expr_vs_activity": "discordant",
            "gating_step": "late_activation",
            "regulator": "Rbpj",
            "role": "activator"
          },
          {
            "evidence_basis": "target_program_behavior",
            "expr_vs_activity": "indeterminate",
            "gating_step": "late_activation",
            "regulator": "Prdm1",
            "role": "activator"
          },
          {
            "evidence_basis": "own_abundance",
            "expr_vs_activity": "discordant",
            "gating_step": "early_release",
            "regulator": "Tcf7",
            "role": "repressor"
          }
        ]
      },
      "nTurns": 54,
      "checkpointSignals": {
        "clustering": false,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": false,
        "nerveJointAnchor": false,
        "multiomeAssembly": true,
        "depthNormalizedAtac": true,
        "regulonInference": true,
        "cd8SubsetIntegration": true,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": false,
        "orthologProjection": false,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": false,
        "mousePseudobulkDesign": false,
        "cancerEpithelialDenominator": false,
        "geneLevelDe": false,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "trajectoryFile": "scbench-long/trajectories/green_2_tme_context_dependence__claude-opus-4-7__claude-code__r2.json",
      "runId": "claude-opus-4-7__claude-code__green_2_tme_context_dependence__r2",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-opus-4-7-claude-code",
      "modelName": "claude-opus-4-7",
      "harness": "claude-code",
      "task": "green_2_tme_context_dependence",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 3,
      "passed": false,
      "score": 7,
      "durationS": 1633.36,
      "cost": 8.279638000000002,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "program_map",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "sig_includes_itgae",
            "role": "gate",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Itgae"
            ]
          },
          {
            "name": "declared_a_known_confounder",
            "role": "gate",
            "field": "$.excluded_confounders[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "cf_signature_gut_contaminant",
            "role": "hard_fail",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "generic_signature_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd103_program_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "<"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "per_model_regulatory_state",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "poised_unfired"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "late_activation"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "closing_committed"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_tuples",
            "kind": "list",
            "matchKey": "regulator",
            "k": 1,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "role": {
                    "op": "equals",
                    "values": [
                      "repressor"
                    ]
                  },
                  "gating_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  },
                  "evidence_basis": {
                    "op": "equals",
                    "values": [
                      "target_program_behavior"
                    ]
                  },
                  "expr_vs_activity": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "locus_differential_gate",
            "role": "gate",
            "field": "locus_differential",
            "kind": "value",
            "expected": "uniform_across_models"
          },
          {
            "name": "formation_limited_by_gate",
            "role": "gate",
            "field": "formation_limited_by",
            "kind": "value",
            "expected": "absent_induction"
          },
          {
            "name": "locus_competent_in_naive_gate",
            "role": "gate",
            "field": "locus_competent_in_naive",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "ready_silent_fraction_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 3,
            "expected": [
              {
                "key": "B16_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_KPC_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "activator_chromatin_footprint_gate",
            "role": "gate",
            "field": "activator_chromatin_footprint",
            "kind": "value",
            "expected": "inconsistent_across_formers"
          },
          {
            "name": "exhaustion_comparable_gate",
            "role": "gate",
            "field": "exhaustion_comparable_across_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_nondeterminant_gate",
            "role": "gate",
            "field": "exhaustion_explains_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_separable_program_gate",
            "role": "gate",
            "field": "exhaustion_separable_program",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "exhaustion_counterexample_gate",
            "role": "gate",
            "field": "exhaustion_counterexample_model",
            "kind": "value",
            "expected": "KPC_GP"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "exhaustion_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "cf_inconsistent_present_without_activator",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "program",
            "entity": "present",
            "valueField": "activator_state",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_no_block_but_absent",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "none",
            "valueField": "program",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_late_block_without_poised_locus",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "late_activation",
            "valueField": "locus_state",
            "expected": [
              "closing_committed",
              "active"
            ]
          },
          {
            "name": "cf_inconsistent_early_block_with_brake_released",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "early_release",
            "valueField": "brake_state",
            "expected": [
              "released"
            ]
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: FAIL\n  x list_match: list_match (answer_field='program_map'): FAIL  (score=0.0)\n  + sig_includes_itgae: passed=True, score=1.0\n  x declared_a_known_confounder: passed=False, score=0.0\n  x list_match: list_match (answer_field='generic_signature_relations'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='cd103_program_relations'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='per_model_regulatory_state'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='regulator_tuples'): FAIL  (score=0.0)\n  + locus_differential_gate: passed=True, score=1.0\n  + formation_limited_by_gate: passed=True, score=1.0\n  + locus_competent_in_naive_gate: passed=True, score=1.0\n  x list_match: list_match (answer_field='ready_silent_fraction_relations'): FAIL  (score=0.0)\n  x activator_chromatin_footprint_gate: passed=False, score=0.0\n  x exhaustion_comparable_gate: passed=False, score=0.0\n  + exhaustion_nondeterminant_gate: passed=True, score=1.0\n  + exhaustion_separable_program_gate: passed=True, score=1.0\n  + exhaustion_counterexample_gate: passed=True, score=1.0\n  x list_match: list_match (answer_field='exhaustion_relations'): FAIL  (score=0.0)\n  x hard_fail 'cf_signature_gut_contaminant': TRIGGERED\n  + hard_fail 'cf_inconsistent_present_without_activator': not triggered\n  + hard_fail 'cf_inconsistent_no_block_but_absent': not triggered\n  + hard_fail 'cf_inconsistent_late_block_without_poised_locus': not triggered\n  + hard_fail 'cf_inconsistent_early_block_with_brake_released': not triggered",
      "metrics": {
        "pass_rule": "all",
        "scoring_count": 17,
        "scoring_passed": 7,
        "scoring_total_score": 7,
        "hard_fail_triggered": [
          "cf_signature_gut_contaminant"
        ]
      },
      "fieldScores": {
        "list_match": 0,
        "sig_includes_itgae": 1,
        "declared_a_known_confounder": 0,
        "locus_differential_gate": 1,
        "formation_limited_by_gate": 1,
        "locus_competent_in_naive_gate": 1,
        "activator_chromatin_footprint_gate": 0,
        "exhaustion_comparable_gate": 0,
        "exhaustion_nondeterminant_gate": 1,
        "exhaustion_separable_program_gate": 1,
        "exhaustion_counterexample_gate": 1
      },
      "graderChecks": [
        {
          "name": "program_map",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "sig_includes_itgae",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "declared_a_known_confounder",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "generic_signature_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "cd103_program_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "per_model_regulatory_state",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "regulator_tuples",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "locus_differential_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "formation_limited_by_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "locus_competent_in_naive_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "ready_silent_fraction_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "activator_chromatin_footprint_gate",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "exhaustion_comparable_gate",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "exhaustion_nondeterminant_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "exhaustion_separable_program_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "exhaustion_counterexample_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "exhaustion_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "cf_signature_gut_contaminant",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "cf_inconsistent_present_without_activator",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_no_block_but_absent",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_late_block_without_poised_locus",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_early_block_with_brake_released",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "activator_chromatin_footprint": "consistent_in_formers",
        "cd103_program_relations": [
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "B16_GP_vs_KPC_GP",
            "relation": "<"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": "<"
          },
          {
            "pair": "B_ALL_GP_vs_KPC_GP",
            "relation": "<"
          },
          {
            "pair": "B_ALL_GP_vs_mWnt_GP",
            "relation": "<"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": ">"
          }
        ],
        "core_residency_signature": [
          "Itgae",
          "Itga1",
          "Cd69",
          "Cxcr6",
          "Rgs1",
          "Crtam",
          "Zfp683",
          "Runx3",
          "Bhlhe40",
          "Prdm1",
          "Nr4a2"
        ],
        "excluded_confounders": [
          "Pdcd1",
          "Tox",
          "Lag3",
          "Havcr2",
          "Tigit",
          "Klf2",
          "S1pr1",
          "Sell",
          "Tcf7",
          "Eomes",
          "Gzmb",
          "Gzmk",
          "Ifng",
          "Mki67"
        ],
        "exhaustion_comparable_across_split": "yes",
        "exhaustion_counterexample_model": "KPC_GP",
        "exhaustion_explains_split": "no",
        "exhaustion_relations": [
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": "<"
          },
          {
            "pair": "B16_GP_vs_KPC_GP",
            "relation": "~"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "B_ALL_GP_vs_KPC_GP",
            "relation": "~"
          },
          {
            "pair": "B_ALL_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": ">"
          }
        ],
        "exhaustion_separable_program": "yes",
        "formation_limited_by": "absent_induction",
        "generic_signature_relations": [
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": "<"
          },
          {
            "pair": "B16_GP_vs_KPC_GP",
            "relation": "~"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "B_ALL_GP_vs_KPC_GP",
            "relation": "~"
          },
          {
            "pair": "B_ALL_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": ">"
          }
        ],
        "locus_competent_in_naive": "yes",
        "locus_differential": "uniform_across_models",
        "per_model_regulatory_state": [
          {
            "activator_state": "present",
            "blocking_step": "none",
            "brake_state": "released",
            "locus_state": "active",
            "model": "KPC_GP",
            "program": "present"
          },
          {
            "activator_state": "partial",
            "blocking_step": "late_activation",
            "brake_state": "partial",
            "locus_state": "poised_unfired",
            "model": "mWnt_GP",
            "program": "partial"
          },
          {
            "activator_state": "partial",
            "blocking_step": "late_activation",
            "brake_state": "released",
            "locus_state": "poised_unfired",
            "model": "B16_GP",
            "program": "absent"
          },
          {
            "activator_state": "present",
            "blocking_step": "early_release",
            "brake_state": "engaged",
            "locus_state": "poised_unfired",
            "model": "B_ALL_GP",
            "program": "absent"
          }
        ],
        "program_map": [
          {
            "call": "present",
            "model": "KPC_GP"
          },
          {
            "call": "partial",
            "model": "mWnt_GP"
          },
          {
            "call": "absent",
            "model": "B16_GP"
          },
          {
            "call": "absent",
            "model": "B_ALL_GP"
          }
        ],
        "ready_silent_fraction_relations": [
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": "<"
          },
          {
            "pair": "B16_GP_vs_KPC_GP",
            "relation": ">"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": "<"
          },
          {
            "pair": "B_ALL_GP_vs_KPC_GP",
            "relation": ">"
          },
          {
            "pair": "B_ALL_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": "<"
          }
        ],
        "regulator_tuples": [
          {
            "evidence_basis": "target_program_behavior",
            "expr_vs_activity": "concordant",
            "gating_step": "late_activation",
            "regulator": "Runx3",
            "role": "activator"
          },
          {
            "evidence_basis": "indeterminate",
            "expr_vs_activity": "indeterminate",
            "gating_step": "late_activation",
            "regulator": "Zfp683",
            "role": "activator"
          },
          {
            "evidence_basis": "target_program_behavior",
            "expr_vs_activity": "discordant",
            "gating_step": "late_activation",
            "regulator": "Bhlhe40",
            "role": "activator"
          },
          {
            "evidence_basis": "target_program_behavior",
            "expr_vs_activity": "discordant",
            "gating_step": "late_activation",
            "regulator": "Prdm1",
            "role": "activator"
          },
          {
            "evidence_basis": "own_abundance",
            "expr_vs_activity": "concordant",
            "gating_step": "early_release",
            "regulator": "Eomes",
            "role": "repressor"
          },
          {
            "evidence_basis": "own_abundance",
            "expr_vs_activity": "concordant",
            "gating_step": "early_release",
            "regulator": "Klf2",
            "role": "repressor"
          },
          {
            "evidence_basis": "own_abundance",
            "expr_vs_activity": "concordant",
            "gating_step": "early_release",
            "regulator": "S1pr1",
            "role": "repressor"
          }
        ]
      },
      "nTurns": 106,
      "checkpointSignals": {
        "clustering": false,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": false,
        "nerveJointAnchor": false,
        "multiomeAssembly": true,
        "depthNormalizedAtac": true,
        "regulonInference": true,
        "cd8SubsetIntegration": true,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": false,
        "orthologProjection": false,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": false,
        "mousePseudobulkDesign": false,
        "cancerEpithelialDenominator": false,
        "geneLevelDe": false,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "trajectoryFile": "scbench-long/trajectories/green_2_tme_context_dependence__claude-opus-4-7__claude-code__r3.json",
      "runId": "claude-opus-4-7__claude-code__green_2_tme_context_dependence__r3",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-opus-4-7-pi",
      "modelName": "claude-opus-4-7",
      "harness": "pi",
      "task": "green_2_tme_context_dependence",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 1,
      "passed": false,
      "score": 7,
      "durationS": 1403.43,
      "cost": null,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "program_map",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "sig_includes_itgae",
            "role": "gate",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Itgae"
            ]
          },
          {
            "name": "declared_a_known_confounder",
            "role": "gate",
            "field": "$.excluded_confounders[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "cf_signature_gut_contaminant",
            "role": "hard_fail",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "generic_signature_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd103_program_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "<"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "per_model_regulatory_state",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "poised_unfired"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "late_activation"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "closing_committed"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_tuples",
            "kind": "list",
            "matchKey": "regulator",
            "k": 1,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "role": {
                    "op": "equals",
                    "values": [
                      "repressor"
                    ]
                  },
                  "gating_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  },
                  "evidence_basis": {
                    "op": "equals",
                    "values": [
                      "target_program_behavior"
                    ]
                  },
                  "expr_vs_activity": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "locus_differential_gate",
            "role": "gate",
            "field": "locus_differential",
            "kind": "value",
            "expected": "uniform_across_models"
          },
          {
            "name": "formation_limited_by_gate",
            "role": "gate",
            "field": "formation_limited_by",
            "kind": "value",
            "expected": "absent_induction"
          },
          {
            "name": "locus_competent_in_naive_gate",
            "role": "gate",
            "field": "locus_competent_in_naive",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "ready_silent_fraction_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 3,
            "expected": [
              {
                "key": "B16_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_KPC_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "activator_chromatin_footprint_gate",
            "role": "gate",
            "field": "activator_chromatin_footprint",
            "kind": "value",
            "expected": "inconsistent_across_formers"
          },
          {
            "name": "exhaustion_comparable_gate",
            "role": "gate",
            "field": "exhaustion_comparable_across_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_nondeterminant_gate",
            "role": "gate",
            "field": "exhaustion_explains_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_separable_program_gate",
            "role": "gate",
            "field": "exhaustion_separable_program",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "exhaustion_counterexample_gate",
            "role": "gate",
            "field": "exhaustion_counterexample_model",
            "kind": "value",
            "expected": "KPC_GP"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "exhaustion_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "cf_inconsistent_present_without_activator",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "program",
            "entity": "present",
            "valueField": "activator_state",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_no_block_but_absent",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "none",
            "valueField": "program",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_late_block_without_poised_locus",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "late_activation",
            "valueField": "locus_state",
            "expected": [
              "closing_committed",
              "active"
            ]
          },
          {
            "name": "cf_inconsistent_early_block_with_brake_released",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "early_release",
            "valueField": "brake_state",
            "expected": [
              "released"
            ]
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: FAIL\n  x list_match: list_match (answer_field='program_map'): FAIL  (score=0.0)\n  + sig_includes_itgae: passed=True, score=1.0\n  x declared_a_known_confounder: passed=False, score=0.0\n  + list_match: list_match (answer_field='generic_signature_relations'): PASS  (score=0.0)\n  x list_match: list_match (answer_field='cd103_program_relations'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='per_model_regulatory_state'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='regulator_tuples'): FAIL  (score=0.0)\n  + locus_differential_gate: passed=True, score=1.0\n  + formation_limited_by_gate: passed=True, score=1.0\n  + locus_competent_in_naive_gate: passed=True, score=1.0\n  x list_match: list_match (answer_field='ready_silent_fraction_relations'): FAIL  (score=0.0)\n  x activator_chromatin_footprint_gate: passed=False, score=0.0\n  x exhaustion_comparable_gate: passed=False, score=0.0\n  + exhaustion_nondeterminant_gate: passed=True, score=1.0\n  + exhaustion_separable_program_gate: passed=True, score=1.0\n  + exhaustion_counterexample_gate: passed=True, score=1.0\n  x list_match: list_match (answer_field='exhaustion_relations'): FAIL  (score=0.0)\n  x hard_fail 'cf_signature_gut_contaminant': TRIGGERED\n  + hard_fail 'cf_inconsistent_present_without_activator': not triggered\n  + hard_fail 'cf_inconsistent_no_block_but_absent': not triggered\n  + hard_fail 'cf_inconsistent_late_block_without_poised_locus': not triggered\n  + hard_fail 'cf_inconsistent_early_block_with_brake_released': not triggered",
      "metrics": {
        "pass_rule": "all",
        "scoring_count": 17,
        "scoring_passed": 8,
        "scoring_total_score": 7,
        "hard_fail_triggered": [
          "cf_signature_gut_contaminant"
        ]
      },
      "fieldScores": {
        "list_match": 0,
        "sig_includes_itgae": 1,
        "declared_a_known_confounder": 0,
        "locus_differential_gate": 1,
        "formation_limited_by_gate": 1,
        "locus_competent_in_naive_gate": 1,
        "activator_chromatin_footprint_gate": 0,
        "exhaustion_comparable_gate": 0,
        "exhaustion_nondeterminant_gate": 1,
        "exhaustion_separable_program_gate": 1,
        "exhaustion_counterexample_gate": 1
      },
      "graderChecks": [
        {
          "name": "program_map",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "sig_includes_itgae",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "declared_a_known_confounder",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "generic_signature_relations",
          "kind": "list_match",
          "passed": true,
          "score": 0
        },
        {
          "name": "cd103_program_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "per_model_regulatory_state",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "regulator_tuples",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "locus_differential_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "formation_limited_by_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "locus_competent_in_naive_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "ready_silent_fraction_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "activator_chromatin_footprint_gate",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "exhaustion_comparable_gate",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "exhaustion_nondeterminant_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "exhaustion_separable_program_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "exhaustion_counterexample_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "exhaustion_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "cf_signature_gut_contaminant",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "cf_inconsistent_present_without_activator",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_no_block_but_absent",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_late_block_without_poised_locus",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_early_block_with_brake_released",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "activator_chromatin_footprint": "consistent_in_formers",
        "cd103_program_relations": [
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_B16_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "mWnt_GP_vs_B16_GP",
            "relation": ">"
          },
          {
            "pair": "mWnt_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": "~"
          }
        ],
        "core_residency_signature": [
          "Itgae",
          "Itga1",
          "Cxcr6",
          "Rgs1",
          "Rgs2",
          "Crtam",
          "Cdh1",
          "Zfp683"
        ],
        "excluded_confounders": [
          "Cd69",
          "Pdcd1",
          "Tox",
          "Havcr2",
          "Lag3",
          "Tigit",
          "Gzmb",
          "Mki67",
          "Ifng"
        ],
        "exhaustion_comparable_across_split": "yes",
        "exhaustion_counterexample_model": "KPC_GP",
        "exhaustion_explains_split": "no",
        "exhaustion_relations": [
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_B16_GP",
            "relation": "~"
          },
          {
            "pair": "KPC_GP_vs_B_ALL_GP",
            "relation": "~"
          },
          {
            "pair": "mWnt_GP_vs_B16_GP",
            "relation": "<"
          },
          {
            "pair": "mWnt_GP_vs_B_ALL_GP",
            "relation": "<"
          },
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": "<"
          }
        ],
        "exhaustion_separable_program": "yes",
        "formation_limited_by": "absent_induction",
        "generic_signature_relations": [
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": "~"
          },
          {
            "pair": "KPC_GP_vs_B16_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "mWnt_GP_vs_B16_GP",
            "relation": ">"
          },
          {
            "pair": "mWnt_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": ">"
          }
        ],
        "locus_competent_in_naive": "yes",
        "locus_differential": "uniform_across_models",
        "per_model_regulatory_state": [
          {
            "activator_state": "present",
            "blocking_step": "none",
            "brake_state": "released",
            "locus_state": "active",
            "model": "KPC_GP",
            "program": "present"
          },
          {
            "activator_state": "present",
            "blocking_step": "early_release",
            "brake_state": "partial",
            "locus_state": "active",
            "model": "mWnt_GP",
            "program": "partial"
          },
          {
            "activator_state": "absent",
            "blocking_step": "late_activation",
            "brake_state": "released",
            "locus_state": "poised_unfired",
            "model": "B16_GP",
            "program": "absent"
          },
          {
            "activator_state": "absent",
            "blocking_step": "both_steps",
            "brake_state": "partial",
            "locus_state": "poised_unfired",
            "model": "B_ALL_GP",
            "program": "absent"
          }
        ],
        "program_map": [
          {
            "call": "present",
            "model": "KPC_GP"
          },
          {
            "call": "partial",
            "model": "mWnt_GP"
          },
          {
            "call": "absent",
            "model": "B16_GP"
          },
          {
            "call": "absent",
            "model": "B_ALL_GP"
          }
        ],
        "ready_silent_fraction_relations": [
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": "<"
          },
          {
            "pair": "KPC_GP_vs_B16_GP",
            "relation": "<"
          },
          {
            "pair": "KPC_GP_vs_B_ALL_GP",
            "relation": "<"
          },
          {
            "pair": "mWnt_GP_vs_B16_GP",
            "relation": ">"
          },
          {
            "pair": "mWnt_GP_vs_B_ALL_GP",
            "relation": "<"
          },
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": "<"
          }
        ],
        "regulator_tuples": [
          {
            "evidence_basis": "own_abundance",
            "expr_vs_activity": "concordant",
            "gating_step": "early_release",
            "regulator": "Klf2",
            "role": "repressor"
          },
          {
            "evidence_basis": "target_program_behavior",
            "expr_vs_activity": "discordant",
            "gating_step": "late_activation",
            "regulator": "Runx3",
            "role": "activator"
          },
          {
            "evidence_basis": "target_program_behavior",
            "expr_vs_activity": "discordant",
            "gating_step": "late_activation",
            "regulator": "Prdm1",
            "role": "activator"
          },
          {
            "evidence_basis": "target_program_behavior",
            "expr_vs_activity": "discordant",
            "gating_step": "late_activation",
            "regulator": "Bhlhe40",
            "role": "activator"
          }
        ]
      },
      "nTurns": 38,
      "checkpointSignals": {
        "clustering": false,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": false,
        "nerveJointAnchor": false,
        "multiomeAssembly": true,
        "depthNormalizedAtac": true,
        "regulonInference": true,
        "cd8SubsetIntegration": true,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": false,
        "orthologProjection": false,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": true,
        "mousePseudobulkDesign": true,
        "cancerEpithelialDenominator": false,
        "geneLevelDe": true,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "trajectoryFile": "scbench-long/trajectories/green_2_tme_context_dependence__claude-opus-4-7__pi__r1.json",
      "runId": "claude-opus-4-7__pi__green_2_tme_context_dependence__r1",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-opus-4-7-pi",
      "modelName": "claude-opus-4-7",
      "harness": "pi",
      "task": "green_2_tme_context_dependence",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 2,
      "passed": false,
      "score": 7,
      "durationS": 2553.65,
      "cost": null,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "program_map",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "sig_includes_itgae",
            "role": "gate",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Itgae"
            ]
          },
          {
            "name": "declared_a_known_confounder",
            "role": "gate",
            "field": "$.excluded_confounders[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "cf_signature_gut_contaminant",
            "role": "hard_fail",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "generic_signature_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd103_program_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "<"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "per_model_regulatory_state",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "poised_unfired"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "late_activation"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "closing_committed"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_tuples",
            "kind": "list",
            "matchKey": "regulator",
            "k": 1,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "role": {
                    "op": "equals",
                    "values": [
                      "repressor"
                    ]
                  },
                  "gating_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  },
                  "evidence_basis": {
                    "op": "equals",
                    "values": [
                      "target_program_behavior"
                    ]
                  },
                  "expr_vs_activity": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "locus_differential_gate",
            "role": "gate",
            "field": "locus_differential",
            "kind": "value",
            "expected": "uniform_across_models"
          },
          {
            "name": "formation_limited_by_gate",
            "role": "gate",
            "field": "formation_limited_by",
            "kind": "value",
            "expected": "absent_induction"
          },
          {
            "name": "locus_competent_in_naive_gate",
            "role": "gate",
            "field": "locus_competent_in_naive",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "ready_silent_fraction_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 3,
            "expected": [
              {
                "key": "B16_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_KPC_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "activator_chromatin_footprint_gate",
            "role": "gate",
            "field": "activator_chromatin_footprint",
            "kind": "value",
            "expected": "inconsistent_across_formers"
          },
          {
            "name": "exhaustion_comparable_gate",
            "role": "gate",
            "field": "exhaustion_comparable_across_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_nondeterminant_gate",
            "role": "gate",
            "field": "exhaustion_explains_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_separable_program_gate",
            "role": "gate",
            "field": "exhaustion_separable_program",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "exhaustion_counterexample_gate",
            "role": "gate",
            "field": "exhaustion_counterexample_model",
            "kind": "value",
            "expected": "KPC_GP"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "exhaustion_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "cf_inconsistent_present_without_activator",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "program",
            "entity": "present",
            "valueField": "activator_state",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_no_block_but_absent",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "none",
            "valueField": "program",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_late_block_without_poised_locus",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "late_activation",
            "valueField": "locus_state",
            "expected": [
              "closing_committed",
              "active"
            ]
          },
          {
            "name": "cf_inconsistent_early_block_with_brake_released",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "early_release",
            "valueField": "brake_state",
            "expected": [
              "released"
            ]
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: FAIL\n  x list_match: list_match (answer_field='program_map'): FAIL  (score=0.0)\n  + sig_includes_itgae: passed=True, score=1.0\n  x declared_a_known_confounder: passed=False, score=0.0\n  + list_match: list_match (answer_field='generic_signature_relations'): PASS  (score=0.0)\n  x list_match: list_match (answer_field='cd103_program_relations'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='per_model_regulatory_state'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='regulator_tuples'): FAIL  (score=0.0)\n  + locus_differential_gate: passed=True, score=1.0\n  + formation_limited_by_gate: passed=True, score=1.0\n  + locus_competent_in_naive_gate: passed=True, score=1.0\n  x list_match: list_match (answer_field='ready_silent_fraction_relations'): FAIL  (score=0.0)\n  x activator_chromatin_footprint_gate: passed=False, score=0.0\n  x exhaustion_comparable_gate: passed=False, score=0.0\n  + exhaustion_nondeterminant_gate: passed=True, score=1.0\n  + exhaustion_separable_program_gate: passed=True, score=1.0\n  + exhaustion_counterexample_gate: passed=True, score=1.0\n  x list_match: list_match (answer_field='exhaustion_relations'): FAIL  (score=0.0)\n  x hard_fail 'cf_signature_gut_contaminant': TRIGGERED\n  + hard_fail 'cf_inconsistent_present_without_activator': not triggered\n  + hard_fail 'cf_inconsistent_no_block_but_absent': not triggered\n  + hard_fail 'cf_inconsistent_late_block_without_poised_locus': not triggered\n  + hard_fail 'cf_inconsistent_early_block_with_brake_released': not triggered",
      "metrics": {
        "pass_rule": "all",
        "scoring_count": 17,
        "scoring_passed": 8,
        "scoring_total_score": 7,
        "hard_fail_triggered": [
          "cf_signature_gut_contaminant"
        ]
      },
      "fieldScores": {
        "list_match": 0,
        "sig_includes_itgae": 1,
        "declared_a_known_confounder": 0,
        "locus_differential_gate": 1,
        "formation_limited_by_gate": 1,
        "locus_competent_in_naive_gate": 1,
        "activator_chromatin_footprint_gate": 0,
        "exhaustion_comparable_gate": 0,
        "exhaustion_nondeterminant_gate": 1,
        "exhaustion_separable_program_gate": 1,
        "exhaustion_counterexample_gate": 1
      },
      "graderChecks": [
        {
          "name": "program_map",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "sig_includes_itgae",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "declared_a_known_confounder",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "generic_signature_relations",
          "kind": "list_match",
          "passed": true,
          "score": 0
        },
        {
          "name": "cd103_program_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "per_model_regulatory_state",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "regulator_tuples",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "locus_differential_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "formation_limited_by_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "locus_competent_in_naive_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "ready_silent_fraction_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "activator_chromatin_footprint_gate",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "exhaustion_comparable_gate",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "exhaustion_nondeterminant_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "exhaustion_separable_program_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "exhaustion_counterexample_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "exhaustion_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "cf_signature_gut_contaminant",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "cf_inconsistent_present_without_activator",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_no_block_but_absent",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_late_block_without_poised_locus",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_early_block_with_brake_released",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "activator_chromatin_footprint": "consistent_in_formers",
        "cd103_program_relations": [
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_B16_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "mWnt_GP_vs_B16_GP",
            "relation": ">"
          },
          {
            "pair": "mWnt_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": ">"
          }
        ],
        "core_residency_signature": [
          "Itgae",
          "Itga1",
          "Cxcr6",
          "Rgs1",
          "Crtam",
          "Xcl1",
          "Cd160",
          "Litaf",
          "Chn2",
          "Inpp4b",
          "Zfp683"
        ],
        "excluded_confounders": [
          "Pdcd1",
          "Tox",
          "Lag3",
          "Havcr2",
          "Tigit",
          "Mki67",
          "Cd69",
          "Gzmb",
          "Ifng",
          "Tnf"
        ],
        "exhaustion_comparable_across_split": "yes",
        "exhaustion_counterexample_model": "KPC_GP",
        "exhaustion_explains_split": "no",
        "exhaustion_relations": [
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_B16_GP",
            "relation": "~"
          },
          {
            "pair": "KPC_GP_vs_B_ALL_GP",
            "relation": "<"
          },
          {
            "pair": "mWnt_GP_vs_B16_GP",
            "relation": "<"
          },
          {
            "pair": "mWnt_GP_vs_B_ALL_GP",
            "relation": "<"
          },
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": "~"
          }
        ],
        "exhaustion_separable_program": "yes",
        "formation_limited_by": "absent_induction",
        "generic_signature_relations": [
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_B16_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "mWnt_GP_vs_B16_GP",
            "relation": "~"
          },
          {
            "pair": "mWnt_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": ">"
          }
        ],
        "locus_competent_in_naive": "yes",
        "locus_differential": "uniform_across_models",
        "per_model_regulatory_state": [
          {
            "activator_state": "present",
            "blocking_step": "none",
            "brake_state": "released",
            "locus_state": "active",
            "model": "KPC_GP",
            "program": "present"
          },
          {
            "activator_state": "present",
            "blocking_step": "early_release",
            "brake_state": "partial",
            "locus_state": "active",
            "model": "mWnt_GP",
            "program": "partial"
          },
          {
            "activator_state": "partial",
            "blocking_step": "late_activation",
            "brake_state": "released",
            "locus_state": "poised_unfired",
            "model": "B16_GP",
            "program": "absent"
          },
          {
            "activator_state": "partial",
            "blocking_step": "both_steps",
            "brake_state": "engaged",
            "locus_state": "poised_unfired",
            "model": "B_ALL_GP",
            "program": "absent"
          }
        ],
        "program_map": [
          {
            "call": "present",
            "model": "KPC_GP"
          },
          {
            "call": "partial",
            "model": "mWnt_GP"
          },
          {
            "call": "absent",
            "model": "B16_GP"
          },
          {
            "call": "absent",
            "model": "B_ALL_GP"
          }
        ],
        "ready_silent_fraction_relations": [
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": "<"
          },
          {
            "pair": "KPC_GP_vs_B16_GP",
            "relation": "<"
          },
          {
            "pair": "KPC_GP_vs_B_ALL_GP",
            "relation": "<"
          },
          {
            "pair": "mWnt_GP_vs_B16_GP",
            "relation": "<"
          },
          {
            "pair": "mWnt_GP_vs_B_ALL_GP",
            "relation": "<"
          },
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": "~"
          }
        ],
        "regulator_tuples": [
          {
            "evidence_basis": "own_abundance",
            "expr_vs_activity": "concordant",
            "gating_step": "early_release",
            "regulator": "Klf2",
            "role": "repressor"
          },
          {
            "evidence_basis": "own_abundance",
            "expr_vs_activity": "concordant",
            "gating_step": "early_release",
            "regulator": "S1pr1",
            "role": "repressor"
          },
          {
            "evidence_basis": "own_abundance",
            "expr_vs_activity": "concordant",
            "gating_step": "early_release",
            "regulator": "Eomes",
            "role": "repressor"
          },
          {
            "evidence_basis": "target_program_behavior",
            "expr_vs_activity": "discordant",
            "gating_step": "late_activation",
            "regulator": "Runx3",
            "role": "activator"
          },
          {
            "evidence_basis": "target_program_behavior",
            "expr_vs_activity": "discordant",
            "gating_step": "late_activation",
            "regulator": "Prdm1",
            "role": "activator"
          },
          {
            "evidence_basis": "own_abundance",
            "expr_vs_activity": "discordant",
            "gating_step": "late_activation",
            "regulator": "Bhlhe40",
            "role": "activator"
          },
          {
            "evidence_basis": "target_program_behavior",
            "expr_vs_activity": "discordant",
            "gating_step": "late_activation",
            "regulator": "Nr4a2",
            "role": "activator"
          },
          {
            "evidence_basis": "own_abundance",
            "expr_vs_activity": "discordant",
            "gating_step": "indeterminate",
            "regulator": "Tox",
            "role": "dual"
          }
        ]
      },
      "nTurns": 59,
      "checkpointSignals": {
        "clustering": false,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": false,
        "nerveJointAnchor": false,
        "multiomeAssembly": true,
        "depthNormalizedAtac": true,
        "regulonInference": true,
        "cd8SubsetIntegration": true,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": false,
        "orthologProjection": false,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": true,
        "mousePseudobulkDesign": false,
        "cancerEpithelialDenominator": false,
        "geneLevelDe": false,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "trajectoryFile": "scbench-long/trajectories/green_2_tme_context_dependence__claude-opus-4-7__pi__r2.json",
      "runId": "claude-opus-4-7__pi__green_2_tme_context_dependence__r2",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-opus-4-7-pi",
      "modelName": "claude-opus-4-7",
      "harness": "pi",
      "task": "green_2_tme_context_dependence",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 3,
      "passed": false,
      "score": 6,
      "durationS": 2815.9,
      "cost": null,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "program_map",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "sig_includes_itgae",
            "role": "gate",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Itgae"
            ]
          },
          {
            "name": "declared_a_known_confounder",
            "role": "gate",
            "field": "$.excluded_confounders[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "cf_signature_gut_contaminant",
            "role": "hard_fail",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "generic_signature_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd103_program_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "<"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "per_model_regulatory_state",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "poised_unfired"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "late_activation"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "closing_committed"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_tuples",
            "kind": "list",
            "matchKey": "regulator",
            "k": 1,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "role": {
                    "op": "equals",
                    "values": [
                      "repressor"
                    ]
                  },
                  "gating_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  },
                  "evidence_basis": {
                    "op": "equals",
                    "values": [
                      "target_program_behavior"
                    ]
                  },
                  "expr_vs_activity": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "locus_differential_gate",
            "role": "gate",
            "field": "locus_differential",
            "kind": "value",
            "expected": "uniform_across_models"
          },
          {
            "name": "formation_limited_by_gate",
            "role": "gate",
            "field": "formation_limited_by",
            "kind": "value",
            "expected": "absent_induction"
          },
          {
            "name": "locus_competent_in_naive_gate",
            "role": "gate",
            "field": "locus_competent_in_naive",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "ready_silent_fraction_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 3,
            "expected": [
              {
                "key": "B16_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_KPC_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "activator_chromatin_footprint_gate",
            "role": "gate",
            "field": "activator_chromatin_footprint",
            "kind": "value",
            "expected": "inconsistent_across_formers"
          },
          {
            "name": "exhaustion_comparable_gate",
            "role": "gate",
            "field": "exhaustion_comparable_across_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_nondeterminant_gate",
            "role": "gate",
            "field": "exhaustion_explains_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_separable_program_gate",
            "role": "gate",
            "field": "exhaustion_separable_program",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "exhaustion_counterexample_gate",
            "role": "gate",
            "field": "exhaustion_counterexample_model",
            "kind": "value",
            "expected": "KPC_GP"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "exhaustion_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "cf_inconsistent_present_without_activator",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "program",
            "entity": "present",
            "valueField": "activator_state",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_no_block_but_absent",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "none",
            "valueField": "program",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_late_block_without_poised_locus",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "late_activation",
            "valueField": "locus_state",
            "expected": [
              "closing_committed",
              "active"
            ]
          },
          {
            "name": "cf_inconsistent_early_block_with_brake_released",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "early_release",
            "valueField": "brake_state",
            "expected": [
              "released"
            ]
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: FAIL\n  x list_match: list_match (answer_field='program_map'): FAIL  (score=0.0)\n  + sig_includes_itgae: passed=True, score=1.0\n  x declared_a_known_confounder: passed=False, score=0.0\n  x list_match: list_match (answer_field='generic_signature_relations'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='cd103_program_relations'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='per_model_regulatory_state'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='regulator_tuples'): FAIL  (score=0.0)\n  + locus_differential_gate: passed=True, score=1.0\n  + formation_limited_by_gate: passed=True, score=1.0\n  + locus_competent_in_naive_gate: passed=True, score=1.0\n  x list_match: list_match (answer_field='ready_silent_fraction_relations'): FAIL  (score=0.0)\n  x activator_chromatin_footprint_gate: passed=False, score=0.0\n  x exhaustion_comparable_gate: passed=False, score=0.0\n  + exhaustion_nondeterminant_gate: passed=True, score=1.0\n  + exhaustion_separable_program_gate: passed=True, score=1.0\n  x exhaustion_counterexample_gate: passed=False, score=0.0\n  x list_match: list_match (answer_field='exhaustion_relations'): FAIL  (score=0.0)\n  x hard_fail 'cf_signature_gut_contaminant': TRIGGERED\n  + hard_fail 'cf_inconsistent_present_without_activator': not triggered\n  + hard_fail 'cf_inconsistent_no_block_but_absent': not triggered\n  + hard_fail 'cf_inconsistent_late_block_without_poised_locus': not triggered\n  + hard_fail 'cf_inconsistent_early_block_with_brake_released': not triggered",
      "metrics": {
        "pass_rule": "all",
        "scoring_count": 17,
        "scoring_passed": 6,
        "scoring_total_score": 6,
        "hard_fail_triggered": [
          "cf_signature_gut_contaminant"
        ]
      },
      "fieldScores": {
        "list_match": 0,
        "sig_includes_itgae": 1,
        "declared_a_known_confounder": 0,
        "locus_differential_gate": 1,
        "formation_limited_by_gate": 1,
        "locus_competent_in_naive_gate": 1,
        "activator_chromatin_footprint_gate": 0,
        "exhaustion_comparable_gate": 0,
        "exhaustion_nondeterminant_gate": 1,
        "exhaustion_separable_program_gate": 1,
        "exhaustion_counterexample_gate": 0
      },
      "graderChecks": [
        {
          "name": "program_map",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "sig_includes_itgae",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "declared_a_known_confounder",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "generic_signature_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "cd103_program_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "per_model_regulatory_state",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "regulator_tuples",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "locus_differential_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "formation_limited_by_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "locus_competent_in_naive_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "ready_silent_fraction_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "activator_chromatin_footprint_gate",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "exhaustion_comparable_gate",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "exhaustion_nondeterminant_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "exhaustion_separable_program_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "exhaustion_counterexample_gate",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "exhaustion_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "cf_signature_gut_contaminant",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "cf_inconsistent_present_without_activator",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_no_block_but_absent",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_late_block_without_poised_locus",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_early_block_with_brake_released",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "activator_chromatin_footprint": "consistent_in_formers",
        "cd103_program_relations": [
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_B16_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "mWnt_GP_vs_B16_GP",
            "relation": ">"
          },
          {
            "pair": "mWnt_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": ">"
          }
        ],
        "core_residency_signature": [
          "Itgae",
          "Itga1",
          "Cd69",
          "Cxcr6",
          "Rgs1",
          "Rgs2",
          "Crtam",
          "Litaf",
          "Zfp683",
          "Runx3",
          "Prdm1",
          "Bhlhe40",
          "Itgb7",
          "Cd244a",
          "Skil",
          "Xcl1",
          "Cd160",
          "Nr4a2",
          "Inpp4b"
        ],
        "excluded_confounders": [
          "Mki67",
          "Top2a",
          "Birc5",
          "Stmn1",
          "Hmgb2",
          "Sell",
          "Ccr7",
          "Tcf7",
          "Lef1",
          "Klf2",
          "S1pr1",
          "S1pr5",
          "Pdcd1",
          "Lag3",
          "Havcr2",
          "Tigit",
          "Tox",
          "Entpd1",
          "Cd101",
          "Ifng",
          "Gzmb",
          "Prf1",
          "Tnf",
          "Gzma",
          "Gzmk",
          "Nkg7",
          "Eomes",
          "Tbx21",
          "Klrg1"
        ],
        "exhaustion_comparable_across_split": "yes",
        "exhaustion_counterexample_model": "mWnt_GP",
        "exhaustion_explains_split": "no",
        "exhaustion_relations": [
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_B16_GP",
            "relation": "~"
          },
          {
            "pair": "KPC_GP_vs_B_ALL_GP",
            "relation": "~"
          },
          {
            "pair": "mWnt_GP_vs_B16_GP",
            "relation": "<"
          },
          {
            "pair": "mWnt_GP_vs_B_ALL_GP",
            "relation": "<"
          },
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": "~"
          }
        ],
        "exhaustion_separable_program": "yes",
        "formation_limited_by": "absent_induction",
        "generic_signature_relations": [
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_B16_GP",
            "relation": "~"
          },
          {
            "pair": "KPC_GP_vs_B_ALL_GP",
            "relation": "~"
          },
          {
            "pair": "mWnt_GP_vs_B16_GP",
            "relation": "<"
          },
          {
            "pair": "mWnt_GP_vs_B_ALL_GP",
            "relation": "<"
          },
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": "<"
          }
        ],
        "locus_competent_in_naive": "yes",
        "locus_differential": "uniform_across_models",
        "per_model_regulatory_state": [
          {
            "activator_state": "present",
            "blocking_step": "none",
            "brake_state": "released",
            "locus_state": "active",
            "model": "KPC_GP",
            "program": "present"
          },
          {
            "activator_state": "present",
            "blocking_step": "early_release",
            "brake_state": "partial",
            "locus_state": "poised_unfired",
            "model": "mWnt_GP",
            "program": "partial"
          },
          {
            "activator_state": "partial",
            "blocking_step": "late_activation",
            "brake_state": "released",
            "locus_state": "poised_unfired",
            "model": "B16_GP",
            "program": "absent"
          },
          {
            "activator_state": "partial",
            "blocking_step": "both_steps",
            "brake_state": "partial",
            "locus_state": "poised_unfired",
            "model": "B_ALL_GP",
            "program": "absent"
          }
        ],
        "program_map": [
          {
            "call": "present",
            "model": "KPC_GP"
          },
          {
            "call": "partial",
            "model": "mWnt_GP"
          },
          {
            "call": "absent",
            "model": "B16_GP"
          },
          {
            "call": "absent",
            "model": "B_ALL_GP"
          }
        ],
        "ready_silent_fraction_relations": [
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": "<"
          },
          {
            "pair": "KPC_GP_vs_B16_GP",
            "relation": "<"
          },
          {
            "pair": "KPC_GP_vs_B_ALL_GP",
            "relation": "<"
          },
          {
            "pair": "mWnt_GP_vs_B16_GP",
            "relation": ">"
          },
          {
            "pair": "mWnt_GP_vs_B_ALL_GP",
            "relation": "<"
          },
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": "<"
          }
        ],
        "regulator_tuples": [
          {
            "evidence_basis": "own_abundance",
            "expr_vs_activity": "concordant",
            "gating_step": "early_release",
            "regulator": "Klf2",
            "role": "repressor"
          },
          {
            "evidence_basis": "own_abundance",
            "expr_vs_activity": "concordant",
            "gating_step": "early_release",
            "regulator": "Eomes",
            "role": "repressor"
          },
          {
            "evidence_basis": "target_program_behavior",
            "expr_vs_activity": "discordant",
            "gating_step": "late_activation",
            "regulator": "Runx3",
            "role": "activator"
          },
          {
            "evidence_basis": "target_program_behavior",
            "expr_vs_activity": "discordant",
            "gating_step": "late_activation",
            "regulator": "Bhlhe40",
            "role": "activator"
          },
          {
            "evidence_basis": "target_program_behavior",
            "expr_vs_activity": "discordant",
            "gating_step": "late_activation",
            "regulator": "Smad7",
            "role": "repressor"
          },
          {
            "evidence_basis": "target_program_behavior",
            "expr_vs_activity": "discordant",
            "gating_step": "late_activation",
            "regulator": "Skil",
            "role": "repressor"
          },
          {
            "evidence_basis": "target_program_behavior",
            "expr_vs_activity": "discordant",
            "gating_step": "late_activation",
            "regulator": "Tox",
            "role": "dual"
          },
          {
            "evidence_basis": "own_abundance",
            "expr_vs_activity": "indeterminate",
            "gating_step": "late_activation",
            "regulator": "Prdm1",
            "role": "activator"
          },
          {
            "evidence_basis": "own_abundance",
            "expr_vs_activity": "indeterminate",
            "gating_step": "late_activation",
            "regulator": "Zfp683",
            "role": "activator"
          }
        ]
      },
      "nTurns": 56,
      "checkpointSignals": {
        "clustering": false,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": false,
        "nerveJointAnchor": false,
        "multiomeAssembly": true,
        "depthNormalizedAtac": true,
        "regulonInference": true,
        "cd8SubsetIntegration": true,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": false,
        "orthologProjection": false,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": false,
        "mousePseudobulkDesign": false,
        "cancerEpithelialDenominator": false,
        "geneLevelDe": true,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "trajectoryFile": "scbench-long/trajectories/green_2_tme_context_dependence__claude-opus-4-7__pi__r3.json",
      "runId": "claude-opus-4-7__pi__green_2_tme_context_dependence__r3",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-opus-4-8-claude-code",
      "modelName": "claude-opus-4-8",
      "harness": "claude-code",
      "task": "green_2_tme_context_dependence",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 1,
      "passed": false,
      "score": 6,
      "durationS": 2742.96,
      "cost": 9.06080775,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "program_map",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "sig_includes_itgae",
            "role": "gate",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Itgae"
            ]
          },
          {
            "name": "declared_a_known_confounder",
            "role": "gate",
            "field": "$.excluded_confounders[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "cf_signature_gut_contaminant",
            "role": "hard_fail",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "generic_signature_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd103_program_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "<"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "per_model_regulatory_state",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "poised_unfired"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "late_activation"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "closing_committed"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_tuples",
            "kind": "list",
            "matchKey": "regulator",
            "k": 1,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "role": {
                    "op": "equals",
                    "values": [
                      "repressor"
                    ]
                  },
                  "gating_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  },
                  "evidence_basis": {
                    "op": "equals",
                    "values": [
                      "target_program_behavior"
                    ]
                  },
                  "expr_vs_activity": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "locus_differential_gate",
            "role": "gate",
            "field": "locus_differential",
            "kind": "value",
            "expected": "uniform_across_models"
          },
          {
            "name": "formation_limited_by_gate",
            "role": "gate",
            "field": "formation_limited_by",
            "kind": "value",
            "expected": "absent_induction"
          },
          {
            "name": "locus_competent_in_naive_gate",
            "role": "gate",
            "field": "locus_competent_in_naive",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "ready_silent_fraction_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 3,
            "expected": [
              {
                "key": "B16_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_KPC_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "activator_chromatin_footprint_gate",
            "role": "gate",
            "field": "activator_chromatin_footprint",
            "kind": "value",
            "expected": "inconsistent_across_formers"
          },
          {
            "name": "exhaustion_comparable_gate",
            "role": "gate",
            "field": "exhaustion_comparable_across_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_nondeterminant_gate",
            "role": "gate",
            "field": "exhaustion_explains_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_separable_program_gate",
            "role": "gate",
            "field": "exhaustion_separable_program",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "exhaustion_counterexample_gate",
            "role": "gate",
            "field": "exhaustion_counterexample_model",
            "kind": "value",
            "expected": "KPC_GP"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "exhaustion_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "cf_inconsistent_present_without_activator",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "program",
            "entity": "present",
            "valueField": "activator_state",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_no_block_but_absent",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "none",
            "valueField": "program",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_late_block_without_poised_locus",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "late_activation",
            "valueField": "locus_state",
            "expected": [
              "closing_committed",
              "active"
            ]
          },
          {
            "name": "cf_inconsistent_early_block_with_brake_released",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "early_release",
            "valueField": "brake_state",
            "expected": [
              "released"
            ]
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: FAIL\n  x list_match: list_match (answer_field='program_map'): FAIL  (score=0.0)\n  + sig_includes_itgae: passed=True, score=1.0\n  + declared_a_known_confounder: passed=True, score=1.0\n  x list_match: list_match (answer_field='generic_signature_relations'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='cd103_program_relations'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='per_model_regulatory_state'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='regulator_tuples'): FAIL  (score=0.0)\n  x locus_differential_gate: passed=False, score=0.0\n  + formation_limited_by_gate: passed=True, score=1.0\n  + locus_competent_in_naive_gate: passed=True, score=1.0\n  x list_match: list_match (answer_field='ready_silent_fraction_relations'): FAIL  (score=0.0)\n  x activator_chromatin_footprint_gate: passed=False, score=0.0\n  x exhaustion_comparable_gate: passed=False, score=0.0\n  + exhaustion_nondeterminant_gate: passed=True, score=1.0\n  + exhaustion_separable_program_gate: passed=True, score=1.0\n  x exhaustion_counterexample_gate: passed=False, score=0.0\n  x list_match: list_match (answer_field='exhaustion_relations'): FAIL  (score=0.0)\n  x hard_fail 'cf_signature_gut_contaminant': TRIGGERED\n  + hard_fail 'cf_inconsistent_present_without_activator': not triggered\n  + hard_fail 'cf_inconsistent_no_block_but_absent': not triggered\n  x hard_fail 'cf_inconsistent_late_block_without_poised_locus': TRIGGERED\n  + hard_fail 'cf_inconsistent_early_block_with_brake_released': not triggered",
      "metrics": {
        "pass_rule": "all",
        "scoring_count": 17,
        "scoring_passed": 6,
        "scoring_total_score": 6,
        "hard_fail_triggered": [
          "cf_signature_gut_contaminant",
          "cf_inconsistent_late_block_without_poised_locus"
        ]
      },
      "fieldScores": {
        "list_match": 0,
        "sig_includes_itgae": 1,
        "declared_a_known_confounder": 1,
        "locus_differential_gate": 0,
        "formation_limited_by_gate": 1,
        "locus_competent_in_naive_gate": 1,
        "activator_chromatin_footprint_gate": 0,
        "exhaustion_comparable_gate": 0,
        "exhaustion_nondeterminant_gate": 1,
        "exhaustion_separable_program_gate": 1,
        "exhaustion_counterexample_gate": 0
      },
      "graderChecks": [
        {
          "name": "program_map",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "sig_includes_itgae",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "declared_a_known_confounder",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "generic_signature_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "cd103_program_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "per_model_regulatory_state",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "regulator_tuples",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "locus_differential_gate",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "formation_limited_by_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "locus_competent_in_naive_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "ready_silent_fraction_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "activator_chromatin_footprint_gate",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "exhaustion_comparable_gate",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "exhaustion_nondeterminant_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "exhaustion_separable_program_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "exhaustion_counterexample_gate",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "exhaustion_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "cf_signature_gut_contaminant",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "cf_inconsistent_present_without_activator",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_no_block_but_absent",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_late_block_without_poised_locus",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "cf_inconsistent_early_block_with_brake_released",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "activator_chromatin_footprint": "consistent_in_formers",
        "cd103_program_relations": [
          {
            "pair": "B16_GP_vs_KPC_GP",
            "relation": "<"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": "<"
          },
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": "~"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "mWnt_GP_vs_B_ALL_GP",
            "relation": ">"
          }
        ],
        "core_residency_signature": [
          "Itgae",
          "Itga1",
          "Cd69",
          "Cxcr6",
          "Rgs1",
          "Rgs2",
          "Zfp683",
          "Crtam",
          "Chn2",
          "Litaf",
          "Skil",
          "Qpct",
          "Cd101",
          "Xcl1"
        ],
        "excluded_confounders": [
          "Tox",
          "Pdcd1",
          "Havcr2",
          "Lag3",
          "Tigit",
          "Entpd1",
          "Ctla4",
          "Mki67",
          "Gzmb",
          "Ccr9"
        ],
        "exhaustion_comparable_across_split": "yes",
        "exhaustion_counterexample_model": "B16_GP",
        "exhaustion_explains_split": "no",
        "exhaustion_relations": [
          {
            "pair": "B16_GP_vs_KPC_GP",
            "relation": "~"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": "~"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_B_ALL_GP",
            "relation": "~"
          },
          {
            "pair": "mWnt_GP_vs_B_ALL_GP",
            "relation": "<"
          }
        ],
        "exhaustion_separable_program": "yes",
        "formation_limited_by": "absent_induction",
        "generic_signature_relations": [
          {
            "pair": "B16_GP_vs_KPC_GP",
            "relation": "<"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": "~"
          },
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "mWnt_GP_vs_B_ALL_GP",
            "relation": ">"
          }
        ],
        "locus_competent_in_naive": "yes",
        "locus_differential": "model_specific_element",
        "per_model_regulatory_state": [
          {
            "activator_state": "absent",
            "blocking_step": "late_activation",
            "brake_state": "released",
            "locus_state": "closing_committed",
            "model": "B16_GP",
            "program": "absent"
          },
          {
            "activator_state": "present",
            "blocking_step": "none",
            "brake_state": "released",
            "locus_state": "active",
            "model": "KPC_GP",
            "program": "present"
          },
          {
            "activator_state": "present",
            "blocking_step": "early_release",
            "brake_state": "partial",
            "locus_state": "active",
            "model": "mWnt_GP",
            "program": "partial"
          },
          {
            "activator_state": "absent",
            "blocking_step": "both_steps",
            "brake_state": "partial",
            "locus_state": "closing_committed",
            "model": "B_ALL_GP",
            "program": "absent"
          }
        ],
        "program_map": [
          {
            "call": "absent",
            "model": "B16_GP"
          },
          {
            "call": "present",
            "model": "KPC_GP"
          },
          {
            "call": "partial",
            "model": "mWnt_GP"
          },
          {
            "call": "absent",
            "model": "B_ALL_GP"
          }
        ],
        "ready_silent_fraction_relations": [
          {
            "pair": "B16_GP_vs_KPC_GP",
            "relation": "~"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": "<"
          },
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": "<"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": "<"
          },
          {
            "pair": "KPC_GP_vs_B_ALL_GP",
            "relation": "<"
          },
          {
            "pair": "mWnt_GP_vs_B_ALL_GP",
            "relation": "~"
          }
        ],
        "regulator_tuples": [
          {
            "evidence_basis": "own_abundance",
            "expr_vs_activity": "concordant",
            "gating_step": "early_release",
            "regulator": "Klf2",
            "role": "repressor"
          },
          {
            "evidence_basis": "own_abundance",
            "expr_vs_activity": "concordant",
            "gating_step": "early_release",
            "regulator": "S1pr1",
            "role": "repressor"
          },
          {
            "evidence_basis": "target_program_behavior",
            "expr_vs_activity": "discordant",
            "gating_step": "late_activation",
            "regulator": "Runx3",
            "role": "activator"
          },
          {
            "evidence_basis": "target_program_behavior",
            "expr_vs_activity": "discordant",
            "gating_step": "late_activation",
            "regulator": "Zfp683",
            "role": "activator"
          }
        ]
      },
      "nTurns": 56,
      "checkpointSignals": {
        "clustering": false,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": true,
        "nerveJointAnchor": false,
        "multiomeAssembly": true,
        "depthNormalizedAtac": true,
        "regulonInference": true,
        "cd8SubsetIntegration": true,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": false,
        "orthologProjection": false,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": true,
        "enrichmentFamilyRanking": false,
        "mousePseudobulkDesign": true,
        "cancerEpithelialDenominator": false,
        "geneLevelDe": true,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "trajectoryFile": "scbench-long/trajectories/green_2_tme_context_dependence__claude-opus-4-8__claude-code__r1.json",
      "runId": "claude-opus-4-8__claude-code__green_2_tme_context_dependence__r1",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-opus-4-8-claude-code",
      "modelName": "claude-opus-4-8",
      "harness": "claude-code",
      "task": "green_2_tme_context_dependence",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 2,
      "passed": false,
      "score": 6,
      "durationS": 2944.85,
      "cost": 10.81325675,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "program_map",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "sig_includes_itgae",
            "role": "gate",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Itgae"
            ]
          },
          {
            "name": "declared_a_known_confounder",
            "role": "gate",
            "field": "$.excluded_confounders[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "cf_signature_gut_contaminant",
            "role": "hard_fail",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "generic_signature_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd103_program_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "<"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "per_model_regulatory_state",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "poised_unfired"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "late_activation"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "closing_committed"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_tuples",
            "kind": "list",
            "matchKey": "regulator",
            "k": 1,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "role": {
                    "op": "equals",
                    "values": [
                      "repressor"
                    ]
                  },
                  "gating_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  },
                  "evidence_basis": {
                    "op": "equals",
                    "values": [
                      "target_program_behavior"
                    ]
                  },
                  "expr_vs_activity": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "locus_differential_gate",
            "role": "gate",
            "field": "locus_differential",
            "kind": "value",
            "expected": "uniform_across_models"
          },
          {
            "name": "formation_limited_by_gate",
            "role": "gate",
            "field": "formation_limited_by",
            "kind": "value",
            "expected": "absent_induction"
          },
          {
            "name": "locus_competent_in_naive_gate",
            "role": "gate",
            "field": "locus_competent_in_naive",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "ready_silent_fraction_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 3,
            "expected": [
              {
                "key": "B16_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_KPC_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "activator_chromatin_footprint_gate",
            "role": "gate",
            "field": "activator_chromatin_footprint",
            "kind": "value",
            "expected": "inconsistent_across_formers"
          },
          {
            "name": "exhaustion_comparable_gate",
            "role": "gate",
            "field": "exhaustion_comparable_across_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_nondeterminant_gate",
            "role": "gate",
            "field": "exhaustion_explains_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_separable_program_gate",
            "role": "gate",
            "field": "exhaustion_separable_program",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "exhaustion_counterexample_gate",
            "role": "gate",
            "field": "exhaustion_counterexample_model",
            "kind": "value",
            "expected": "KPC_GP"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "exhaustion_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "cf_inconsistent_present_without_activator",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "program",
            "entity": "present",
            "valueField": "activator_state",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_no_block_but_absent",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "none",
            "valueField": "program",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_late_block_without_poised_locus",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "late_activation",
            "valueField": "locus_state",
            "expected": [
              "closing_committed",
              "active"
            ]
          },
          {
            "name": "cf_inconsistent_early_block_with_brake_released",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "early_release",
            "valueField": "brake_state",
            "expected": [
              "released"
            ]
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: FAIL\n  x list_match: list_match (answer_field='program_map'): FAIL  (score=0.0)\n  + sig_includes_itgae: passed=True, score=1.0\n  x declared_a_known_confounder: passed=False, score=0.0\n  + list_match: list_match (answer_field='generic_signature_relations'): PASS  (score=0.0)\n  x list_match: list_match (answer_field='cd103_program_relations'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='per_model_regulatory_state'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='regulator_tuples'): FAIL  (score=0.0)\n  + locus_differential_gate: passed=True, score=1.0\n  + formation_limited_by_gate: passed=True, score=1.0\n  + locus_competent_in_naive_gate: passed=True, score=1.0\n  x list_match: list_match (answer_field='ready_silent_fraction_relations'): FAIL  (score=0.0)\n  x activator_chromatin_footprint_gate: passed=False, score=0.0\n  x exhaustion_comparable_gate: passed=False, score=0.0\n  + exhaustion_nondeterminant_gate: passed=True, score=1.0\n  + exhaustion_separable_program_gate: passed=True, score=1.0\n  x exhaustion_counterexample_gate: passed=False, score=0.0\n  x list_match: list_match (answer_field='exhaustion_relations'): FAIL  (score=0.0)\n  x hard_fail 'cf_signature_gut_contaminant': TRIGGERED\n  + hard_fail 'cf_inconsistent_present_without_activator': not triggered\n  + hard_fail 'cf_inconsistent_no_block_but_absent': not triggered\n  + hard_fail 'cf_inconsistent_late_block_without_poised_locus': not triggered\n  + hard_fail 'cf_inconsistent_early_block_with_brake_released': not triggered",
      "metrics": {
        "pass_rule": "all",
        "scoring_count": 17,
        "scoring_passed": 7,
        "scoring_total_score": 6,
        "hard_fail_triggered": [
          "cf_signature_gut_contaminant"
        ]
      },
      "fieldScores": {
        "list_match": 0,
        "sig_includes_itgae": 1,
        "declared_a_known_confounder": 0,
        "locus_differential_gate": 1,
        "formation_limited_by_gate": 1,
        "locus_competent_in_naive_gate": 1,
        "activator_chromatin_footprint_gate": 0,
        "exhaustion_comparable_gate": 0,
        "exhaustion_nondeterminant_gate": 1,
        "exhaustion_separable_program_gate": 1,
        "exhaustion_counterexample_gate": 0
      },
      "graderChecks": [
        {
          "name": "program_map",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "sig_includes_itgae",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "declared_a_known_confounder",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "generic_signature_relations",
          "kind": "list_match",
          "passed": true,
          "score": 0
        },
        {
          "name": "cd103_program_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "per_model_regulatory_state",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "regulator_tuples",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "locus_differential_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "formation_limited_by_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "locus_competent_in_naive_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "ready_silent_fraction_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "activator_chromatin_footprint_gate",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "exhaustion_comparable_gate",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "exhaustion_nondeterminant_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "exhaustion_separable_program_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "exhaustion_counterexample_gate",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "exhaustion_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "cf_signature_gut_contaminant",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "cf_inconsistent_present_without_activator",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_no_block_but_absent",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_late_block_without_poised_locus",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_early_block_with_brake_released",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "activator_chromatin_footprint": "consistent_in_formers",
        "cd103_program_relations": [
          {
            "pair": "KPC_GP_vs_B16_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": "<"
          },
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": "~"
          },
          {
            "pair": "mWnt_GP_vs_B_ALL_GP",
            "relation": ">"
          }
        ],
        "core_residency_signature": [
          "Itgae",
          "Cd69",
          "Itga1",
          "Cxcr6",
          "Rgs1",
          "Chn2",
          "Cd101",
          "Inpp4b",
          "Skil",
          "Pmepa1",
          "Xcl1"
        ],
        "excluded_confounders": [
          "Pdcd1",
          "Havcr2",
          "Lag3",
          "Tox",
          "Entpd1",
          "Tigit"
        ],
        "exhaustion_comparable_across_split": "yes",
        "exhaustion_counterexample_model": "mWnt_GP",
        "exhaustion_explains_split": "no",
        "exhaustion_relations": [
          {
            "pair": "KPC_GP_vs_B16_GP",
            "relation": "~"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_B_ALL_GP",
            "relation": "~"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": "~"
          },
          {
            "pair": "mWnt_GP_vs_B_ALL_GP",
            "relation": "<"
          }
        ],
        "exhaustion_separable_program": "yes",
        "formation_limited_by": "absent_induction",
        "generic_signature_relations": [
          {
            "pair": "KPC_GP_vs_B16_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": "~"
          },
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "mWnt_GP_vs_B_ALL_GP",
            "relation": ">"
          }
        ],
        "locus_competent_in_naive": "yes",
        "locus_differential": "uniform_across_models",
        "per_model_regulatory_state": [
          {
            "activator_state": "present",
            "blocking_step": "none",
            "brake_state": "released",
            "locus_state": "active",
            "model": "KPC_GP",
            "program": "present"
          },
          {
            "activator_state": "partial",
            "blocking_step": "early_release",
            "brake_state": "partial",
            "locus_state": "active",
            "model": "mWnt_GP",
            "program": "partial"
          },
          {
            "activator_state": "absent",
            "blocking_step": "late_activation",
            "brake_state": "released",
            "locus_state": "poised_unfired",
            "model": "B16_GP",
            "program": "absent"
          },
          {
            "activator_state": "absent",
            "blocking_step": "both_steps",
            "brake_state": "partial",
            "locus_state": "poised_unfired",
            "model": "B_ALL_GP",
            "program": "absent"
          }
        ],
        "program_map": [
          {
            "call": "present",
            "model": "KPC_GP"
          },
          {
            "call": "partial",
            "model": "mWnt_GP"
          },
          {
            "call": "absent",
            "model": "B16_GP"
          },
          {
            "call": "absent",
            "model": "B_ALL_GP"
          }
        ],
        "ready_silent_fraction_relations": [
          {
            "pair": "KPC_GP_vs_B16_GP",
            "relation": "<"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": "~"
          },
          {
            "pair": "KPC_GP_vs_B_ALL_GP",
            "relation": "~"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "mWnt_GP_vs_B_ALL_GP",
            "relation": "~"
          }
        ],
        "regulator_tuples": [
          {
            "evidence_basis": "target_program_behavior",
            "expr_vs_activity": "discordant",
            "gating_step": "late_activation",
            "regulator": "Runx3",
            "role": "activator"
          },
          {
            "evidence_basis": "own_abundance",
            "expr_vs_activity": "concordant",
            "gating_step": "early_release",
            "regulator": "Klf2",
            "role": "repressor"
          }
        ]
      },
      "nTurns": 65,
      "checkpointSignals": {
        "clustering": false,
        "markerAnnotation": true,
        "scoreBasedAnnotation": true,
        "normalization": true,
        "nerveJointAnchor": false,
        "multiomeAssembly": true,
        "depthNormalizedAtac": true,
        "regulonInference": true,
        "cd8SubsetIntegration": true,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": false,
        "orthologProjection": false,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": true,
        "enrichmentFamilyRanking": false,
        "mousePseudobulkDesign": true,
        "cancerEpithelialDenominator": false,
        "geneLevelDe": true,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "trajectoryFile": "scbench-long/trajectories/green_2_tme_context_dependence__claude-opus-4-8__claude-code__r2.json",
      "runId": "claude-opus-4-8__claude-code__green_2_tme_context_dependence__r2",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-opus-4-8-claude-code",
      "modelName": "claude-opus-4-8",
      "harness": "claude-code",
      "task": "green_2_tme_context_dependence",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 3,
      "passed": false,
      "score": 6,
      "durationS": 2044.61,
      "cost": 7.271492000000001,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "program_map",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "sig_includes_itgae",
            "role": "gate",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Itgae"
            ]
          },
          {
            "name": "declared_a_known_confounder",
            "role": "gate",
            "field": "$.excluded_confounders[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "cf_signature_gut_contaminant",
            "role": "hard_fail",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "generic_signature_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd103_program_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "<"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "per_model_regulatory_state",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "poised_unfired"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "late_activation"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "closing_committed"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_tuples",
            "kind": "list",
            "matchKey": "regulator",
            "k": 1,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "role": {
                    "op": "equals",
                    "values": [
                      "repressor"
                    ]
                  },
                  "gating_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  },
                  "evidence_basis": {
                    "op": "equals",
                    "values": [
                      "target_program_behavior"
                    ]
                  },
                  "expr_vs_activity": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "locus_differential_gate",
            "role": "gate",
            "field": "locus_differential",
            "kind": "value",
            "expected": "uniform_across_models"
          },
          {
            "name": "formation_limited_by_gate",
            "role": "gate",
            "field": "formation_limited_by",
            "kind": "value",
            "expected": "absent_induction"
          },
          {
            "name": "locus_competent_in_naive_gate",
            "role": "gate",
            "field": "locus_competent_in_naive",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "ready_silent_fraction_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 3,
            "expected": [
              {
                "key": "B16_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_KPC_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "activator_chromatin_footprint_gate",
            "role": "gate",
            "field": "activator_chromatin_footprint",
            "kind": "value",
            "expected": "inconsistent_across_formers"
          },
          {
            "name": "exhaustion_comparable_gate",
            "role": "gate",
            "field": "exhaustion_comparable_across_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_nondeterminant_gate",
            "role": "gate",
            "field": "exhaustion_explains_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_separable_program_gate",
            "role": "gate",
            "field": "exhaustion_separable_program",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "exhaustion_counterexample_gate",
            "role": "gate",
            "field": "exhaustion_counterexample_model",
            "kind": "value",
            "expected": "KPC_GP"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "exhaustion_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "cf_inconsistent_present_without_activator",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "program",
            "entity": "present",
            "valueField": "activator_state",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_no_block_but_absent",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "none",
            "valueField": "program",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_late_block_without_poised_locus",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "late_activation",
            "valueField": "locus_state",
            "expected": [
              "closing_committed",
              "active"
            ]
          },
          {
            "name": "cf_inconsistent_early_block_with_brake_released",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "early_release",
            "valueField": "brake_state",
            "expected": [
              "released"
            ]
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: FAIL\n  x list_match: list_match (answer_field='program_map'): FAIL  (score=0.0)\n  + sig_includes_itgae: passed=True, score=1.0\n  + declared_a_known_confounder: passed=True, score=1.0\n  x list_match: list_match (answer_field='generic_signature_relations'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='cd103_program_relations'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='per_model_regulatory_state'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='regulator_tuples'): FAIL  (score=0.0)\n  x locus_differential_gate: passed=False, score=0.0\n  + formation_limited_by_gate: passed=True, score=1.0\n  + locus_competent_in_naive_gate: passed=True, score=1.0\n  x list_match: list_match (answer_field='ready_silent_fraction_relations'): FAIL  (score=0.0)\n  x activator_chromatin_footprint_gate: passed=False, score=0.0\n  x exhaustion_comparable_gate: passed=False, score=0.0\n  + exhaustion_nondeterminant_gate: passed=True, score=1.0\n  + exhaustion_separable_program_gate: passed=True, score=1.0\n  x exhaustion_counterexample_gate: passed=False, score=0.0\n  x list_match: list_match (answer_field='exhaustion_relations'): FAIL  (score=0.0)\n  x hard_fail 'cf_signature_gut_contaminant': TRIGGERED\n  + hard_fail 'cf_inconsistent_present_without_activator': not triggered\n  + hard_fail 'cf_inconsistent_no_block_but_absent': not triggered\n  x hard_fail 'cf_inconsistent_late_block_without_poised_locus': TRIGGERED\n  + hard_fail 'cf_inconsistent_early_block_with_brake_released': not triggered",
      "metrics": {
        "pass_rule": "all",
        "scoring_count": 17,
        "scoring_passed": 6,
        "scoring_total_score": 6,
        "hard_fail_triggered": [
          "cf_signature_gut_contaminant",
          "cf_inconsistent_late_block_without_poised_locus"
        ]
      },
      "fieldScores": {
        "list_match": 0,
        "sig_includes_itgae": 1,
        "declared_a_known_confounder": 1,
        "locus_differential_gate": 0,
        "formation_limited_by_gate": 1,
        "locus_competent_in_naive_gate": 1,
        "activator_chromatin_footprint_gate": 0,
        "exhaustion_comparable_gate": 0,
        "exhaustion_nondeterminant_gate": 1,
        "exhaustion_separable_program_gate": 1,
        "exhaustion_counterexample_gate": 0
      },
      "graderChecks": [
        {
          "name": "program_map",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "sig_includes_itgae",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "declared_a_known_confounder",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "generic_signature_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "cd103_program_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "per_model_regulatory_state",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "regulator_tuples",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "locus_differential_gate",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "formation_limited_by_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "locus_competent_in_naive_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "ready_silent_fraction_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "activator_chromatin_footprint_gate",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "exhaustion_comparable_gate",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "exhaustion_nondeterminant_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "exhaustion_separable_program_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "exhaustion_counterexample_gate",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "exhaustion_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "cf_signature_gut_contaminant",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "cf_inconsistent_present_without_activator",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_no_block_but_absent",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_late_block_without_poised_locus",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "cf_inconsistent_early_block_with_brake_released",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "activator_chromatin_footprint": "consistent_in_formers",
        "cd103_program_relations": [
          {
            "pair": "B16_GP_vs_KPC_GP",
            "relation": "<"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": "<"
          },
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": "~"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "mWnt_GP_vs_B_ALL_GP",
            "relation": ">"
          }
        ],
        "core_residency_signature": [
          "Itgae",
          "Itga1",
          "Cdh1",
          "Chn2",
          "Cd244a",
          "Rgs1",
          "Skil"
        ],
        "excluded_confounders": [
          "Cxcr6",
          "Cd69",
          "Nr4a2",
          "Rgs2",
          "Lag3",
          "Ccr9",
          "Tox",
          "Pdcd1",
          "Havcr2",
          "Gzmb"
        ],
        "exhaustion_comparable_across_split": "yes",
        "exhaustion_counterexample_model": "mWnt_GP",
        "exhaustion_explains_split": "no",
        "exhaustion_relations": [
          {
            "pair": "B16_GP_vs_KPC_GP",
            "relation": "~"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": "~"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "mWnt_GP_vs_B_ALL_GP",
            "relation": "<"
          }
        ],
        "exhaustion_separable_program": "yes",
        "formation_limited_by": "absent_induction",
        "generic_signature_relations": [
          {
            "pair": "B16_GP_vs_KPC_GP",
            "relation": "~"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": "~"
          },
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": "~"
          },
          {
            "pair": "KPC_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "mWnt_GP_vs_B_ALL_GP",
            "relation": ">"
          }
        ],
        "locus_competent_in_naive": "yes",
        "locus_differential": "model_specific_element",
        "per_model_regulatory_state": [
          {
            "activator_state": "present",
            "blocking_step": "none",
            "brake_state": "released",
            "locus_state": "active",
            "model": "KPC_GP",
            "program": "present"
          },
          {
            "activator_state": "absent",
            "blocking_step": "late_activation",
            "brake_state": "released",
            "locus_state": "closing_committed",
            "model": "B16_GP",
            "program": "absent"
          },
          {
            "activator_state": "partial",
            "blocking_step": "early_release",
            "brake_state": "partial",
            "locus_state": "poised_unfired",
            "model": "mWnt_GP",
            "program": "partial"
          },
          {
            "activator_state": "absent",
            "blocking_step": "both_steps",
            "brake_state": "partial",
            "locus_state": "repressed",
            "model": "B_ALL_GP",
            "program": "absent"
          }
        ],
        "program_map": [
          {
            "call": "absent",
            "model": "B16_GP"
          },
          {
            "call": "present",
            "model": "KPC_GP"
          },
          {
            "call": "partial",
            "model": "mWnt_GP"
          },
          {
            "call": "absent",
            "model": "B_ALL_GP"
          }
        ],
        "ready_silent_fraction_relations": [
          {
            "pair": "B16_GP_vs_KPC_GP",
            "relation": ">"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": "<"
          },
          {
            "pair": "KPC_GP_vs_B_ALL_GP",
            "relation": "<"
          },
          {
            "pair": "mWnt_GP_vs_B_ALL_GP",
            "relation": "<"
          }
        ],
        "regulator_tuples": [
          {
            "evidence_basis": "target_program_behavior",
            "expr_vs_activity": "discordant",
            "gating_step": "late_activation",
            "regulator": "Runx3",
            "role": "activator"
          },
          {
            "evidence_basis": "own_abundance",
            "expr_vs_activity": "concordant",
            "gating_step": "early_release",
            "regulator": "Tcf7",
            "role": "repressor"
          },
          {
            "evidence_basis": "own_abundance",
            "expr_vs_activity": "concordant",
            "gating_step": "early_release",
            "regulator": "Klf2",
            "role": "repressor"
          }
        ]
      },
      "nTurns": 49,
      "checkpointSignals": {
        "clustering": false,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": false,
        "nerveJointAnchor": false,
        "multiomeAssembly": true,
        "depthNormalizedAtac": false,
        "regulonInference": true,
        "cd8SubsetIntegration": true,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": false,
        "orthologProjection": false,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": false,
        "mousePseudobulkDesign": true,
        "cancerEpithelialDenominator": false,
        "geneLevelDe": true,
        "agingSignatureControl": true,
        "senescencePanelScoring": false
      },
      "trajectoryFile": "scbench-long/trajectories/green_2_tme_context_dependence__claude-opus-4-8__claude-code__r3.json",
      "runId": "claude-opus-4-8__claude-code__green_2_tme_context_dependence__r3",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-opus-4-8-pi",
      "modelName": "claude-opus-4-8",
      "harness": "pi",
      "task": "green_2_tme_context_dependence",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 1,
      "passed": false,
      "score": 7,
      "durationS": 2844.18,
      "cost": 5.998139500000001,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "program_map",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "sig_includes_itgae",
            "role": "gate",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Itgae"
            ]
          },
          {
            "name": "declared_a_known_confounder",
            "role": "gate",
            "field": "$.excluded_confounders[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "cf_signature_gut_contaminant",
            "role": "hard_fail",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "generic_signature_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd103_program_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "<"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "per_model_regulatory_state",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "poised_unfired"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "late_activation"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "closing_committed"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_tuples",
            "kind": "list",
            "matchKey": "regulator",
            "k": 1,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "role": {
                    "op": "equals",
                    "values": [
                      "repressor"
                    ]
                  },
                  "gating_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  },
                  "evidence_basis": {
                    "op": "equals",
                    "values": [
                      "target_program_behavior"
                    ]
                  },
                  "expr_vs_activity": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "locus_differential_gate",
            "role": "gate",
            "field": "locus_differential",
            "kind": "value",
            "expected": "uniform_across_models"
          },
          {
            "name": "formation_limited_by_gate",
            "role": "gate",
            "field": "formation_limited_by",
            "kind": "value",
            "expected": "absent_induction"
          },
          {
            "name": "locus_competent_in_naive_gate",
            "role": "gate",
            "field": "locus_competent_in_naive",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "ready_silent_fraction_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 3,
            "expected": [
              {
                "key": "B16_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_KPC_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "activator_chromatin_footprint_gate",
            "role": "gate",
            "field": "activator_chromatin_footprint",
            "kind": "value",
            "expected": "inconsistent_across_formers"
          },
          {
            "name": "exhaustion_comparable_gate",
            "role": "gate",
            "field": "exhaustion_comparable_across_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_nondeterminant_gate",
            "role": "gate",
            "field": "exhaustion_explains_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_separable_program_gate",
            "role": "gate",
            "field": "exhaustion_separable_program",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "exhaustion_counterexample_gate",
            "role": "gate",
            "field": "exhaustion_counterexample_model",
            "kind": "value",
            "expected": "KPC_GP"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "exhaustion_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "cf_inconsistent_present_without_activator",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "program",
            "entity": "present",
            "valueField": "activator_state",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_no_block_but_absent",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "none",
            "valueField": "program",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_late_block_without_poised_locus",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "late_activation",
            "valueField": "locus_state",
            "expected": [
              "closing_committed",
              "active"
            ]
          },
          {
            "name": "cf_inconsistent_early_block_with_brake_released",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "early_release",
            "valueField": "brake_state",
            "expected": [
              "released"
            ]
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: FAIL\n  x list_match: list_match (answer_field='program_map'): FAIL  (score=0.0)\n  + sig_includes_itgae: passed=True, score=1.0\n  x declared_a_known_confounder: passed=False, score=0.0\n  x list_match: list_match (answer_field='generic_signature_relations'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='cd103_program_relations'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='per_model_regulatory_state'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='regulator_tuples'): FAIL  (score=0.0)\n  + locus_differential_gate: passed=True, score=1.0\n  + formation_limited_by_gate: passed=True, score=1.0\n  + locus_competent_in_naive_gate: passed=True, score=1.0\n  x list_match: list_match (answer_field='ready_silent_fraction_relations'): FAIL  (score=0.0)\n  x activator_chromatin_footprint_gate: passed=False, score=0.0\n  x exhaustion_comparable_gate: passed=False, score=0.0\n  + exhaustion_nondeterminant_gate: passed=True, score=1.0\n  + exhaustion_separable_program_gate: passed=True, score=1.0\n  + exhaustion_counterexample_gate: passed=True, score=1.0\n  x list_match: list_match (answer_field='exhaustion_relations'): FAIL  (score=0.0)\n  x hard_fail 'cf_signature_gut_contaminant': TRIGGERED\n  + hard_fail 'cf_inconsistent_present_without_activator': not triggered\n  + hard_fail 'cf_inconsistent_no_block_but_absent': not triggered\n  + hard_fail 'cf_inconsistent_late_block_without_poised_locus': not triggered\n  + hard_fail 'cf_inconsistent_early_block_with_brake_released': not triggered",
      "metrics": {
        "pass_rule": "all",
        "scoring_count": 17,
        "scoring_passed": 7,
        "scoring_total_score": 7,
        "hard_fail_triggered": [
          "cf_signature_gut_contaminant"
        ]
      },
      "fieldScores": {
        "list_match": 0,
        "sig_includes_itgae": 1,
        "declared_a_known_confounder": 0,
        "locus_differential_gate": 1,
        "formation_limited_by_gate": 1,
        "locus_competent_in_naive_gate": 1,
        "activator_chromatin_footprint_gate": 0,
        "exhaustion_comparable_gate": 0,
        "exhaustion_nondeterminant_gate": 1,
        "exhaustion_separable_program_gate": 1,
        "exhaustion_counterexample_gate": 1
      },
      "graderChecks": [
        {
          "name": "program_map",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "sig_includes_itgae",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "declared_a_known_confounder",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "generic_signature_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "cd103_program_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "per_model_regulatory_state",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "regulator_tuples",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "locus_differential_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "formation_limited_by_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "locus_competent_in_naive_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "ready_silent_fraction_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "activator_chromatin_footprint_gate",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "exhaustion_comparable_gate",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "exhaustion_nondeterminant_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "exhaustion_separable_program_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "exhaustion_counterexample_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "exhaustion_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "cf_signature_gut_contaminant",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "cf_inconsistent_present_without_activator",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_no_block_but_absent",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_late_block_without_poised_locus",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_early_block_with_brake_released",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "activator_chromatin_footprint": "consistent_in_formers",
        "cd103_program_relations": [
          {
            "pair": "B16_GP_vs_KPC_GP",
            "relation": "<"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": "<"
          },
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": "~"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "mWnt_GP_vs_B_ALL_GP",
            "relation": ">"
          }
        ],
        "core_residency_signature": [
          "Itgae",
          "Itga1",
          "Cd69",
          "Cxcr6",
          "Zfp683",
          "Cd101",
          "Cdh1",
          "Pmepa1",
          "P2rx7",
          "Ccr9",
          "Skil",
          "Tgfbr3",
          "Ahr",
          "Qpct",
          "Nt5e"
        ],
        "excluded_confounders": [
          "Pdcd1",
          "Havcr2",
          "Lag3",
          "Tigit",
          "Tox",
          "Entpd1",
          "Ctla4"
        ],
        "exhaustion_comparable_across_split": "yes",
        "exhaustion_counterexample_model": "KPC_GP",
        "exhaustion_explains_split": "no",
        "exhaustion_relations": [
          {
            "pair": "B16_GP_vs_KPC_GP",
            "relation": "~"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": "~"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_B_ALL_GP",
            "relation": "~"
          },
          {
            "pair": "mWnt_GP_vs_B_ALL_GP",
            "relation": "<"
          }
        ],
        "exhaustion_separable_program": "yes",
        "formation_limited_by": "absent_induction",
        "generic_signature_relations": [
          {
            "pair": "B16_GP_vs_KPC_GP",
            "relation": "<"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": "<"
          },
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "mWnt_GP_vs_B_ALL_GP",
            "relation": ">"
          }
        ],
        "locus_competent_in_naive": "yes",
        "locus_differential": "uniform_across_models",
        "per_model_regulatory_state": [
          {
            "activator_state": "absent",
            "blocking_step": "late_activation",
            "brake_state": "released",
            "locus_state": "poised_unfired",
            "model": "B16_GP",
            "program": "absent"
          },
          {
            "activator_state": "present",
            "blocking_step": "none",
            "brake_state": "released",
            "locus_state": "active",
            "model": "KPC_GP",
            "program": "present"
          },
          {
            "activator_state": "present",
            "blocking_step": "early_release",
            "brake_state": "partial",
            "locus_state": "active",
            "model": "mWnt_GP",
            "program": "partial"
          },
          {
            "activator_state": "absent",
            "blocking_step": "both_steps",
            "brake_state": "partial",
            "locus_state": "poised_unfired",
            "model": "B_ALL_GP",
            "program": "absent"
          }
        ],
        "program_map": [
          {
            "call": "absent",
            "model": "B16_GP"
          },
          {
            "call": "present",
            "model": "KPC_GP"
          },
          {
            "call": "partial",
            "model": "mWnt_GP"
          },
          {
            "call": "absent",
            "model": "B_ALL_GP"
          }
        ],
        "ready_silent_fraction_relations": [
          {
            "pair": "B16_GP_vs_KPC_GP",
            "relation": ">"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": "~"
          },
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": "~"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": "<"
          },
          {
            "pair": "KPC_GP_vs_B_ALL_GP",
            "relation": "<"
          },
          {
            "pair": "mWnt_GP_vs_B_ALL_GP",
            "relation": "~"
          }
        ],
        "regulator_tuples": [
          {
            "evidence_basis": "target_program_behavior",
            "expr_vs_activity": "discordant",
            "gating_step": "late_activation",
            "regulator": "Runx3",
            "role": "activator"
          },
          {
            "evidence_basis": "own_abundance",
            "expr_vs_activity": "concordant",
            "gating_step": "early_release",
            "regulator": "Klf2",
            "role": "repressor"
          }
        ]
      },
      "nTurns": 40,
      "checkpointSignals": {
        "clustering": false,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": true,
        "nerveJointAnchor": false,
        "multiomeAssembly": true,
        "depthNormalizedAtac": true,
        "regulonInference": true,
        "cd8SubsetIntegration": true,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": false,
        "orthologProjection": true,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": true,
        "enrichmentFamilyRanking": false,
        "mousePseudobulkDesign": true,
        "cancerEpithelialDenominator": false,
        "geneLevelDe": true,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "trajectoryFile": "scbench-long/trajectories/green_2_tme_context_dependence__claude-opus-4-8__pi__r1.json",
      "runId": "claude-opus-4-8__pi__green_2_tme_context_dependence__r1",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-opus-4-8-pi",
      "modelName": "claude-opus-4-8",
      "harness": "pi",
      "task": "green_2_tme_context_dependence",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 2,
      "passed": false,
      "score": 7,
      "durationS": 3305.94,
      "cost": 9.31742725,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "program_map",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "sig_includes_itgae",
            "role": "gate",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Itgae"
            ]
          },
          {
            "name": "declared_a_known_confounder",
            "role": "gate",
            "field": "$.excluded_confounders[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "cf_signature_gut_contaminant",
            "role": "hard_fail",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "generic_signature_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd103_program_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "<"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "per_model_regulatory_state",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "poised_unfired"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "late_activation"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "closing_committed"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_tuples",
            "kind": "list",
            "matchKey": "regulator",
            "k": 1,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "role": {
                    "op": "equals",
                    "values": [
                      "repressor"
                    ]
                  },
                  "gating_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  },
                  "evidence_basis": {
                    "op": "equals",
                    "values": [
                      "target_program_behavior"
                    ]
                  },
                  "expr_vs_activity": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "locus_differential_gate",
            "role": "gate",
            "field": "locus_differential",
            "kind": "value",
            "expected": "uniform_across_models"
          },
          {
            "name": "formation_limited_by_gate",
            "role": "gate",
            "field": "formation_limited_by",
            "kind": "value",
            "expected": "absent_induction"
          },
          {
            "name": "locus_competent_in_naive_gate",
            "role": "gate",
            "field": "locus_competent_in_naive",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "ready_silent_fraction_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 3,
            "expected": [
              {
                "key": "B16_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_KPC_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "activator_chromatin_footprint_gate",
            "role": "gate",
            "field": "activator_chromatin_footprint",
            "kind": "value",
            "expected": "inconsistent_across_formers"
          },
          {
            "name": "exhaustion_comparable_gate",
            "role": "gate",
            "field": "exhaustion_comparable_across_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_nondeterminant_gate",
            "role": "gate",
            "field": "exhaustion_explains_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_separable_program_gate",
            "role": "gate",
            "field": "exhaustion_separable_program",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "exhaustion_counterexample_gate",
            "role": "gate",
            "field": "exhaustion_counterexample_model",
            "kind": "value",
            "expected": "KPC_GP"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "exhaustion_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "cf_inconsistent_present_without_activator",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "program",
            "entity": "present",
            "valueField": "activator_state",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_no_block_but_absent",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "none",
            "valueField": "program",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_late_block_without_poised_locus",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "late_activation",
            "valueField": "locus_state",
            "expected": [
              "closing_committed",
              "active"
            ]
          },
          {
            "name": "cf_inconsistent_early_block_with_brake_released",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "early_release",
            "valueField": "brake_state",
            "expected": [
              "released"
            ]
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: FAIL\n  x list_match: list_match (answer_field='program_map'): FAIL  (score=0.0)\n  + sig_includes_itgae: passed=True, score=1.0\n  x declared_a_known_confounder: passed=False, score=0.0\n  + list_match: list_match (answer_field='generic_signature_relations'): PASS  (score=0.0)\n  x list_match: list_match (answer_field='cd103_program_relations'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='per_model_regulatory_state'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='regulator_tuples'): FAIL  (score=0.0)\n  + locus_differential_gate: passed=True, score=1.0\n  + formation_limited_by_gate: passed=True, score=1.0\n  + locus_competent_in_naive_gate: passed=True, score=1.0\n  x list_match: list_match (answer_field='ready_silent_fraction_relations'): FAIL  (score=0.0)\n  x activator_chromatin_footprint_gate: passed=False, score=0.0\n  x exhaustion_comparable_gate: passed=False, score=0.0\n  + exhaustion_nondeterminant_gate: passed=True, score=1.0\n  + exhaustion_separable_program_gate: passed=True, score=1.0\n  + exhaustion_counterexample_gate: passed=True, score=1.0\n  x list_match: list_match (answer_field='exhaustion_relations'): FAIL  (score=0.0)\n  x hard_fail 'cf_signature_gut_contaminant': TRIGGERED\n  + hard_fail 'cf_inconsistent_present_without_activator': not triggered\n  + hard_fail 'cf_inconsistent_no_block_but_absent': not triggered\n  x hard_fail 'cf_inconsistent_late_block_without_poised_locus': TRIGGERED\n  + hard_fail 'cf_inconsistent_early_block_with_brake_released': not triggered",
      "metrics": {
        "pass_rule": "all",
        "scoring_count": 17,
        "scoring_passed": 8,
        "scoring_total_score": 7,
        "hard_fail_triggered": [
          "cf_signature_gut_contaminant",
          "cf_inconsistent_late_block_without_poised_locus"
        ]
      },
      "fieldScores": {
        "list_match": 0,
        "sig_includes_itgae": 1,
        "declared_a_known_confounder": 0,
        "locus_differential_gate": 1,
        "formation_limited_by_gate": 1,
        "locus_competent_in_naive_gate": 1,
        "activator_chromatin_footprint_gate": 0,
        "exhaustion_comparable_gate": 0,
        "exhaustion_nondeterminant_gate": 1,
        "exhaustion_separable_program_gate": 1,
        "exhaustion_counterexample_gate": 1
      },
      "graderChecks": [
        {
          "name": "program_map",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "sig_includes_itgae",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "declared_a_known_confounder",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "generic_signature_relations",
          "kind": "list_match",
          "passed": true,
          "score": 0
        },
        {
          "name": "cd103_program_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "per_model_regulatory_state",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "regulator_tuples",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "locus_differential_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "formation_limited_by_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "locus_competent_in_naive_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "ready_silent_fraction_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "activator_chromatin_footprint_gate",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "exhaustion_comparable_gate",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "exhaustion_nondeterminant_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "exhaustion_separable_program_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "exhaustion_counterexample_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "exhaustion_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "cf_signature_gut_contaminant",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "cf_inconsistent_present_without_activator",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_no_block_but_absent",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_late_block_without_poised_locus",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "cf_inconsistent_early_block_with_brake_released",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "activator_chromatin_footprint": "consistent_in_formers",
        "cd103_program_relations": [
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_B16_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "mWnt_GP_vs_B16_GP",
            "relation": ">"
          },
          {
            "pair": "mWnt_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": "~"
          }
        ],
        "core_residency_signature": [
          "Itgae",
          "Itga1",
          "Itgb7",
          "Cdh1",
          "Cd101",
          "Chn2",
          "Qpct",
          "Skil",
          "Rgs1",
          "Xcl1",
          "P2rx7",
          "Cd69",
          "Pmepa1",
          "Acvr2a"
        ],
        "excluded_confounders": [
          "Pdcd1",
          "Havcr2",
          "Lag3",
          "Tigit",
          "Entpd1",
          "Tox",
          "Ctla4",
          "Tnfrsf9",
          "Gzmb",
          "Gzmk",
          "Cd160",
          "Cd244a"
        ],
        "exhaustion_comparable_across_split": "yes",
        "exhaustion_counterexample_model": "KPC_GP",
        "exhaustion_explains_split": "no",
        "exhaustion_relations": [
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_B16_GP",
            "relation": "~"
          },
          {
            "pair": "KPC_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "mWnt_GP_vs_B16_GP",
            "relation": "<"
          },
          {
            "pair": "mWnt_GP_vs_B_ALL_GP",
            "relation": "<"
          },
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": ">"
          }
        ],
        "exhaustion_separable_program": "yes",
        "formation_limited_by": "absent_induction",
        "generic_signature_relations": [
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": "~"
          },
          {
            "pair": "KPC_GP_vs_B16_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "mWnt_GP_vs_B16_GP",
            "relation": ">"
          },
          {
            "pair": "mWnt_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": ">"
          }
        ],
        "locus_competent_in_naive": "yes",
        "locus_differential": "uniform_across_models",
        "per_model_regulatory_state": [
          {
            "activator_state": "present",
            "blocking_step": "none",
            "brake_state": "released",
            "locus_state": "active",
            "model": "KPC_GP",
            "program": "present"
          },
          {
            "activator_state": "present",
            "blocking_step": "late_activation",
            "brake_state": "released",
            "locus_state": "poised_unfired",
            "model": "mWnt_GP",
            "program": "partial"
          },
          {
            "activator_state": "absent",
            "blocking_step": "late_activation",
            "brake_state": "released",
            "locus_state": "closing_committed",
            "model": "B16_GP",
            "program": "absent"
          },
          {
            "activator_state": "absent",
            "blocking_step": "both_steps",
            "brake_state": "partial",
            "locus_state": "poised_unfired",
            "model": "B_ALL_GP",
            "program": "absent"
          }
        ],
        "program_map": [
          {
            "call": "present",
            "model": "KPC_GP"
          },
          {
            "call": "partial",
            "model": "mWnt_GP"
          },
          {
            "call": "absent",
            "model": "B16_GP"
          },
          {
            "call": "absent",
            "model": "B_ALL_GP"
          }
        ],
        "ready_silent_fraction_relations": [
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": "<"
          },
          {
            "pair": "KPC_GP_vs_B16_GP",
            "relation": "<"
          },
          {
            "pair": "KPC_GP_vs_B_ALL_GP",
            "relation": "<"
          },
          {
            "pair": "mWnt_GP_vs_B16_GP",
            "relation": "~"
          },
          {
            "pair": "mWnt_GP_vs_B_ALL_GP",
            "relation": "~"
          },
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": "~"
          }
        ],
        "regulator_tuples": [
          {
            "evidence_basis": "target_program_behavior",
            "expr_vs_activity": "discordant",
            "gating_step": "late_activation",
            "regulator": "Runx3",
            "role": "activator"
          },
          {
            "evidence_basis": "own_abundance",
            "expr_vs_activity": "concordant",
            "gating_step": "early_release",
            "regulator": "Klf2",
            "role": "repressor"
          }
        ]
      },
      "nTurns": 46,
      "checkpointSignals": {
        "clustering": false,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": true,
        "nerveJointAnchor": false,
        "multiomeAssembly": true,
        "depthNormalizedAtac": true,
        "regulonInference": true,
        "cd8SubsetIntegration": true,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": false,
        "orthologProjection": false,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": true,
        "enrichmentFamilyRanking": true,
        "mousePseudobulkDesign": true,
        "cancerEpithelialDenominator": true,
        "geneLevelDe": true,
        "agingSignatureControl": true,
        "senescencePanelScoring": false
      },
      "trajectoryFile": "scbench-long/trajectories/green_2_tme_context_dependence__claude-opus-4-8__pi__r2.json",
      "runId": "claude-opus-4-8__pi__green_2_tme_context_dependence__r2",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-opus-4-8-pi",
      "modelName": "claude-opus-4-8",
      "harness": "pi",
      "task": "green_2_tme_context_dependence",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 3,
      "passed": false,
      "score": 7,
      "durationS": 2524.79,
      "cost": 5.8967290000000006,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "program_map",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "sig_includes_itgae",
            "role": "gate",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Itgae"
            ]
          },
          {
            "name": "declared_a_known_confounder",
            "role": "gate",
            "field": "$.excluded_confounders[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "cf_signature_gut_contaminant",
            "role": "hard_fail",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "generic_signature_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd103_program_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "<"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "per_model_regulatory_state",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "poised_unfired"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "late_activation"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "closing_committed"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_tuples",
            "kind": "list",
            "matchKey": "regulator",
            "k": 1,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "role": {
                    "op": "equals",
                    "values": [
                      "repressor"
                    ]
                  },
                  "gating_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  },
                  "evidence_basis": {
                    "op": "equals",
                    "values": [
                      "target_program_behavior"
                    ]
                  },
                  "expr_vs_activity": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "locus_differential_gate",
            "role": "gate",
            "field": "locus_differential",
            "kind": "value",
            "expected": "uniform_across_models"
          },
          {
            "name": "formation_limited_by_gate",
            "role": "gate",
            "field": "formation_limited_by",
            "kind": "value",
            "expected": "absent_induction"
          },
          {
            "name": "locus_competent_in_naive_gate",
            "role": "gate",
            "field": "locus_competent_in_naive",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "ready_silent_fraction_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 3,
            "expected": [
              {
                "key": "B16_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_KPC_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "activator_chromatin_footprint_gate",
            "role": "gate",
            "field": "activator_chromatin_footprint",
            "kind": "value",
            "expected": "inconsistent_across_formers"
          },
          {
            "name": "exhaustion_comparable_gate",
            "role": "gate",
            "field": "exhaustion_comparable_across_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_nondeterminant_gate",
            "role": "gate",
            "field": "exhaustion_explains_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_separable_program_gate",
            "role": "gate",
            "field": "exhaustion_separable_program",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "exhaustion_counterexample_gate",
            "role": "gate",
            "field": "exhaustion_counterexample_model",
            "kind": "value",
            "expected": "KPC_GP"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "exhaustion_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "cf_inconsistent_present_without_activator",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "program",
            "entity": "present",
            "valueField": "activator_state",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_no_block_but_absent",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "none",
            "valueField": "program",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_late_block_without_poised_locus",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "late_activation",
            "valueField": "locus_state",
            "expected": [
              "closing_committed",
              "active"
            ]
          },
          {
            "name": "cf_inconsistent_early_block_with_brake_released",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "early_release",
            "valueField": "brake_state",
            "expected": [
              "released"
            ]
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: FAIL\n  x list_match: list_match (answer_field='program_map'): FAIL  (score=0.0)\n  + sig_includes_itgae: passed=True, score=1.0\n  x declared_a_known_confounder: passed=False, score=0.0\n  + list_match: list_match (answer_field='generic_signature_relations'): PASS  (score=0.0)\n  x list_match: list_match (answer_field='cd103_program_relations'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='per_model_regulatory_state'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='regulator_tuples'): FAIL  (score=0.0)\n  + locus_differential_gate: passed=True, score=1.0\n  + formation_limited_by_gate: passed=True, score=1.0\n  + locus_competent_in_naive_gate: passed=True, score=1.0\n  x list_match: list_match (answer_field='ready_silent_fraction_relations'): FAIL  (score=0.0)\n  x activator_chromatin_footprint_gate: passed=False, score=0.0\n  x exhaustion_comparable_gate: passed=False, score=0.0\n  + exhaustion_nondeterminant_gate: passed=True, score=1.0\n  + exhaustion_separable_program_gate: passed=True, score=1.0\n  + exhaustion_counterexample_gate: passed=True, score=1.0\n  x list_match: list_match (answer_field='exhaustion_relations'): FAIL  (score=0.0)\n  x hard_fail 'cf_signature_gut_contaminant': TRIGGERED\n  + hard_fail 'cf_inconsistent_present_without_activator': not triggered\n  + hard_fail 'cf_inconsistent_no_block_but_absent': not triggered\n  x hard_fail 'cf_inconsistent_late_block_without_poised_locus': TRIGGERED\n  + hard_fail 'cf_inconsistent_early_block_with_brake_released': not triggered",
      "metrics": {
        "pass_rule": "all",
        "scoring_count": 17,
        "scoring_passed": 8,
        "scoring_total_score": 7,
        "hard_fail_triggered": [
          "cf_signature_gut_contaminant",
          "cf_inconsistent_late_block_without_poised_locus"
        ]
      },
      "fieldScores": {
        "list_match": 0,
        "sig_includes_itgae": 1,
        "declared_a_known_confounder": 0,
        "locus_differential_gate": 1,
        "formation_limited_by_gate": 1,
        "locus_competent_in_naive_gate": 1,
        "activator_chromatin_footprint_gate": 0,
        "exhaustion_comparable_gate": 0,
        "exhaustion_nondeterminant_gate": 1,
        "exhaustion_separable_program_gate": 1,
        "exhaustion_counterexample_gate": 1
      },
      "graderChecks": [
        {
          "name": "program_map",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "sig_includes_itgae",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "declared_a_known_confounder",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "generic_signature_relations",
          "kind": "list_match",
          "passed": true,
          "score": 0
        },
        {
          "name": "cd103_program_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "per_model_regulatory_state",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "regulator_tuples",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "locus_differential_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "formation_limited_by_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "locus_competent_in_naive_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "ready_silent_fraction_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "activator_chromatin_footprint_gate",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "exhaustion_comparable_gate",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "exhaustion_nondeterminant_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "exhaustion_separable_program_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "exhaustion_counterexample_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "exhaustion_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "cf_signature_gut_contaminant",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "cf_inconsistent_present_without_activator",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_no_block_but_absent",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_late_block_without_poised_locus",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "cf_inconsistent_early_block_with_brake_released",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "activator_chromatin_footprint": "consistent_in_formers",
        "cd103_program_relations": [
          {
            "pair": "KPC_GP_vs_B16_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": "<"
          },
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "mWnt_GP_vs_B_ALL_GP",
            "relation": ">"
          }
        ],
        "core_residency_signature": [
          "Itgae",
          "Itga1",
          "Cd101",
          "Chn2",
          "Rgs1",
          "Skil",
          "Qpct"
        ],
        "excluded_confounders": [
          "Cd69",
          "Cxcr6",
          "Rgs2",
          "Litaf",
          "Bhlhe40",
          "Nr4a2",
          "Tox",
          "Gzmb",
          "Cd160"
        ],
        "exhaustion_comparable_across_split": "yes",
        "exhaustion_counterexample_model": "KPC_GP",
        "exhaustion_explains_split": "no",
        "exhaustion_relations": [
          {
            "pair": "KPC_GP_vs_B16_GP",
            "relation": "~"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_B_ALL_GP",
            "relation": "~"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": "~"
          },
          {
            "pair": "mWnt_GP_vs_B_ALL_GP",
            "relation": "<"
          }
        ],
        "exhaustion_separable_program": "yes",
        "formation_limited_by": "absent_induction",
        "generic_signature_relations": [
          {
            "pair": "KPC_GP_vs_B16_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": "~"
          },
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "mWnt_GP_vs_B_ALL_GP",
            "relation": ">"
          }
        ],
        "locus_competent_in_naive": "yes",
        "locus_differential": "uniform_across_models",
        "per_model_regulatory_state": [
          {
            "activator_state": "present",
            "blocking_step": "none",
            "brake_state": "released",
            "locus_state": "active",
            "model": "KPC_GP",
            "program": "present"
          },
          {
            "activator_state": "absent",
            "blocking_step": "late_activation",
            "brake_state": "released",
            "locus_state": "closing_committed",
            "model": "B16_GP",
            "program": "absent"
          },
          {
            "activator_state": "present",
            "blocking_step": "early_release",
            "brake_state": "partial",
            "locus_state": "active",
            "model": "mWnt_GP",
            "program": "partial"
          },
          {
            "activator_state": "absent",
            "blocking_step": "both_steps",
            "brake_state": "partial",
            "locus_state": "closing_committed",
            "model": "B_ALL_GP",
            "program": "absent"
          }
        ],
        "program_map": [
          {
            "call": "present",
            "model": "KPC_GP"
          },
          {
            "call": "absent",
            "model": "B16_GP"
          },
          {
            "call": "partial",
            "model": "mWnt_GP"
          },
          {
            "call": "absent",
            "model": "B_ALL_GP"
          }
        ],
        "ready_silent_fraction_relations": [
          {
            "pair": "KPC_GP_vs_B16_GP",
            "relation": "<"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": "<"
          },
          {
            "pair": "KPC_GP_vs_B_ALL_GP",
            "relation": "<"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": "~"
          },
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": "~"
          },
          {
            "pair": "mWnt_GP_vs_B_ALL_GP",
            "relation": "~"
          }
        ],
        "regulator_tuples": [
          {
            "evidence_basis": "target_program_behavior",
            "expr_vs_activity": "discordant",
            "gating_step": "late_activation",
            "regulator": "Runx3",
            "role": "activator"
          },
          {
            "evidence_basis": "own_abundance",
            "expr_vs_activity": "concordant",
            "gating_step": "early_release",
            "regulator": "Klf2",
            "role": "repressor"
          }
        ]
      },
      "nTurns": 42,
      "checkpointSignals": {
        "clustering": true,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": false,
        "nerveJointAnchor": false,
        "multiomeAssembly": true,
        "depthNormalizedAtac": true,
        "regulonInference": true,
        "cd8SubsetIntegration": true,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": false,
        "orthologProjection": false,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": true,
        "enrichmentFamilyRanking": true,
        "mousePseudobulkDesign": true,
        "cancerEpithelialDenominator": false,
        "geneLevelDe": true,
        "agingSignatureControl": true,
        "senescencePanelScoring": false
      },
      "trajectoryFile": "scbench-long/trajectories/green_2_tme_context_dependence__claude-opus-4-8__pi__r3.json",
      "runId": "claude-opus-4-8__pi__green_2_tme_context_dependence__r3",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-sonnet-4-6-pi",
      "modelName": "claude-sonnet-4-6",
      "harness": "pi",
      "task": "green_2_tme_context_dependence",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 1,
      "passed": false,
      "score": 7,
      "durationS": 3173.88,
      "cost": 2.5366936499999997,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "program_map",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "sig_includes_itgae",
            "role": "gate",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Itgae"
            ]
          },
          {
            "name": "declared_a_known_confounder",
            "role": "gate",
            "field": "$.excluded_confounders[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "cf_signature_gut_contaminant",
            "role": "hard_fail",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "generic_signature_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd103_program_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "<"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "per_model_regulatory_state",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "poised_unfired"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "late_activation"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "closing_committed"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_tuples",
            "kind": "list",
            "matchKey": "regulator",
            "k": 1,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "role": {
                    "op": "equals",
                    "values": [
                      "repressor"
                    ]
                  },
                  "gating_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  },
                  "evidence_basis": {
                    "op": "equals",
                    "values": [
                      "target_program_behavior"
                    ]
                  },
                  "expr_vs_activity": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "locus_differential_gate",
            "role": "gate",
            "field": "locus_differential",
            "kind": "value",
            "expected": "uniform_across_models"
          },
          {
            "name": "formation_limited_by_gate",
            "role": "gate",
            "field": "formation_limited_by",
            "kind": "value",
            "expected": "absent_induction"
          },
          {
            "name": "locus_competent_in_naive_gate",
            "role": "gate",
            "field": "locus_competent_in_naive",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "ready_silent_fraction_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 3,
            "expected": [
              {
                "key": "B16_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_KPC_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "activator_chromatin_footprint_gate",
            "role": "gate",
            "field": "activator_chromatin_footprint",
            "kind": "value",
            "expected": "inconsistent_across_formers"
          },
          {
            "name": "exhaustion_comparable_gate",
            "role": "gate",
            "field": "exhaustion_comparable_across_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_nondeterminant_gate",
            "role": "gate",
            "field": "exhaustion_explains_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_separable_program_gate",
            "role": "gate",
            "field": "exhaustion_separable_program",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "exhaustion_counterexample_gate",
            "role": "gate",
            "field": "exhaustion_counterexample_model",
            "kind": "value",
            "expected": "KPC_GP"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "exhaustion_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "cf_inconsistent_present_without_activator",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "program",
            "entity": "present",
            "valueField": "activator_state",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_no_block_but_absent",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "none",
            "valueField": "program",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_late_block_without_poised_locus",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "late_activation",
            "valueField": "locus_state",
            "expected": [
              "closing_committed",
              "active"
            ]
          },
          {
            "name": "cf_inconsistent_early_block_with_brake_released",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "early_release",
            "valueField": "brake_state",
            "expected": [
              "released"
            ]
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: FAIL\n  x list_match: list_match (answer_field='program_map'): FAIL  (score=0.0)\n  + sig_includes_itgae: passed=True, score=1.0\n  x declared_a_known_confounder: passed=False, score=0.0\n  x list_match: list_match (answer_field='generic_signature_relations'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='cd103_program_relations'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='per_model_regulatory_state'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='regulator_tuples'): FAIL  (score=0.0)\n  + locus_differential_gate: passed=True, score=1.0\n  + formation_limited_by_gate: passed=True, score=1.0\n  + locus_competent_in_naive_gate: passed=True, score=1.0\n  x list_match: list_match (answer_field='ready_silent_fraction_relations'): FAIL  (score=0.0)\n  x activator_chromatin_footprint_gate: passed=False, score=0.0\n  x exhaustion_comparable_gate: passed=False, score=0.0\n  + exhaustion_nondeterminant_gate: passed=True, score=1.0\n  + exhaustion_separable_program_gate: passed=True, score=1.0\n  + exhaustion_counterexample_gate: passed=True, score=1.0\n  x list_match: list_match (answer_field='exhaustion_relations'): FAIL  (score=0.0)\n  + hard_fail 'cf_signature_gut_contaminant': not triggered\n  + hard_fail 'cf_inconsistent_present_without_activator': not triggered\n  + hard_fail 'cf_inconsistent_no_block_but_absent': not triggered\n  + hard_fail 'cf_inconsistent_late_block_without_poised_locus': not triggered\n  + hard_fail 'cf_inconsistent_early_block_with_brake_released': not triggered",
      "metrics": {
        "pass_rule": "all",
        "scoring_count": 17,
        "scoring_passed": 7,
        "scoring_total_score": 7,
        "hard_fail_triggered": []
      },
      "fieldScores": {
        "list_match": 0,
        "sig_includes_itgae": 1,
        "declared_a_known_confounder": 0,
        "locus_differential_gate": 1,
        "formation_limited_by_gate": 1,
        "locus_competent_in_naive_gate": 1,
        "activator_chromatin_footprint_gate": 0,
        "exhaustion_comparable_gate": 0,
        "exhaustion_nondeterminant_gate": 1,
        "exhaustion_separable_program_gate": 1,
        "exhaustion_counterexample_gate": 1
      },
      "graderChecks": [
        {
          "name": "program_map",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "sig_includes_itgae",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "declared_a_known_confounder",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "generic_signature_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "cd103_program_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "per_model_regulatory_state",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "regulator_tuples",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "locus_differential_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "formation_limited_by_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "locus_competent_in_naive_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "ready_silent_fraction_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "activator_chromatin_footprint_gate",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "exhaustion_comparable_gate",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "exhaustion_nondeterminant_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "exhaustion_separable_program_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "exhaustion_counterexample_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "exhaustion_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "cf_signature_gut_contaminant",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_present_without_activator",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_no_block_but_absent",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_late_block_without_poised_locus",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_early_block_with_brake_released",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "activator_chromatin_footprint": "consistent_in_formers",
        "cd103_program_relations": [
          {
            "pair": "B16_GP_vs_KPC_GP",
            "relation": "<"
          },
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": "~"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": "<"
          },
          {
            "pair": "KPC_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "B_ALL_GP_vs_mWnt_GP",
            "relation": "<"
          }
        ],
        "core_residency_signature": [
          "Itgae",
          "Cd69",
          "Runx3",
          "Itgb7",
          "Tgfbr2"
        ],
        "excluded_confounders": [
          "Bhlhe40",
          "Tgfb1",
          "Cxcr6"
        ],
        "exhaustion_comparable_across_split": "yes",
        "exhaustion_counterexample_model": "KPC_GP",
        "exhaustion_explains_split": "no",
        "exhaustion_relations": [
          {
            "pair": "B16_GP_vs_KPC_GP",
            "relation": "~"
          },
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": "~"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "B_ALL_GP_vs_mWnt_GP",
            "relation": ">"
          }
        ],
        "exhaustion_separable_program": "yes",
        "formation_limited_by": "absent_induction",
        "generic_signature_relations": [
          {
            "pair": "B16_GP_vs_KPC_GP",
            "relation": "<"
          },
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": "~"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": "<"
          },
          {
            "pair": "KPC_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": "<"
          },
          {
            "pair": "B_ALL_GP_vs_mWnt_GP",
            "relation": "<"
          }
        ],
        "locus_competent_in_naive": "yes",
        "locus_differential": "uniform_across_models",
        "per_model_regulatory_state": [
          {
            "activator_state": "present",
            "blocking_step": "none",
            "brake_state": "released",
            "locus_state": "active",
            "model": "KPC_GP",
            "program": "present"
          },
          {
            "activator_state": "present",
            "blocking_step": "early_release",
            "brake_state": "engaged",
            "locus_state": "poised_unfired",
            "model": "mWnt_GP",
            "program": "partial"
          },
          {
            "activator_state": "partial",
            "blocking_step": "late_activation",
            "brake_state": "released",
            "locus_state": "poised_unfired",
            "model": "B16_GP",
            "program": "absent"
          },
          {
            "activator_state": "partial",
            "blocking_step": "both_steps",
            "brake_state": "engaged",
            "locus_state": "poised_unfired",
            "model": "B_ALL_GP",
            "program": "absent"
          }
        ],
        "program_map": [
          {
            "call": "absent",
            "model": "B16_GP"
          },
          {
            "call": "present",
            "model": "KPC_GP"
          },
          {
            "call": "absent",
            "model": "B_ALL_GP"
          },
          {
            "call": "partial",
            "model": "mWnt_GP"
          }
        ],
        "ready_silent_fraction_relations": [
          {
            "pair": "B16_GP_vs_KPC_GP",
            "relation": ">"
          },
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": "<"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": "<"
          },
          {
            "pair": "KPC_GP_vs_B_ALL_GP",
            "relation": "<"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": "<"
          },
          {
            "pair": "B_ALL_GP_vs_mWnt_GP",
            "relation": ">"
          }
        ],
        "regulator_tuples": [
          {
            "evidence_basis": "target_program_behavior",
            "expr_vs_activity": "discordant",
            "gating_step": "late_activation",
            "regulator": "Runx3",
            "role": "activator"
          },
          {
            "evidence_basis": "own_abundance",
            "expr_vs_activity": "concordant",
            "gating_step": "early_release",
            "regulator": "S1pr1",
            "role": "repressor"
          },
          {
            "evidence_basis": "target_program_behavior",
            "expr_vs_activity": "concordant",
            "gating_step": "late_activation",
            "regulator": "Eomes",
            "role": "repressor"
          },
          {
            "evidence_basis": "target_program_behavior",
            "expr_vs_activity": "concordant",
            "gating_step": "late_activation",
            "regulator": "Tgfbr2",
            "role": "activator"
          }
        ]
      },
      "nTurns": 55,
      "checkpointSignals": {
        "clustering": false,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": false,
        "nerveJointAnchor": false,
        "multiomeAssembly": true,
        "depthNormalizedAtac": true,
        "regulonInference": true,
        "cd8SubsetIntegration": true,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": false,
        "orthologProjection": false,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": true,
        "mousePseudobulkDesign": false,
        "cancerEpithelialDenominator": false,
        "geneLevelDe": true,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "trajectoryFile": "scbench-long/trajectories/green_2_tme_context_dependence__claude-sonnet-4-6__pi__r1.json",
      "runId": "claude-sonnet-4-6__pi__green_2_tme_context_dependence__r1",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-sonnet-4-6-pi",
      "modelName": "claude-sonnet-4-6",
      "harness": "pi",
      "task": "green_2_tme_context_dependence",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 2,
      "passed": false,
      "score": 8,
      "durationS": 3317.63,
      "cost": 3.0422343,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "program_map",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "sig_includes_itgae",
            "role": "gate",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Itgae"
            ]
          },
          {
            "name": "declared_a_known_confounder",
            "role": "gate",
            "field": "$.excluded_confounders[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "cf_signature_gut_contaminant",
            "role": "hard_fail",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "generic_signature_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd103_program_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "<"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "per_model_regulatory_state",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "poised_unfired"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "late_activation"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "closing_committed"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_tuples",
            "kind": "list",
            "matchKey": "regulator",
            "k": 1,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "role": {
                    "op": "equals",
                    "values": [
                      "repressor"
                    ]
                  },
                  "gating_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  },
                  "evidence_basis": {
                    "op": "equals",
                    "values": [
                      "target_program_behavior"
                    ]
                  },
                  "expr_vs_activity": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "locus_differential_gate",
            "role": "gate",
            "field": "locus_differential",
            "kind": "value",
            "expected": "uniform_across_models"
          },
          {
            "name": "formation_limited_by_gate",
            "role": "gate",
            "field": "formation_limited_by",
            "kind": "value",
            "expected": "absent_induction"
          },
          {
            "name": "locus_competent_in_naive_gate",
            "role": "gate",
            "field": "locus_competent_in_naive",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "ready_silent_fraction_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 3,
            "expected": [
              {
                "key": "B16_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_KPC_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "activator_chromatin_footprint_gate",
            "role": "gate",
            "field": "activator_chromatin_footprint",
            "kind": "value",
            "expected": "inconsistent_across_formers"
          },
          {
            "name": "exhaustion_comparable_gate",
            "role": "gate",
            "field": "exhaustion_comparable_across_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_nondeterminant_gate",
            "role": "gate",
            "field": "exhaustion_explains_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_separable_program_gate",
            "role": "gate",
            "field": "exhaustion_separable_program",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "exhaustion_counterexample_gate",
            "role": "gate",
            "field": "exhaustion_counterexample_model",
            "kind": "value",
            "expected": "KPC_GP"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "exhaustion_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "cf_inconsistent_present_without_activator",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "program",
            "entity": "present",
            "valueField": "activator_state",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_no_block_but_absent",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "none",
            "valueField": "program",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_late_block_without_poised_locus",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "late_activation",
            "valueField": "locus_state",
            "expected": [
              "closing_committed",
              "active"
            ]
          },
          {
            "name": "cf_inconsistent_early_block_with_brake_released",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "early_release",
            "valueField": "brake_state",
            "expected": [
              "released"
            ]
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: FAIL\n  x list_match: list_match (answer_field='program_map'): FAIL  (score=0.0)\n  + sig_includes_itgae: passed=True, score=1.0\n  x declared_a_known_confounder: passed=False, score=0.0\n  x list_match: list_match (answer_field='generic_signature_relations'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='cd103_program_relations'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='per_model_regulatory_state'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='regulator_tuples'): FAIL  (score=0.0)\n  + locus_differential_gate: passed=True, score=1.0\n  + formation_limited_by_gate: passed=True, score=1.0\n  + locus_competent_in_naive_gate: passed=True, score=1.0\n  x list_match: list_match (answer_field='ready_silent_fraction_relations'): FAIL  (score=0.0)\n  x activator_chromatin_footprint_gate: passed=False, score=0.0\n  + exhaustion_comparable_gate: passed=True, score=1.0\n  + exhaustion_nondeterminant_gate: passed=True, score=1.0\n  + exhaustion_separable_program_gate: passed=True, score=1.0\n  + exhaustion_counterexample_gate: passed=True, score=1.0\n  x list_match: list_match (answer_field='exhaustion_relations'): FAIL  (score=0.0)\n  + hard_fail 'cf_signature_gut_contaminant': not triggered\n  + hard_fail 'cf_inconsistent_present_without_activator': not triggered\n  + hard_fail 'cf_inconsistent_no_block_but_absent': not triggered\n  + hard_fail 'cf_inconsistent_late_block_without_poised_locus': not triggered\n  + hard_fail 'cf_inconsistent_early_block_with_brake_released': not triggered",
      "metrics": {
        "pass_rule": "all",
        "scoring_count": 17,
        "scoring_passed": 8,
        "scoring_total_score": 8,
        "hard_fail_triggered": []
      },
      "fieldScores": {
        "list_match": 0,
        "sig_includes_itgae": 1,
        "declared_a_known_confounder": 0,
        "locus_differential_gate": 1,
        "formation_limited_by_gate": 1,
        "locus_competent_in_naive_gate": 1,
        "activator_chromatin_footprint_gate": 0,
        "exhaustion_comparable_gate": 1,
        "exhaustion_nondeterminant_gate": 1,
        "exhaustion_separable_program_gate": 1,
        "exhaustion_counterexample_gate": 1
      },
      "graderChecks": [
        {
          "name": "program_map",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "sig_includes_itgae",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "declared_a_known_confounder",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "generic_signature_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "cd103_program_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "per_model_regulatory_state",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "regulator_tuples",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "locus_differential_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "formation_limited_by_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "locus_competent_in_naive_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "ready_silent_fraction_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "activator_chromatin_footprint_gate",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "exhaustion_comparable_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "exhaustion_nondeterminant_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "exhaustion_separable_program_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "exhaustion_counterexample_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "exhaustion_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "cf_signature_gut_contaminant",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_present_without_activator",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_no_block_but_absent",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_late_block_without_poised_locus",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_early_block_with_brake_released",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "activator_chromatin_footprint": "consistent_in_formers",
        "cd103_program_relations": [
          {
            "pair": "B16_GP_vs_KPC_GP",
            "relation": "<"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": "<"
          },
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": "~"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "mWnt_GP_vs_B_ALL_GP",
            "relation": ">"
          }
        ],
        "core_residency_signature": [
          "Itgae",
          "Cd69",
          "Runx3",
          "Cxcr6",
          "Tgfbr2"
        ],
        "excluded_confounders": [
          "Tox",
          "Eomes",
          "Pdcd1",
          "Havcr2"
        ],
        "exhaustion_comparable_across_split": "no",
        "exhaustion_counterexample_model": "KPC_GP",
        "exhaustion_explains_split": "no",
        "exhaustion_relations": [
          {
            "pair": "B16_GP_vs_KPC_GP",
            "relation": "~"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": "<"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_B_ALL_GP",
            "relation": "<"
          },
          {
            "pair": "mWnt_GP_vs_B_ALL_GP",
            "relation": "<"
          }
        ],
        "exhaustion_separable_program": "yes",
        "formation_limited_by": "absent_induction",
        "generic_signature_relations": [
          {
            "pair": "B16_GP_vs_KPC_GP",
            "relation": "~"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": "<"
          },
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": "<"
          },
          {
            "pair": "KPC_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "mWnt_GP_vs_B_ALL_GP",
            "relation": ">"
          }
        ],
        "locus_competent_in_naive": "yes",
        "locus_differential": "uniform_across_models",
        "per_model_regulatory_state": [
          {
            "activator_state": "present",
            "blocking_step": "none",
            "brake_state": "released",
            "locus_state": "active",
            "model": "KPC_GP",
            "program": "present"
          },
          {
            "activator_state": "partial",
            "blocking_step": "late_activation",
            "brake_state": "released",
            "locus_state": "poised_unfired",
            "model": "mWnt_GP",
            "program": "partial"
          },
          {
            "activator_state": "absent",
            "blocking_step": "late_activation",
            "brake_state": "partial",
            "locus_state": "poised_unfired",
            "model": "B16_GP",
            "program": "absent"
          },
          {
            "activator_state": "present",
            "blocking_step": "early_release",
            "brake_state": "engaged",
            "locus_state": "poised_unfired",
            "model": "B_ALL_GP",
            "program": "absent"
          }
        ],
        "program_map": [
          {
            "call": "absent",
            "model": "B16_GP"
          },
          {
            "call": "present",
            "model": "KPC_GP"
          },
          {
            "call": "partial",
            "model": "mWnt_GP"
          },
          {
            "call": "absent",
            "model": "B_ALL_GP"
          }
        ],
        "ready_silent_fraction_relations": [
          {
            "pair": "B16_GP_vs_KPC_GP",
            "relation": ">"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": "~"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": "<"
          },
          {
            "pair": "KPC_GP_vs_B_ALL_GP",
            "relation": "<"
          },
          {
            "pair": "mWnt_GP_vs_B_ALL_GP",
            "relation": "<"
          }
        ],
        "regulator_tuples": [
          {
            "evidence_basis": "target_program_behavior",
            "expr_vs_activity": "discordant",
            "gating_step": "late_activation",
            "regulator": "Runx3",
            "role": "activator"
          },
          {
            "evidence_basis": "own_abundance",
            "expr_vs_activity": "concordant",
            "gating_step": "early_release",
            "regulator": "Eomes",
            "role": "repressor"
          }
        ]
      },
      "nTurns": 51,
      "checkpointSignals": {
        "clustering": false,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": false,
        "nerveJointAnchor": false,
        "multiomeAssembly": true,
        "depthNormalizedAtac": true,
        "regulonInference": true,
        "cd8SubsetIntegration": true,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": false,
        "orthologProjection": false,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": false,
        "mousePseudobulkDesign": false,
        "cancerEpithelialDenominator": false,
        "geneLevelDe": false,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "trajectoryFile": "scbench-long/trajectories/green_2_tme_context_dependence__claude-sonnet-4-6__pi__r2.json",
      "runId": "claude-sonnet-4-6__pi__green_2_tme_context_dependence__r2",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-sonnet-4-6-pi",
      "modelName": "claude-sonnet-4-6",
      "harness": "pi",
      "task": "green_2_tme_context_dependence",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 3,
      "passed": false,
      "score": 8,
      "durationS": 4004.8,
      "cost": 3.024141,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "program_map",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "sig_includes_itgae",
            "role": "gate",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Itgae"
            ]
          },
          {
            "name": "declared_a_known_confounder",
            "role": "gate",
            "field": "$.excluded_confounders[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "cf_signature_gut_contaminant",
            "role": "hard_fail",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "generic_signature_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd103_program_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "<"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "per_model_regulatory_state",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "poised_unfired"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "late_activation"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "closing_committed"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_tuples",
            "kind": "list",
            "matchKey": "regulator",
            "k": 1,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "role": {
                    "op": "equals",
                    "values": [
                      "repressor"
                    ]
                  },
                  "gating_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  },
                  "evidence_basis": {
                    "op": "equals",
                    "values": [
                      "target_program_behavior"
                    ]
                  },
                  "expr_vs_activity": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "locus_differential_gate",
            "role": "gate",
            "field": "locus_differential",
            "kind": "value",
            "expected": "uniform_across_models"
          },
          {
            "name": "formation_limited_by_gate",
            "role": "gate",
            "field": "formation_limited_by",
            "kind": "value",
            "expected": "absent_induction"
          },
          {
            "name": "locus_competent_in_naive_gate",
            "role": "gate",
            "field": "locus_competent_in_naive",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "ready_silent_fraction_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 3,
            "expected": [
              {
                "key": "B16_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_KPC_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "activator_chromatin_footprint_gate",
            "role": "gate",
            "field": "activator_chromatin_footprint",
            "kind": "value",
            "expected": "inconsistent_across_formers"
          },
          {
            "name": "exhaustion_comparable_gate",
            "role": "gate",
            "field": "exhaustion_comparable_across_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_nondeterminant_gate",
            "role": "gate",
            "field": "exhaustion_explains_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_separable_program_gate",
            "role": "gate",
            "field": "exhaustion_separable_program",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "exhaustion_counterexample_gate",
            "role": "gate",
            "field": "exhaustion_counterexample_model",
            "kind": "value",
            "expected": "KPC_GP"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "exhaustion_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "cf_inconsistent_present_without_activator",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "program",
            "entity": "present",
            "valueField": "activator_state",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_no_block_but_absent",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "none",
            "valueField": "program",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_late_block_without_poised_locus",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "late_activation",
            "valueField": "locus_state",
            "expected": [
              "closing_committed",
              "active"
            ]
          },
          {
            "name": "cf_inconsistent_early_block_with_brake_released",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "early_release",
            "valueField": "brake_state",
            "expected": [
              "released"
            ]
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: FAIL\n  x list_match: list_match (answer_field='program_map'): FAIL  (score=0.0)\n  + sig_includes_itgae: passed=True, score=1.0\n  x declared_a_known_confounder: passed=False, score=0.0\n  x list_match: list_match (answer_field='generic_signature_relations'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='cd103_program_relations'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='per_model_regulatory_state'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='regulator_tuples'): FAIL  (score=0.0)\n  + locus_differential_gate: passed=True, score=1.0\n  + formation_limited_by_gate: passed=True, score=1.0\n  + locus_competent_in_naive_gate: passed=True, score=1.0\n  x list_match: list_match (answer_field='ready_silent_fraction_relations'): FAIL  (score=0.0)\n  + activator_chromatin_footprint_gate: passed=True, score=1.0\n  + exhaustion_comparable_gate: passed=True, score=1.0\n  + exhaustion_nondeterminant_gate: passed=True, score=1.0\n  + exhaustion_separable_program_gate: passed=True, score=1.0\n  x exhaustion_counterexample_gate: passed=False, score=0.0\n  + list_match: list_match (answer_field='exhaustion_relations'): PASS  (score=0.0)\n  + hard_fail 'cf_signature_gut_contaminant': not triggered\n  + hard_fail 'cf_inconsistent_present_without_activator': not triggered\n  + hard_fail 'cf_inconsistent_no_block_but_absent': not triggered\n  + hard_fail 'cf_inconsistent_late_block_without_poised_locus': not triggered\n  + hard_fail 'cf_inconsistent_early_block_with_brake_released': not triggered",
      "metrics": {
        "pass_rule": "all",
        "scoring_count": 17,
        "scoring_passed": 9,
        "scoring_total_score": 8,
        "hard_fail_triggered": []
      },
      "fieldScores": {
        "list_match": 0,
        "sig_includes_itgae": 1,
        "declared_a_known_confounder": 0,
        "locus_differential_gate": 1,
        "formation_limited_by_gate": 1,
        "locus_competent_in_naive_gate": 1,
        "activator_chromatin_footprint_gate": 1,
        "exhaustion_comparable_gate": 1,
        "exhaustion_nondeterminant_gate": 1,
        "exhaustion_separable_program_gate": 1,
        "exhaustion_counterexample_gate": 0
      },
      "graderChecks": [
        {
          "name": "program_map",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "sig_includes_itgae",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "declared_a_known_confounder",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "generic_signature_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "cd103_program_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "per_model_regulatory_state",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "regulator_tuples",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "locus_differential_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "formation_limited_by_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "locus_competent_in_naive_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "ready_silent_fraction_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "activator_chromatin_footprint_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "exhaustion_comparable_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "exhaustion_nondeterminant_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "exhaustion_separable_program_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "exhaustion_counterexample_gate",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "exhaustion_relations",
          "kind": "list_match",
          "passed": true,
          "score": 0
        },
        {
          "name": "cf_signature_gut_contaminant",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_present_without_activator",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_no_block_but_absent",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_late_block_without_poised_locus",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_early_block_with_brake_released",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "activator_chromatin_footprint": "inconsistent_across_formers",
        "cd103_program_relations": [
          {
            "pair": "KPC_GP_vs_B16_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": "~"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": "<"
          },
          {
            "pair": "B_ALL_GP_vs_mWnt_GP",
            "relation": "<"
          }
        ],
        "core_residency_signature": [
          "Itgae",
          "Cxcr6",
          "Runx3",
          "Zeb1",
          "Smad3",
          "Cd69",
          "Tgfbr2"
        ],
        "excluded_confounders": [
          "Eomes",
          "Gzmb",
          "Irf8",
          "Klf2",
          "Tox"
        ],
        "exhaustion_comparable_across_split": "no",
        "exhaustion_counterexample_model": "mWnt_GP",
        "exhaustion_explains_split": "no",
        "exhaustion_relations": [
          {
            "pair": "KPC_GP_vs_B16_GP",
            "relation": "~"
          },
          {
            "pair": "KPC_GP_vs_B_ALL_GP",
            "relation": "<"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": "<"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "B_ALL_GP_vs_mWnt_GP",
            "relation": ">"
          }
        ],
        "exhaustion_separable_program": "yes",
        "formation_limited_by": "absent_induction",
        "generic_signature_relations": [
          {
            "pair": "KPC_GP_vs_B16_GP",
            "relation": "~"
          },
          {
            "pair": "KPC_GP_vs_B_ALL_GP",
            "relation": "~"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": "<"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "B_ALL_GP_vs_mWnt_GP",
            "relation": ">"
          }
        ],
        "locus_competent_in_naive": "yes",
        "locus_differential": "uniform_across_models",
        "per_model_regulatory_state": [
          {
            "activator_state": "present",
            "blocking_step": "none",
            "brake_state": "released",
            "locus_state": "active",
            "model": "KPC_GP",
            "program": "present"
          },
          {
            "activator_state": "partial",
            "blocking_step": "late_activation",
            "brake_state": "partial",
            "locus_state": "poised_unfired",
            "model": "B16_GP",
            "program": "absent"
          },
          {
            "activator_state": "partial",
            "blocking_step": "early_release",
            "brake_state": "engaged",
            "locus_state": "poised_unfired",
            "model": "B_ALL_GP",
            "program": "absent"
          },
          {
            "activator_state": "present",
            "blocking_step": "early_release",
            "brake_state": "partial",
            "locus_state": "poised_unfired",
            "model": "mWnt_GP",
            "program": "partial"
          }
        ],
        "program_map": [
          {
            "call": "present",
            "model": "KPC_GP"
          },
          {
            "call": "absent",
            "model": "B16_GP"
          },
          {
            "call": "absent",
            "model": "B_ALL_GP"
          },
          {
            "call": "partial",
            "model": "mWnt_GP"
          }
        ],
        "ready_silent_fraction_relations": [
          {
            "pair": "KPC_GP_vs_B16_GP",
            "relation": "<"
          },
          {
            "pair": "KPC_GP_vs_B_ALL_GP",
            "relation": "<"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": "~"
          },
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": "<"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "B_ALL_GP_vs_mWnt_GP",
            "relation": ">"
          }
        ],
        "regulator_tuples": [
          {
            "evidence_basis": "own_abundance",
            "expr_vs_activity": "discordant",
            "gating_step": "late_activation",
            "regulator": "Runx3",
            "role": "activator"
          },
          {
            "evidence_basis": "own_abundance",
            "expr_vs_activity": "concordant",
            "gating_step": "late_activation",
            "regulator": "Smad3",
            "role": "activator"
          },
          {
            "evidence_basis": "own_abundance",
            "expr_vs_activity": "concordant",
            "gating_step": "early_release",
            "regulator": "Eomes",
            "role": "repressor"
          },
          {
            "evidence_basis": "own_abundance",
            "expr_vs_activity": "concordant",
            "gating_step": "early_release",
            "regulator": "Klf2",
            "role": "repressor"
          }
        ]
      },
      "nTurns": 60,
      "checkpointSignals": {
        "clustering": false,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": false,
        "nerveJointAnchor": false,
        "multiomeAssembly": true,
        "depthNormalizedAtac": true,
        "regulonInference": true,
        "cd8SubsetIntegration": true,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": false,
        "orthologProjection": false,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": true,
        "mousePseudobulkDesign": false,
        "cancerEpithelialDenominator": false,
        "geneLevelDe": false,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "trajectoryFile": "scbench-long/trajectories/green_2_tme_context_dependence__claude-sonnet-4-6__pi__r3.json",
      "runId": "claude-sonnet-4-6__pi__green_2_tme_context_dependence__r3",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-gemini-3-1-pro-preview-pi",
      "modelName": "gemini-3.1-pro-preview",
      "harness": "pi",
      "task": "green_2_tme_context_dependence",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 1,
      "passed": false,
      "score": 6,
      "durationS": 1339.51,
      "cost": 0.8394930000000002,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "program_map",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "sig_includes_itgae",
            "role": "gate",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Itgae"
            ]
          },
          {
            "name": "declared_a_known_confounder",
            "role": "gate",
            "field": "$.excluded_confounders[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "cf_signature_gut_contaminant",
            "role": "hard_fail",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "generic_signature_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd103_program_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "<"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "per_model_regulatory_state",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "poised_unfired"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "late_activation"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "closing_committed"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_tuples",
            "kind": "list",
            "matchKey": "regulator",
            "k": 1,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "role": {
                    "op": "equals",
                    "values": [
                      "repressor"
                    ]
                  },
                  "gating_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  },
                  "evidence_basis": {
                    "op": "equals",
                    "values": [
                      "target_program_behavior"
                    ]
                  },
                  "expr_vs_activity": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "locus_differential_gate",
            "role": "gate",
            "field": "locus_differential",
            "kind": "value",
            "expected": "uniform_across_models"
          },
          {
            "name": "formation_limited_by_gate",
            "role": "gate",
            "field": "formation_limited_by",
            "kind": "value",
            "expected": "absent_induction"
          },
          {
            "name": "locus_competent_in_naive_gate",
            "role": "gate",
            "field": "locus_competent_in_naive",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "ready_silent_fraction_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 3,
            "expected": [
              {
                "key": "B16_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_KPC_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "activator_chromatin_footprint_gate",
            "role": "gate",
            "field": "activator_chromatin_footprint",
            "kind": "value",
            "expected": "inconsistent_across_formers"
          },
          {
            "name": "exhaustion_comparable_gate",
            "role": "gate",
            "field": "exhaustion_comparable_across_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_nondeterminant_gate",
            "role": "gate",
            "field": "exhaustion_explains_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_separable_program_gate",
            "role": "gate",
            "field": "exhaustion_separable_program",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "exhaustion_counterexample_gate",
            "role": "gate",
            "field": "exhaustion_counterexample_model",
            "kind": "value",
            "expected": "KPC_GP"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "exhaustion_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "cf_inconsistent_present_without_activator",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "program",
            "entity": "present",
            "valueField": "activator_state",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_no_block_but_absent",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "none",
            "valueField": "program",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_late_block_without_poised_locus",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "late_activation",
            "valueField": "locus_state",
            "expected": [
              "closing_committed",
              "active"
            ]
          },
          {
            "name": "cf_inconsistent_early_block_with_brake_released",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "early_release",
            "valueField": "brake_state",
            "expected": [
              "released"
            ]
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: FAIL\n  x list_match: list_match (answer_field='program_map'): FAIL  (score=0.0)\n  + sig_includes_itgae: passed=True, score=1.0\n  x declared_a_known_confounder: passed=False, score=0.0\n  x list_match: list_match (answer_field='generic_signature_relations'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='cd103_program_relations'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='per_model_regulatory_state'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='regulator_tuples'): FAIL  (score=0.0)\n  + locus_differential_gate: passed=True, score=1.0\n  x formation_limited_by_gate: passed=False, score=0.0\n  + locus_competent_in_naive_gate: passed=True, score=1.0\n  x list_match: list_match (answer_field='ready_silent_fraction_relations'): FAIL  (score=0.0)\n  x activator_chromatin_footprint_gate: passed=False, score=0.0\n  x exhaustion_comparable_gate: passed=False, score=0.0\n  + exhaustion_nondeterminant_gate: passed=True, score=1.0\n  + exhaustion_separable_program_gate: passed=True, score=1.0\n  + exhaustion_counterexample_gate: passed=True, score=1.0\n  x list_match: list_match (answer_field='exhaustion_relations'): FAIL  (score=0.0)\n  x hard_fail 'cf_signature_gut_contaminant': TRIGGERED\n  + hard_fail 'cf_inconsistent_present_without_activator': not triggered\n  + hard_fail 'cf_inconsistent_no_block_but_absent': not triggered\n  + hard_fail 'cf_inconsistent_late_block_without_poised_locus': not triggered\n  + hard_fail 'cf_inconsistent_early_block_with_brake_released': not triggered",
      "metrics": {
        "pass_rule": "all",
        "scoring_count": 17,
        "scoring_passed": 6,
        "scoring_total_score": 6,
        "hard_fail_triggered": [
          "cf_signature_gut_contaminant"
        ]
      },
      "fieldScores": {
        "list_match": 0,
        "sig_includes_itgae": 1,
        "declared_a_known_confounder": 0,
        "locus_differential_gate": 1,
        "formation_limited_by_gate": 0,
        "locus_competent_in_naive_gate": 1,
        "activator_chromatin_footprint_gate": 0,
        "exhaustion_comparable_gate": 0,
        "exhaustion_nondeterminant_gate": 1,
        "exhaustion_separable_program_gate": 1,
        "exhaustion_counterexample_gate": 1
      },
      "graderChecks": [
        {
          "name": "program_map",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "sig_includes_itgae",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "declared_a_known_confounder",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "generic_signature_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "cd103_program_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "per_model_regulatory_state",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "regulator_tuples",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "locus_differential_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "formation_limited_by_gate",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "locus_competent_in_naive_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "ready_silent_fraction_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "activator_chromatin_footprint_gate",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "exhaustion_comparable_gate",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "exhaustion_nondeterminant_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "exhaustion_separable_program_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "exhaustion_counterexample_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "exhaustion_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "cf_signature_gut_contaminant",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "cf_inconsistent_present_without_activator",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_no_block_but_absent",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_late_block_without_poised_locus",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_early_block_with_brake_released",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "activator_chromatin_footprint": "consistent_in_formers",
        "cd103_program_relations": [
          {
            "pair": "B16_GP_vs_KPC_GP",
            "relation": "<"
          },
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": "~"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": "<"
          },
          {
            "pair": "KPC_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "B_ALL_GP_vs_mWnt_GP",
            "relation": "<"
          }
        ],
        "core_residency_signature": [
          "Itga1",
          "Itgae",
          "Plac8",
          "Cd3g",
          "Prkch",
          "Cd160",
          "Abcb1a",
          "AU020206",
          "Lncpint",
          "Mctp2",
          "Smyd3",
          "Plcxd2",
          "Nedd9",
          "Foxn3",
          "Fosb",
          "Cd3e",
          "Pde3b",
          "Ctsw",
          "Foxo1",
          "Epb41"
        ],
        "excluded_confounders": [
          "Tox",
          "Pdcd1",
          "Ctla4",
          "Lag3",
          "Havcr2",
          "Gzmb",
          "Nr4a2",
          "Ifi27l2a",
          "Setbp1"
        ],
        "exhaustion_comparable_across_split": "yes",
        "exhaustion_counterexample_model": "KPC_GP",
        "exhaustion_explains_split": "no",
        "exhaustion_relations": [
          {
            "pair": "B16_GP_vs_KPC_GP",
            "relation": "~"
          },
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "B_ALL_GP_vs_mWnt_GP",
            "relation": ">"
          }
        ],
        "exhaustion_separable_program": "yes",
        "formation_limited_by": "both",
        "generic_signature_relations": [
          {
            "pair": "B16_GP_vs_KPC_GP",
            "relation": "<"
          },
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": "<"
          },
          {
            "pair": "KPC_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "B_ALL_GP_vs_mWnt_GP",
            "relation": "<"
          }
        ],
        "locus_competent_in_naive": "yes",
        "locus_differential": "uniform_across_models",
        "per_model_regulatory_state": [
          {
            "activator_state": "absent",
            "blocking_step": "early_release",
            "brake_state": "engaged",
            "locus_state": "repressed",
            "model": "B16_GP",
            "program": "absent"
          },
          {
            "activator_state": "present",
            "blocking_step": "late_activation",
            "brake_state": "engaged",
            "locus_state": "poised_unfired",
            "model": "B_ALL_GP",
            "program": "absent"
          },
          {
            "activator_state": "present",
            "blocking_step": "none",
            "brake_state": "released",
            "locus_state": "active",
            "model": "KPC_GP",
            "program": "present"
          },
          {
            "activator_state": "partial",
            "blocking_step": "none",
            "brake_state": "released",
            "locus_state": "active",
            "model": "mWnt_GP",
            "program": "partial"
          }
        ],
        "program_map": [
          {
            "call": "absent",
            "model": "B16_GP"
          },
          {
            "call": "absent",
            "model": "B_ALL_GP"
          },
          {
            "call": "present",
            "model": "KPC_GP"
          },
          {
            "call": "partial",
            "model": "mWnt_GP"
          }
        ],
        "ready_silent_fraction_relations": [
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": "<"
          },
          {
            "pair": "B16_GP_vs_KPC_GP",
            "relation": ">"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": "<"
          },
          {
            "pair": "B_ALL_GP_vs_KPC_GP",
            "relation": ">"
          },
          {
            "pair": "B_ALL_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": "<"
          }
        ],
        "regulator_tuples": [
          {
            "evidence_basis": "target_program_behavior",
            "expr_vs_activity": "discordant",
            "gating_step": "early_release",
            "regulator": "Runx3",
            "role": "activator"
          },
          {
            "evidence_basis": "own_abundance",
            "expr_vs_activity": "concordant",
            "gating_step": "late_activation",
            "regulator": "Zeb2",
            "role": "repressor"
          }
        ]
      },
      "nTurns": 49,
      "checkpointSignals": {
        "clustering": false,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": false,
        "nerveJointAnchor": false,
        "multiomeAssembly": true,
        "depthNormalizedAtac": false,
        "regulonInference": true,
        "cd8SubsetIntegration": true,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": false,
        "orthologProjection": false,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": false,
        "mousePseudobulkDesign": false,
        "cancerEpithelialDenominator": false,
        "geneLevelDe": false,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "trajectoryFile": "scbench-long/trajectories/green_2_tme_context_dependence__gemini-3.1-pro-preview__pi__r1.json",
      "runId": "gemini-3.1-pro-preview__pi__green_2_tme_context_dependence__r1",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-gemini-3-1-pro-preview-pi",
      "modelName": "gemini-3.1-pro-preview",
      "harness": "pi",
      "task": "green_2_tme_context_dependence",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 2,
      "passed": false,
      "score": 7,
      "durationS": 940.67,
      "cost": 0.7088425999999999,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "program_map",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "sig_includes_itgae",
            "role": "gate",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Itgae"
            ]
          },
          {
            "name": "declared_a_known_confounder",
            "role": "gate",
            "field": "$.excluded_confounders[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "cf_signature_gut_contaminant",
            "role": "hard_fail",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "generic_signature_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd103_program_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "<"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "per_model_regulatory_state",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "poised_unfired"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "late_activation"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "closing_committed"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_tuples",
            "kind": "list",
            "matchKey": "regulator",
            "k": 1,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "role": {
                    "op": "equals",
                    "values": [
                      "repressor"
                    ]
                  },
                  "gating_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  },
                  "evidence_basis": {
                    "op": "equals",
                    "values": [
                      "target_program_behavior"
                    ]
                  },
                  "expr_vs_activity": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "locus_differential_gate",
            "role": "gate",
            "field": "locus_differential",
            "kind": "value",
            "expected": "uniform_across_models"
          },
          {
            "name": "formation_limited_by_gate",
            "role": "gate",
            "field": "formation_limited_by",
            "kind": "value",
            "expected": "absent_induction"
          },
          {
            "name": "locus_competent_in_naive_gate",
            "role": "gate",
            "field": "locus_competent_in_naive",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "ready_silent_fraction_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 3,
            "expected": [
              {
                "key": "B16_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_KPC_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "activator_chromatin_footprint_gate",
            "role": "gate",
            "field": "activator_chromatin_footprint",
            "kind": "value",
            "expected": "inconsistent_across_formers"
          },
          {
            "name": "exhaustion_comparable_gate",
            "role": "gate",
            "field": "exhaustion_comparable_across_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_nondeterminant_gate",
            "role": "gate",
            "field": "exhaustion_explains_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_separable_program_gate",
            "role": "gate",
            "field": "exhaustion_separable_program",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "exhaustion_counterexample_gate",
            "role": "gate",
            "field": "exhaustion_counterexample_model",
            "kind": "value",
            "expected": "KPC_GP"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "exhaustion_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "cf_inconsistent_present_without_activator",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "program",
            "entity": "present",
            "valueField": "activator_state",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_no_block_but_absent",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "none",
            "valueField": "program",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_late_block_without_poised_locus",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "late_activation",
            "valueField": "locus_state",
            "expected": [
              "closing_committed",
              "active"
            ]
          },
          {
            "name": "cf_inconsistent_early_block_with_brake_released",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "early_release",
            "valueField": "brake_state",
            "expected": [
              "released"
            ]
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: FAIL\n  x list_match: list_match (answer_field='program_map'): FAIL  (score=0.0)\n  + sig_includes_itgae: passed=True, score=1.0\n  x declared_a_known_confounder: passed=False, score=0.0\n  + list_match: list_match (answer_field='generic_signature_relations'): PASS  (score=0.0)\n  x list_match: list_match (answer_field='cd103_program_relations'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='per_model_regulatory_state'): FAIL  (score=0.0)\n  + list_match: list_match (answer_field='regulator_tuples'): PASS  (score=0.0)\n  + locus_differential_gate: passed=True, score=1.0\n  + formation_limited_by_gate: passed=True, score=1.0\n  + locus_competent_in_naive_gate: passed=True, score=1.0\n  x list_match: list_match (answer_field='ready_silent_fraction_relations'): FAIL  (score=0.0)\n  x activator_chromatin_footprint_gate: passed=False, score=0.0\n  + exhaustion_comparable_gate: passed=True, score=1.0\n  + exhaustion_nondeterminant_gate: passed=True, score=1.0\n  + exhaustion_separable_program_gate: passed=True, score=1.0\n  x exhaustion_counterexample_gate: passed=False, score=0.0\n  x list_match: list_match (answer_field='exhaustion_relations'): FAIL  (score=0.0)\n  + hard_fail 'cf_signature_gut_contaminant': not triggered\n  + hard_fail 'cf_inconsistent_present_without_activator': not triggered\n  + hard_fail 'cf_inconsistent_no_block_but_absent': not triggered\n  + hard_fail 'cf_inconsistent_late_block_without_poised_locus': not triggered\n  + hard_fail 'cf_inconsistent_early_block_with_brake_released': not triggered",
      "metrics": {
        "pass_rule": "all",
        "scoring_count": 17,
        "scoring_passed": 9,
        "scoring_total_score": 7,
        "hard_fail_triggered": []
      },
      "fieldScores": {
        "list_match": 0,
        "sig_includes_itgae": 1,
        "declared_a_known_confounder": 0,
        "locus_differential_gate": 1,
        "formation_limited_by_gate": 1,
        "locus_competent_in_naive_gate": 1,
        "activator_chromatin_footprint_gate": 0,
        "exhaustion_comparable_gate": 1,
        "exhaustion_nondeterminant_gate": 1,
        "exhaustion_separable_program_gate": 1,
        "exhaustion_counterexample_gate": 0
      },
      "graderChecks": [
        {
          "name": "program_map",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "sig_includes_itgae",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "declared_a_known_confounder",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "generic_signature_relations",
          "kind": "list_match",
          "passed": true,
          "score": 0
        },
        {
          "name": "cd103_program_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "per_model_regulatory_state",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "regulator_tuples",
          "kind": "list_match",
          "passed": true,
          "score": 0
        },
        {
          "name": "locus_differential_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "formation_limited_by_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "locus_competent_in_naive_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "ready_silent_fraction_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "activator_chromatin_footprint_gate",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "exhaustion_comparable_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "exhaustion_nondeterminant_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "exhaustion_separable_program_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "exhaustion_counterexample_gate",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "exhaustion_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "cf_signature_gut_contaminant",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_present_without_activator",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_no_block_but_absent",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_late_block_without_poised_locus",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_early_block_with_brake_released",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "activator_chromatin_footprint": "absent",
        "cd103_program_relations": [
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_B16_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "mWnt_GP_vs_B16_GP",
            "relation": ">"
          },
          {
            "pair": "mWnt_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": "~"
          }
        ],
        "core_residency_signature": [
          "Itgae"
        ],
        "excluded_confounders": [
          "Tox",
          "Pdcd1",
          "Havcr2",
          "Tigit",
          "Lag3",
          "Ctla4"
        ],
        "exhaustion_comparable_across_split": "no",
        "exhaustion_counterexample_model": "mWnt_GP",
        "exhaustion_explains_split": "no",
        "exhaustion_relations": [
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_B16_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_B_ALL_GP",
            "relation": "~"
          },
          {
            "pair": "mWnt_GP_vs_B16_GP",
            "relation": "<"
          },
          {
            "pair": "mWnt_GP_vs_B_ALL_GP",
            "relation": "<"
          },
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": "<"
          }
        ],
        "exhaustion_separable_program": "yes",
        "formation_limited_by": "absent_induction",
        "generic_signature_relations": [
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": "<"
          },
          {
            "pair": "KPC_GP_vs_B16_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "mWnt_GP_vs_B16_GP",
            "relation": ">"
          },
          {
            "pair": "mWnt_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": ">"
          }
        ],
        "locus_competent_in_naive": "yes",
        "locus_differential": "uniform_across_models",
        "per_model_regulatory_state": [
          {
            "activator_state": "present",
            "blocking_step": "none",
            "brake_state": "released",
            "locus_state": "active",
            "model": "KPC_GP",
            "program": "present"
          },
          {
            "activator_state": "present",
            "blocking_step": "none",
            "brake_state": "partial",
            "locus_state": "active",
            "model": "mWnt_GP",
            "program": "partial"
          },
          {
            "activator_state": "absent",
            "blocking_step": "late_activation",
            "brake_state": "released",
            "locus_state": "poised_unfired",
            "model": "B16_GP",
            "program": "absent"
          },
          {
            "activator_state": "present",
            "blocking_step": "early_release",
            "brake_state": "engaged",
            "locus_state": "poised_unfired",
            "model": "B_ALL_GP",
            "program": "absent"
          }
        ],
        "program_map": [
          {
            "call": "present",
            "model": "KPC_GP"
          },
          {
            "call": "partial",
            "model": "mWnt_GP"
          },
          {
            "call": "absent",
            "model": "B16_GP"
          },
          {
            "call": "absent",
            "model": "B_ALL_GP"
          }
        ],
        "ready_silent_fraction_relations": [
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": "<"
          },
          {
            "pair": "KPC_GP_vs_B16_GP",
            "relation": "<"
          },
          {
            "pair": "KPC_GP_vs_B_ALL_GP",
            "relation": "<"
          },
          {
            "pair": "mWnt_GP_vs_B16_GP",
            "relation": "<"
          },
          {
            "pair": "mWnt_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": ">"
          }
        ],
        "regulator_tuples": [
          {
            "evidence_basis": "target_program_behavior",
            "expr_vs_activity": "concordant",
            "gating_step": "early_release",
            "regulator": "Klf2",
            "role": "repressor"
          },
          {
            "evidence_basis": "target_program_behavior",
            "expr_vs_activity": "discordant",
            "gating_step": "late_activation",
            "regulator": "Tgfbr2",
            "role": "activator"
          }
        ]
      },
      "nTurns": 32,
      "checkpointSignals": {
        "clustering": false,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": false,
        "nerveJointAnchor": false,
        "multiomeAssembly": true,
        "depthNormalizedAtac": true,
        "regulonInference": true,
        "cd8SubsetIntegration": true,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": false,
        "orthologProjection": false,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": false,
        "mousePseudobulkDesign": false,
        "cancerEpithelialDenominator": false,
        "geneLevelDe": false,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "trajectoryFile": "scbench-long/trajectories/green_2_tme_context_dependence__gemini-3.1-pro-preview__pi__r2.json",
      "runId": "gemini-3.1-pro-preview__pi__green_2_tme_context_dependence__r2",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-gemini-3-1-pro-preview-pi",
      "modelName": "gemini-3.1-pro-preview",
      "harness": "pi",
      "task": "green_2_tme_context_dependence",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 3,
      "passed": false,
      "score": 8,
      "durationS": 1101.12,
      "cost": 0.6839697999999999,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "program_map",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "sig_includes_itgae",
            "role": "gate",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Itgae"
            ]
          },
          {
            "name": "declared_a_known_confounder",
            "role": "gate",
            "field": "$.excluded_confounders[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "cf_signature_gut_contaminant",
            "role": "hard_fail",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "generic_signature_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd103_program_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "<"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "per_model_regulatory_state",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "poised_unfired"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "late_activation"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "closing_committed"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_tuples",
            "kind": "list",
            "matchKey": "regulator",
            "k": 1,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "role": {
                    "op": "equals",
                    "values": [
                      "repressor"
                    ]
                  },
                  "gating_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  },
                  "evidence_basis": {
                    "op": "equals",
                    "values": [
                      "target_program_behavior"
                    ]
                  },
                  "expr_vs_activity": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "locus_differential_gate",
            "role": "gate",
            "field": "locus_differential",
            "kind": "value",
            "expected": "uniform_across_models"
          },
          {
            "name": "formation_limited_by_gate",
            "role": "gate",
            "field": "formation_limited_by",
            "kind": "value",
            "expected": "absent_induction"
          },
          {
            "name": "locus_competent_in_naive_gate",
            "role": "gate",
            "field": "locus_competent_in_naive",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "ready_silent_fraction_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 3,
            "expected": [
              {
                "key": "B16_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_KPC_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "activator_chromatin_footprint_gate",
            "role": "gate",
            "field": "activator_chromatin_footprint",
            "kind": "value",
            "expected": "inconsistent_across_formers"
          },
          {
            "name": "exhaustion_comparable_gate",
            "role": "gate",
            "field": "exhaustion_comparable_across_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_nondeterminant_gate",
            "role": "gate",
            "field": "exhaustion_explains_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_separable_program_gate",
            "role": "gate",
            "field": "exhaustion_separable_program",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "exhaustion_counterexample_gate",
            "role": "gate",
            "field": "exhaustion_counterexample_model",
            "kind": "value",
            "expected": "KPC_GP"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "exhaustion_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "cf_inconsistent_present_without_activator",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "program",
            "entity": "present",
            "valueField": "activator_state",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_no_block_but_absent",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "none",
            "valueField": "program",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_late_block_without_poised_locus",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "late_activation",
            "valueField": "locus_state",
            "expected": [
              "closing_committed",
              "active"
            ]
          },
          {
            "name": "cf_inconsistent_early_block_with_brake_released",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "early_release",
            "valueField": "brake_state",
            "expected": [
              "released"
            ]
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: FAIL\n  x list_match: list_match (answer_field='program_map'): FAIL  (score=0.0)\n  + sig_includes_itgae: passed=True, score=1.0\n  + declared_a_known_confounder: passed=True, score=1.0\n  + list_match: list_match (answer_field='generic_signature_relations'): PASS  (score=0.0)\n  x list_match: list_match (answer_field='cd103_program_relations'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='per_model_regulatory_state'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='regulator_tuples'): FAIL  (score=0.0)\n  + locus_differential_gate: passed=True, score=1.0\n  + formation_limited_by_gate: passed=True, score=1.0\n  x locus_competent_in_naive_gate: passed=False, score=0.0\n  x list_match: list_match (answer_field='ready_silent_fraction_relations'): FAIL  (score=0.0)\n  x activator_chromatin_footprint_gate: passed=False, score=0.0\n  + exhaustion_comparable_gate: passed=True, score=1.0\n  + exhaustion_nondeterminant_gate: passed=True, score=1.0\n  + exhaustion_separable_program_gate: passed=True, score=1.0\n  + exhaustion_counterexample_gate: passed=True, score=1.0\n  x list_match: list_match (answer_field='exhaustion_relations'): FAIL  (score=0.0)\n  x hard_fail 'cf_signature_gut_contaminant': TRIGGERED\n  + hard_fail 'cf_inconsistent_present_without_activator': not triggered\n  + hard_fail 'cf_inconsistent_no_block_but_absent': not triggered\n  + hard_fail 'cf_inconsistent_late_block_without_poised_locus': not triggered\n  + hard_fail 'cf_inconsistent_early_block_with_brake_released': not triggered",
      "metrics": {
        "pass_rule": "all",
        "scoring_count": 17,
        "scoring_passed": 9,
        "scoring_total_score": 8,
        "hard_fail_triggered": [
          "cf_signature_gut_contaminant"
        ]
      },
      "fieldScores": {
        "list_match": 0,
        "sig_includes_itgae": 1,
        "declared_a_known_confounder": 1,
        "locus_differential_gate": 1,
        "formation_limited_by_gate": 1,
        "locus_competent_in_naive_gate": 0,
        "activator_chromatin_footprint_gate": 0,
        "exhaustion_comparable_gate": 1,
        "exhaustion_nondeterminant_gate": 1,
        "exhaustion_separable_program_gate": 1,
        "exhaustion_counterexample_gate": 1
      },
      "graderChecks": [
        {
          "name": "program_map",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "sig_includes_itgae",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "declared_a_known_confounder",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "generic_signature_relations",
          "kind": "list_match",
          "passed": true,
          "score": 0
        },
        {
          "name": "cd103_program_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "per_model_regulatory_state",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "regulator_tuples",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "locus_differential_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "formation_limited_by_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "locus_competent_in_naive_gate",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "ready_silent_fraction_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "activator_chromatin_footprint_gate",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "exhaustion_comparable_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "exhaustion_nondeterminant_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "exhaustion_separable_program_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "exhaustion_counterexample_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "exhaustion_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "cf_signature_gut_contaminant",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "cf_inconsistent_present_without_activator",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_no_block_but_absent",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_late_block_without_poised_locus",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_early_block_with_brake_released",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "activator_chromatin_footprint": "consistent_in_formers",
        "cd103_program_relations": [
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_B16_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "mWnt_GP_vs_KPC_GP",
            "relation": "<"
          },
          {
            "pair": "mWnt_GP_vs_B16_GP",
            "relation": ">"
          },
          {
            "pair": "mWnt_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "B16_GP_vs_KPC_GP",
            "relation": "<"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": "<"
          },
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": "~"
          },
          {
            "pair": "B_ALL_GP_vs_KPC_GP",
            "relation": "<"
          },
          {
            "pair": "B_ALL_GP_vs_mWnt_GP",
            "relation": "<"
          },
          {
            "pair": "B_ALL_GP_vs_B16_GP",
            "relation": "~"
          }
        ],
        "core_residency_signature": [
          "Itgae",
          "Itga1",
          "Chn2",
          "Abi3"
        ],
        "excluded_confounders": [
          "Ccr9",
          "Pmepa1",
          "Tox",
          "Xcl1",
          "Cd160",
          "Pdcd1"
        ],
        "exhaustion_comparable_across_split": "no",
        "exhaustion_counterexample_model": "KPC_GP",
        "exhaustion_explains_split": "no",
        "exhaustion_relations": [
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_B16_GP",
            "relation": "<"
          },
          {
            "pair": "KPC_GP_vs_B_ALL_GP",
            "relation": "<"
          },
          {
            "pair": "mWnt_GP_vs_KPC_GP",
            "relation": "<"
          },
          {
            "pair": "mWnt_GP_vs_B16_GP",
            "relation": "<"
          },
          {
            "pair": "mWnt_GP_vs_B_ALL_GP",
            "relation": "<"
          },
          {
            "pair": "B16_GP_vs_KPC_GP",
            "relation": ">"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": "~"
          },
          {
            "pair": "B_ALL_GP_vs_KPC_GP",
            "relation": ">"
          },
          {
            "pair": "B_ALL_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "B_ALL_GP_vs_B16_GP",
            "relation": "~"
          }
        ],
        "exhaustion_separable_program": "yes",
        "formation_limited_by": "absent_induction",
        "generic_signature_relations": [
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_B16_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "mWnt_GP_vs_KPC_GP",
            "relation": "<"
          },
          {
            "pair": "mWnt_GP_vs_B16_GP",
            "relation": ">"
          },
          {
            "pair": "mWnt_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "B16_GP_vs_KPC_GP",
            "relation": "<"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": "<"
          },
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "B_ALL_GP_vs_KPC_GP",
            "relation": "<"
          },
          {
            "pair": "B_ALL_GP_vs_mWnt_GP",
            "relation": "<"
          },
          {
            "pair": "B_ALL_GP_vs_B16_GP",
            "relation": "<"
          }
        ],
        "locus_competent_in_naive": "no",
        "locus_differential": "uniform_across_models",
        "per_model_regulatory_state": [
          {
            "activator_state": "present",
            "blocking_step": "none",
            "brake_state": "released",
            "locus_state": "active",
            "model": "KPC_GP",
            "program": "present"
          },
          {
            "activator_state": "present",
            "blocking_step": "none",
            "brake_state": "released",
            "locus_state": "active",
            "model": "mWnt_GP",
            "program": "partial"
          },
          {
            "activator_state": "present",
            "blocking_step": "late_activation",
            "brake_state": "engaged",
            "locus_state": "poised_unfired",
            "model": "B16_GP",
            "program": "absent"
          },
          {
            "activator_state": "present",
            "blocking_step": "late_activation",
            "brake_state": "engaged",
            "locus_state": "poised_unfired",
            "model": "B_ALL_GP",
            "program": "absent"
          }
        ],
        "program_map": [
          {
            "call": "present",
            "model": "KPC_GP"
          },
          {
            "call": "partial",
            "model": "mWnt_GP"
          },
          {
            "call": "absent",
            "model": "B16_GP"
          },
          {
            "call": "absent",
            "model": "B_ALL_GP"
          }
        ],
        "ready_silent_fraction_relations": [
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": "<"
          },
          {
            "pair": "KPC_GP_vs_B16_GP",
            "relation": "<"
          },
          {
            "pair": "KPC_GP_vs_B_ALL_GP",
            "relation": "<"
          },
          {
            "pair": "mWnt_GP_vs_KPC_GP",
            "relation": ">"
          },
          {
            "pair": "mWnt_GP_vs_B16_GP",
            "relation": "~"
          },
          {
            "pair": "mWnt_GP_vs_B_ALL_GP",
            "relation": "<"
          },
          {
            "pair": "B16_GP_vs_KPC_GP",
            "relation": ">"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": "~"
          },
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": "<"
          },
          {
            "pair": "B_ALL_GP_vs_KPC_GP",
            "relation": ">"
          },
          {
            "pair": "B_ALL_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "B_ALL_GP_vs_B16_GP",
            "relation": ">"
          }
        ],
        "regulator_tuples": [
          {
            "evidence_basis": "target_program_behavior",
            "expr_vs_activity": "discordant",
            "gating_step": "late_activation",
            "regulator": "Runx3",
            "role": "activator"
          }
        ]
      },
      "nTurns": 34,
      "checkpointSignals": {
        "clustering": false,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": false,
        "nerveJointAnchor": false,
        "multiomeAssembly": true,
        "depthNormalizedAtac": false,
        "regulonInference": true,
        "cd8SubsetIntegration": true,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": false,
        "orthologProjection": false,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": false,
        "mousePseudobulkDesign": false,
        "cancerEpithelialDenominator": false,
        "geneLevelDe": false,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "trajectoryFile": "scbench-long/trajectories/green_2_tme_context_dependence__gemini-3.1-pro-preview__pi__r3.json",
      "runId": "gemini-3.1-pro-preview__pi__green_2_tme_context_dependence__r3",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-gemini-3-5-flash-pi",
      "modelName": "gemini-3.5-flash",
      "harness": "pi",
      "task": "green_2_tme_context_dependence",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 1,
      "passed": false,
      "score": 7,
      "durationS": 1183.44,
      "cost": 1.5233244000000001,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "program_map",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "sig_includes_itgae",
            "role": "gate",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Itgae"
            ]
          },
          {
            "name": "declared_a_known_confounder",
            "role": "gate",
            "field": "$.excluded_confounders[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "cf_signature_gut_contaminant",
            "role": "hard_fail",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "generic_signature_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd103_program_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "<"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "per_model_regulatory_state",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "poised_unfired"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "late_activation"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "closing_committed"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_tuples",
            "kind": "list",
            "matchKey": "regulator",
            "k": 1,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "role": {
                    "op": "equals",
                    "values": [
                      "repressor"
                    ]
                  },
                  "gating_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  },
                  "evidence_basis": {
                    "op": "equals",
                    "values": [
                      "target_program_behavior"
                    ]
                  },
                  "expr_vs_activity": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "locus_differential_gate",
            "role": "gate",
            "field": "locus_differential",
            "kind": "value",
            "expected": "uniform_across_models"
          },
          {
            "name": "formation_limited_by_gate",
            "role": "gate",
            "field": "formation_limited_by",
            "kind": "value",
            "expected": "absent_induction"
          },
          {
            "name": "locus_competent_in_naive_gate",
            "role": "gate",
            "field": "locus_competent_in_naive",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "ready_silent_fraction_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 3,
            "expected": [
              {
                "key": "B16_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_KPC_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "activator_chromatin_footprint_gate",
            "role": "gate",
            "field": "activator_chromatin_footprint",
            "kind": "value",
            "expected": "inconsistent_across_formers"
          },
          {
            "name": "exhaustion_comparable_gate",
            "role": "gate",
            "field": "exhaustion_comparable_across_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_nondeterminant_gate",
            "role": "gate",
            "field": "exhaustion_explains_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_separable_program_gate",
            "role": "gate",
            "field": "exhaustion_separable_program",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "exhaustion_counterexample_gate",
            "role": "gate",
            "field": "exhaustion_counterexample_model",
            "kind": "value",
            "expected": "KPC_GP"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "exhaustion_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "cf_inconsistent_present_without_activator",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "program",
            "entity": "present",
            "valueField": "activator_state",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_no_block_but_absent",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "none",
            "valueField": "program",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_late_block_without_poised_locus",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "late_activation",
            "valueField": "locus_state",
            "expected": [
              "closing_committed",
              "active"
            ]
          },
          {
            "name": "cf_inconsistent_early_block_with_brake_released",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "early_release",
            "valueField": "brake_state",
            "expected": [
              "released"
            ]
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: FAIL\n  x list_match: list_match (answer_field='program_map'): FAIL  (score=0.0)\n  + sig_includes_itgae: passed=True, score=1.0\n  x declared_a_known_confounder: passed=False, score=0.0\n  x list_match: list_match (answer_field='generic_signature_relations'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='cd103_program_relations'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='per_model_regulatory_state'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='regulator_tuples'): FAIL  (score=0.0)\n  + locus_differential_gate: passed=True, score=1.0\n  + formation_limited_by_gate: passed=True, score=1.0\n  + locus_competent_in_naive_gate: passed=True, score=1.0\n  x list_match: list_match (answer_field='ready_silent_fraction_relations'): FAIL  (score=0.0)\n  x activator_chromatin_footprint_gate: passed=False, score=0.0\n  + exhaustion_comparable_gate: passed=True, score=1.0\n  + exhaustion_nondeterminant_gate: passed=True, score=1.0\n  + exhaustion_separable_program_gate: passed=True, score=1.0\n  x exhaustion_counterexample_gate: passed=False, score=0.0\n  x list_match: list_match (answer_field='exhaustion_relations'): FAIL  (score=0.0)\n  + hard_fail 'cf_signature_gut_contaminant': not triggered\n  + hard_fail 'cf_inconsistent_present_without_activator': not triggered\n  + hard_fail 'cf_inconsistent_no_block_but_absent': not triggered\n  + hard_fail 'cf_inconsistent_late_block_without_poised_locus': not triggered\n  + hard_fail 'cf_inconsistent_early_block_with_brake_released': not triggered",
      "metrics": {
        "pass_rule": "all",
        "scoring_count": 17,
        "scoring_passed": 7,
        "scoring_total_score": 7,
        "hard_fail_triggered": []
      },
      "fieldScores": {
        "list_match": 0,
        "sig_includes_itgae": 1,
        "declared_a_known_confounder": 0,
        "locus_differential_gate": 1,
        "formation_limited_by_gate": 1,
        "locus_competent_in_naive_gate": 1,
        "activator_chromatin_footprint_gate": 0,
        "exhaustion_comparable_gate": 1,
        "exhaustion_nondeterminant_gate": 1,
        "exhaustion_separable_program_gate": 1,
        "exhaustion_counterexample_gate": 0
      },
      "graderChecks": [
        {
          "name": "program_map",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "sig_includes_itgae",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "declared_a_known_confounder",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "generic_signature_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "cd103_program_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "per_model_regulatory_state",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "regulator_tuples",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "locus_differential_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "formation_limited_by_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "locus_competent_in_naive_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "ready_silent_fraction_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "activator_chromatin_footprint_gate",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "exhaustion_comparable_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "exhaustion_nondeterminant_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "exhaustion_separable_program_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "exhaustion_counterexample_gate",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "exhaustion_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "cf_signature_gut_contaminant",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_present_without_activator",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_no_block_but_absent",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_late_block_without_poised_locus",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_early_block_with_brake_released",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "activator_chromatin_footprint": "consistent_in_formers",
        "cd103_program_relations": [
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": "~"
          },
          {
            "pair": "B16_GP_vs_KPC_GP",
            "relation": "<"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": "<"
          },
          {
            "pair": "B_ALL_GP_vs_KPC_GP",
            "relation": "<"
          },
          {
            "pair": "B_ALL_GP_vs_mWnt_GP",
            "relation": "<"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": ">"
          }
        ],
        "core_residency_signature": [
          "Itgae"
        ],
        "excluded_confounders": [
          "Cd69"
        ],
        "exhaustion_comparable_across_split": "no",
        "exhaustion_counterexample_model": "mWnt_GP",
        "exhaustion_explains_split": "no",
        "exhaustion_relations": [
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "B16_GP_vs_KPC_GP",
            "relation": ">"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "B_ALL_GP_vs_KPC_GP",
            "relation": ">"
          },
          {
            "pair": "B_ALL_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": ">"
          }
        ],
        "exhaustion_separable_program": "yes",
        "formation_limited_by": "absent_induction",
        "generic_signature_relations": [
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": "<"
          },
          {
            "pair": "B16_GP_vs_KPC_GP",
            "relation": "<"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": "<"
          },
          {
            "pair": "B_ALL_GP_vs_KPC_GP",
            "relation": "<"
          },
          {
            "pair": "B_ALL_GP_vs_mWnt_GP",
            "relation": "<"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": "~"
          }
        ],
        "locus_competent_in_naive": "yes",
        "locus_differential": "uniform_across_models",
        "per_model_regulatory_state": [
          {
            "activator_state": "absent",
            "blocking_step": "late_activation",
            "brake_state": "released",
            "locus_state": "poised_unfired",
            "model": "B16_GP",
            "program": "absent"
          },
          {
            "activator_state": "absent",
            "blocking_step": "both_steps",
            "brake_state": "engaged",
            "locus_state": "repressed",
            "model": "B_ALL_GP",
            "program": "absent"
          },
          {
            "activator_state": "present",
            "blocking_step": "none",
            "brake_state": "released",
            "locus_state": "active",
            "model": "KPC_GP",
            "program": "present"
          },
          {
            "activator_state": "present",
            "blocking_step": "early_release",
            "brake_state": "engaged",
            "locus_state": "active",
            "model": "mWnt_GP",
            "program": "partial"
          }
        ],
        "program_map": [
          {
            "call": "absent",
            "model": "B16_GP"
          },
          {
            "call": "absent",
            "model": "B_ALL_GP"
          },
          {
            "call": "present",
            "model": "KPC_GP"
          },
          {
            "call": "partial",
            "model": "mWnt_GP"
          }
        ],
        "ready_silent_fraction_relations": [
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "B16_GP_vs_KPC_GP",
            "relation": ">"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "B_ALL_GP_vs_KPC_GP",
            "relation": ">"
          },
          {
            "pair": "B_ALL_GP_vs_mWnt_GP",
            "relation": "<"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": "<"
          }
        ],
        "regulator_tuples": [
          {
            "evidence_basis": "own_abundance",
            "expr_vs_activity": "concordant",
            "gating_step": "early_release",
            "regulator": "Klf2",
            "role": "repressor"
          },
          {
            "evidence_basis": "target_program_behavior",
            "expr_vs_activity": "discordant",
            "gating_step": "late_activation",
            "regulator": "Batf",
            "role": "activator"
          }
        ]
      },
      "nTurns": 60,
      "checkpointSignals": {
        "clustering": false,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": true,
        "nerveJointAnchor": false,
        "multiomeAssembly": true,
        "depthNormalizedAtac": true,
        "regulonInference": true,
        "cd8SubsetIntegration": true,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": true,
        "orthologProjection": false,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": true,
        "enrichmentFamilyRanking": true,
        "mousePseudobulkDesign": false,
        "cancerEpithelialDenominator": true,
        "geneLevelDe": false,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "trajectoryFile": "scbench-long/trajectories/green_2_tme_context_dependence__gemini-3.5-flash__pi__r1.json",
      "runId": "gemini-3.5-flash__pi__green_2_tme_context_dependence__r1",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-gemini-3-5-flash-pi",
      "modelName": "gemini-3.5-flash",
      "harness": "pi",
      "task": "green_2_tme_context_dependence",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 2,
      "passed": false,
      "score": 7,
      "durationS": 1758.74,
      "cost": 1.6195333500000004,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "program_map",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "sig_includes_itgae",
            "role": "gate",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Itgae"
            ]
          },
          {
            "name": "declared_a_known_confounder",
            "role": "gate",
            "field": "$.excluded_confounders[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "cf_signature_gut_contaminant",
            "role": "hard_fail",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "generic_signature_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd103_program_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "<"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "per_model_regulatory_state",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "poised_unfired"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "late_activation"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "closing_committed"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_tuples",
            "kind": "list",
            "matchKey": "regulator",
            "k": 1,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "role": {
                    "op": "equals",
                    "values": [
                      "repressor"
                    ]
                  },
                  "gating_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  },
                  "evidence_basis": {
                    "op": "equals",
                    "values": [
                      "target_program_behavior"
                    ]
                  },
                  "expr_vs_activity": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "locus_differential_gate",
            "role": "gate",
            "field": "locus_differential",
            "kind": "value",
            "expected": "uniform_across_models"
          },
          {
            "name": "formation_limited_by_gate",
            "role": "gate",
            "field": "formation_limited_by",
            "kind": "value",
            "expected": "absent_induction"
          },
          {
            "name": "locus_competent_in_naive_gate",
            "role": "gate",
            "field": "locus_competent_in_naive",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "ready_silent_fraction_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 3,
            "expected": [
              {
                "key": "B16_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_KPC_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "activator_chromatin_footprint_gate",
            "role": "gate",
            "field": "activator_chromatin_footprint",
            "kind": "value",
            "expected": "inconsistent_across_formers"
          },
          {
            "name": "exhaustion_comparable_gate",
            "role": "gate",
            "field": "exhaustion_comparable_across_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_nondeterminant_gate",
            "role": "gate",
            "field": "exhaustion_explains_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_separable_program_gate",
            "role": "gate",
            "field": "exhaustion_separable_program",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "exhaustion_counterexample_gate",
            "role": "gate",
            "field": "exhaustion_counterexample_model",
            "kind": "value",
            "expected": "KPC_GP"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "exhaustion_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "cf_inconsistent_present_without_activator",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "program",
            "entity": "present",
            "valueField": "activator_state",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_no_block_but_absent",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "none",
            "valueField": "program",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_late_block_without_poised_locus",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "late_activation",
            "valueField": "locus_state",
            "expected": [
              "closing_committed",
              "active"
            ]
          },
          {
            "name": "cf_inconsistent_early_block_with_brake_released",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "early_release",
            "valueField": "brake_state",
            "expected": [
              "released"
            ]
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: FAIL\n  x list_match: list_match (answer_field='program_map'): FAIL  (score=0.0)\n  + sig_includes_itgae: passed=True, score=1.0\n  x declared_a_known_confounder: passed=False, score=0.0\n  x list_match: list_match (answer_field='generic_signature_relations'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='cd103_program_relations'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='per_model_regulatory_state'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='regulator_tuples'): FAIL  (score=0.0)\n  + locus_differential_gate: passed=True, score=1.0\n  + formation_limited_by_gate: passed=True, score=1.0\n  + locus_competent_in_naive_gate: passed=True, score=1.0\n  x list_match: list_match (answer_field='ready_silent_fraction_relations'): FAIL  (score=0.0)\n  x activator_chromatin_footprint_gate: passed=False, score=0.0\n  + exhaustion_comparable_gate: passed=True, score=1.0\n  + exhaustion_nondeterminant_gate: passed=True, score=1.0\n  + exhaustion_separable_program_gate: passed=True, score=1.0\n  x exhaustion_counterexample_gate: passed=False, score=0.0\n  x list_match: list_match (answer_field='exhaustion_relations'): FAIL  (score=0.0)\n  x hard_fail 'cf_signature_gut_contaminant': TRIGGERED\n  + hard_fail 'cf_inconsistent_present_without_activator': not triggered\n  + hard_fail 'cf_inconsistent_no_block_but_absent': not triggered\n  + hard_fail 'cf_inconsistent_late_block_without_poised_locus': not triggered\n  + hard_fail 'cf_inconsistent_early_block_with_brake_released': not triggered",
      "metrics": {
        "pass_rule": "all",
        "scoring_count": 17,
        "scoring_passed": 7,
        "scoring_total_score": 7,
        "hard_fail_triggered": [
          "cf_signature_gut_contaminant"
        ]
      },
      "fieldScores": {
        "list_match": 0,
        "sig_includes_itgae": 1,
        "declared_a_known_confounder": 0,
        "locus_differential_gate": 1,
        "formation_limited_by_gate": 1,
        "locus_competent_in_naive_gate": 1,
        "activator_chromatin_footprint_gate": 0,
        "exhaustion_comparable_gate": 1,
        "exhaustion_nondeterminant_gate": 1,
        "exhaustion_separable_program_gate": 1,
        "exhaustion_counterexample_gate": 0
      },
      "graderChecks": [
        {
          "name": "program_map",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "sig_includes_itgae",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "declared_a_known_confounder",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "generic_signature_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "cd103_program_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "per_model_regulatory_state",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "regulator_tuples",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "locus_differential_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "formation_limited_by_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "locus_competent_in_naive_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "ready_silent_fraction_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "activator_chromatin_footprint_gate",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "exhaustion_comparable_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "exhaustion_nondeterminant_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "exhaustion_separable_program_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "exhaustion_counterexample_gate",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "exhaustion_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "cf_signature_gut_contaminant",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "cf_inconsistent_present_without_activator",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_no_block_but_absent",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_late_block_without_poised_locus",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_early_block_with_brake_released",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "activator_chromatin_footprint": "consistent_in_formers",
        "cd103_program_relations": [
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": "~"
          },
          {
            "pair": "B16_GP_vs_KPC_GP",
            "relation": "<"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": "<"
          },
          {
            "pair": "B_ALL_GP_vs_KPC_GP",
            "relation": "<"
          },
          {
            "pair": "B_ALL_GP_vs_mWnt_GP",
            "relation": "<"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": ">"
          }
        ],
        "core_residency_signature": [
          "Itgae",
          "Itga1"
        ],
        "excluded_confounders": [
          "Pdcd1",
          "Havcr2",
          "Lag3",
          "Tox",
          "Entpd1",
          "Ctla4"
        ],
        "exhaustion_comparable_across_split": "no",
        "exhaustion_counterexample_model": "mWnt_GP",
        "exhaustion_explains_split": "no",
        "exhaustion_relations": [
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": "<"
          },
          {
            "pair": "B16_GP_vs_KPC_GP",
            "relation": "~"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "B_ALL_GP_vs_KPC_GP",
            "relation": ">"
          },
          {
            "pair": "B_ALL_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": ">"
          }
        ],
        "exhaustion_separable_program": "yes",
        "formation_limited_by": "absent_induction",
        "generic_signature_relations": [
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "B16_GP_vs_KPC_GP",
            "relation": "<"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": "<"
          },
          {
            "pair": "B_ALL_GP_vs_KPC_GP",
            "relation": "<"
          },
          {
            "pair": "B_ALL_GP_vs_mWnt_GP",
            "relation": "<"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": ">"
          }
        ],
        "locus_competent_in_naive": "yes",
        "locus_differential": "uniform_across_models",
        "per_model_regulatory_state": [
          {
            "activator_state": "absent",
            "blocking_step": "late_activation",
            "brake_state": "released",
            "locus_state": "poised_unfired",
            "model": "B16_GP",
            "program": "absent"
          },
          {
            "activator_state": "absent",
            "blocking_step": "both_steps",
            "brake_state": "engaged",
            "locus_state": "poised_unfired",
            "model": "B_ALL_GP",
            "program": "absent"
          },
          {
            "activator_state": "present",
            "blocking_step": "none",
            "brake_state": "released",
            "locus_state": "active",
            "model": "KPC_GP",
            "program": "present"
          },
          {
            "activator_state": "present",
            "blocking_step": "early_release",
            "brake_state": "engaged",
            "locus_state": "active",
            "model": "mWnt_GP",
            "program": "partial"
          }
        ],
        "program_map": [
          {
            "call": "absent",
            "model": "B16_GP"
          },
          {
            "call": "absent",
            "model": "B_ALL_GP"
          },
          {
            "call": "present",
            "model": "KPC_GP"
          },
          {
            "call": "partial",
            "model": "mWnt_GP"
          }
        ],
        "ready_silent_fraction_relations": [
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "B16_GP_vs_KPC_GP",
            "relation": ">"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "B_ALL_GP_vs_KPC_GP",
            "relation": "~"
          },
          {
            "pair": "B_ALL_GP_vs_mWnt_GP",
            "relation": "~"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": "~"
          }
        ],
        "regulator_tuples": [
          {
            "evidence_basis": "own_abundance",
            "expr_vs_activity": "concordant",
            "gating_step": "early_release",
            "regulator": "Klf2",
            "role": "repressor"
          },
          {
            "evidence_basis": "own_abundance",
            "expr_vs_activity": "concordant",
            "gating_step": "late_activation",
            "regulator": "Smad3",
            "role": "activator"
          }
        ]
      },
      "nTurns": 78,
      "checkpointSignals": {
        "clustering": false,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": false,
        "nerveJointAnchor": false,
        "multiomeAssembly": true,
        "depthNormalizedAtac": true,
        "regulonInference": true,
        "cd8SubsetIntegration": true,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": false,
        "orthologProjection": false,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": false,
        "mousePseudobulkDesign": true,
        "cancerEpithelialDenominator": false,
        "geneLevelDe": true,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "trajectoryFile": "scbench-long/trajectories/green_2_tme_context_dependence__gemini-3.5-flash__pi__r2.json",
      "runId": "gemini-3.5-flash__pi__green_2_tme_context_dependence__r2",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-gemini-3-5-flash-pi",
      "modelName": "gemini-3.5-flash",
      "harness": "pi",
      "task": "green_2_tme_context_dependence",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 3,
      "passed": false,
      "score": 7,
      "durationS": 1162.67,
      "cost": 2.7983538,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "program_map",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "sig_includes_itgae",
            "role": "gate",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Itgae"
            ]
          },
          {
            "name": "declared_a_known_confounder",
            "role": "gate",
            "field": "$.excluded_confounders[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "cf_signature_gut_contaminant",
            "role": "hard_fail",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "generic_signature_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd103_program_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "<"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "per_model_regulatory_state",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "poised_unfired"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "late_activation"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "closing_committed"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_tuples",
            "kind": "list",
            "matchKey": "regulator",
            "k": 1,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "role": {
                    "op": "equals",
                    "values": [
                      "repressor"
                    ]
                  },
                  "gating_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  },
                  "evidence_basis": {
                    "op": "equals",
                    "values": [
                      "target_program_behavior"
                    ]
                  },
                  "expr_vs_activity": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "locus_differential_gate",
            "role": "gate",
            "field": "locus_differential",
            "kind": "value",
            "expected": "uniform_across_models"
          },
          {
            "name": "formation_limited_by_gate",
            "role": "gate",
            "field": "formation_limited_by",
            "kind": "value",
            "expected": "absent_induction"
          },
          {
            "name": "locus_competent_in_naive_gate",
            "role": "gate",
            "field": "locus_competent_in_naive",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "ready_silent_fraction_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 3,
            "expected": [
              {
                "key": "B16_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_KPC_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "activator_chromatin_footprint_gate",
            "role": "gate",
            "field": "activator_chromatin_footprint",
            "kind": "value",
            "expected": "inconsistent_across_formers"
          },
          {
            "name": "exhaustion_comparable_gate",
            "role": "gate",
            "field": "exhaustion_comparable_across_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_nondeterminant_gate",
            "role": "gate",
            "field": "exhaustion_explains_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_separable_program_gate",
            "role": "gate",
            "field": "exhaustion_separable_program",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "exhaustion_counterexample_gate",
            "role": "gate",
            "field": "exhaustion_counterexample_model",
            "kind": "value",
            "expected": "KPC_GP"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "exhaustion_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "cf_inconsistent_present_without_activator",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "program",
            "entity": "present",
            "valueField": "activator_state",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_no_block_but_absent",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "none",
            "valueField": "program",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_late_block_without_poised_locus",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "late_activation",
            "valueField": "locus_state",
            "expected": [
              "closing_committed",
              "active"
            ]
          },
          {
            "name": "cf_inconsistent_early_block_with_brake_released",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "early_release",
            "valueField": "brake_state",
            "expected": [
              "released"
            ]
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: FAIL\n  x list_match: list_match (answer_field='program_map'): FAIL  (score=0.0)\n  + sig_includes_itgae: passed=True, score=1.0\n  x declared_a_known_confounder: passed=False, score=0.0\n  + list_match: list_match (answer_field='generic_signature_relations'): PASS  (score=0.0)\n  x list_match: list_match (answer_field='cd103_program_relations'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='per_model_regulatory_state'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='regulator_tuples'): FAIL  (score=0.0)\n  + locus_differential_gate: passed=True, score=1.0\n  + formation_limited_by_gate: passed=True, score=1.0\n  + locus_competent_in_naive_gate: passed=True, score=1.0\n  x list_match: list_match (answer_field='ready_silent_fraction_relations'): FAIL  (score=0.0)\n  x activator_chromatin_footprint_gate: passed=False, score=0.0\n  + exhaustion_comparable_gate: passed=True, score=1.0\n  + exhaustion_nondeterminant_gate: passed=True, score=1.0\n  + exhaustion_separable_program_gate: passed=True, score=1.0\n  x exhaustion_counterexample_gate: passed=False, score=0.0\n  + list_match: list_match (answer_field='exhaustion_relations'): PASS  (score=0.0)\n  + hard_fail 'cf_signature_gut_contaminant': not triggered\n  + hard_fail 'cf_inconsistent_present_without_activator': not triggered\n  + hard_fail 'cf_inconsistent_no_block_but_absent': not triggered\n  + hard_fail 'cf_inconsistent_late_block_without_poised_locus': not triggered\n  + hard_fail 'cf_inconsistent_early_block_with_brake_released': not triggered",
      "metrics": {
        "pass_rule": "all",
        "scoring_count": 17,
        "scoring_passed": 9,
        "scoring_total_score": 7,
        "hard_fail_triggered": []
      },
      "fieldScores": {
        "list_match": 0,
        "sig_includes_itgae": 1,
        "declared_a_known_confounder": 0,
        "locus_differential_gate": 1,
        "formation_limited_by_gate": 1,
        "locus_competent_in_naive_gate": 1,
        "activator_chromatin_footprint_gate": 0,
        "exhaustion_comparable_gate": 1,
        "exhaustion_nondeterminant_gate": 1,
        "exhaustion_separable_program_gate": 1,
        "exhaustion_counterexample_gate": 0
      },
      "graderChecks": [
        {
          "name": "program_map",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "sig_includes_itgae",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "declared_a_known_confounder",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "generic_signature_relations",
          "kind": "list_match",
          "passed": true,
          "score": 0
        },
        {
          "name": "cd103_program_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "per_model_regulatory_state",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "regulator_tuples",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "locus_differential_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "formation_limited_by_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "locus_competent_in_naive_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "ready_silent_fraction_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "activator_chromatin_footprint_gate",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "exhaustion_comparable_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "exhaustion_nondeterminant_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "exhaustion_separable_program_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "exhaustion_counterexample_gate",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "exhaustion_relations",
          "kind": "list_match",
          "passed": true,
          "score": 0
        },
        {
          "name": "cf_signature_gut_contaminant",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_present_without_activator",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_no_block_but_absent",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_late_block_without_poised_locus",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_early_block_with_brake_released",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "activator_chromatin_footprint": "consistent_in_formers",
        "cd103_program_relations": [
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "mWnt_GP_vs_KPC_GP",
            "relation": "<"
          },
          {
            "pair": "KPC_GP_vs_B16_GP",
            "relation": ">"
          },
          {
            "pair": "B16_GP_vs_KPC_GP",
            "relation": "<"
          },
          {
            "pair": "KPC_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "B_ALL_GP_vs_KPC_GP",
            "relation": "<"
          },
          {
            "pair": "mWnt_GP_vs_B16_GP",
            "relation": ">"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": "<"
          },
          {
            "pair": "mWnt_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "B_ALL_GP_vs_mWnt_GP",
            "relation": "<"
          },
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": "~"
          },
          {
            "pair": "B_ALL_GP_vs_B16_GP",
            "relation": "~"
          }
        ],
        "core_residency_signature": [
          "Itgae",
          "Cxcr6",
          "Xcl1",
          "Cish",
          "Rgs1"
        ],
        "excluded_confounders": [
          "Cd69",
          "Fos",
          "Jun",
          "Nr4a1"
        ],
        "exhaustion_comparable_across_split": "no",
        "exhaustion_counterexample_model": "mWnt_GP",
        "exhaustion_explains_split": "no",
        "exhaustion_relations": [
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "mWnt_GP_vs_KPC_GP",
            "relation": "<"
          },
          {
            "pair": "KPC_GP_vs_B16_GP",
            "relation": "~"
          },
          {
            "pair": "B16_GP_vs_KPC_GP",
            "relation": "~"
          },
          {
            "pair": "KPC_GP_vs_B_ALL_GP",
            "relation": "<"
          },
          {
            "pair": "B_ALL_GP_vs_KPC_GP",
            "relation": ">"
          },
          {
            "pair": "mWnt_GP_vs_B16_GP",
            "relation": "<"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "mWnt_GP_vs_B_ALL_GP",
            "relation": "<"
          },
          {
            "pair": "B_ALL_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": "<"
          },
          {
            "pair": "B_ALL_GP_vs_B16_GP",
            "relation": ">"
          }
        ],
        "exhaustion_separable_program": "yes",
        "formation_limited_by": "absent_induction",
        "generic_signature_relations": [
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "mWnt_GP_vs_KPC_GP",
            "relation": "<"
          },
          {
            "pair": "KPC_GP_vs_B16_GP",
            "relation": ">"
          },
          {
            "pair": "B16_GP_vs_KPC_GP",
            "relation": "<"
          },
          {
            "pair": "KPC_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "B_ALL_GP_vs_KPC_GP",
            "relation": "<"
          },
          {
            "pair": "mWnt_GP_vs_B16_GP",
            "relation": "~"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": "~"
          },
          {
            "pair": "mWnt_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "B_ALL_GP_vs_mWnt_GP",
            "relation": "<"
          },
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "B_ALL_GP_vs_B16_GP",
            "relation": "<"
          }
        ],
        "locus_competent_in_naive": "yes",
        "locus_differential": "uniform_across_models",
        "per_model_regulatory_state": [
          {
            "activator_state": "present",
            "blocking_step": "none",
            "brake_state": "released",
            "locus_state": "active",
            "model": "KPC_GP",
            "program": "present"
          },
          {
            "activator_state": "partial",
            "blocking_step": "none",
            "brake_state": "partial",
            "locus_state": "active",
            "model": "mWnt_GP",
            "program": "partial"
          },
          {
            "activator_state": "absent",
            "blocking_step": "late_activation",
            "brake_state": "released",
            "locus_state": "poised_unfired",
            "model": "B16_GP",
            "program": "absent"
          },
          {
            "activator_state": "absent",
            "blocking_step": "both_steps",
            "brake_state": "engaged",
            "locus_state": "repressed",
            "model": "B_ALL_GP",
            "program": "absent"
          }
        ],
        "program_map": [
          {
            "call": "present",
            "model": "KPC_GP"
          },
          {
            "call": "partial",
            "model": "mWnt_GP"
          },
          {
            "call": "absent",
            "model": "B16_GP"
          },
          {
            "call": "absent",
            "model": "B_ALL_GP"
          }
        ],
        "ready_silent_fraction_relations": [
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": "<"
          },
          {
            "pair": "mWnt_GP_vs_KPC_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_B16_GP",
            "relation": "<"
          },
          {
            "pair": "B16_GP_vs_KPC_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_B_ALL_GP",
            "relation": "<"
          },
          {
            "pair": "B_ALL_GP_vs_KPC_GP",
            "relation": ">"
          },
          {
            "pair": "mWnt_GP_vs_B16_GP",
            "relation": "<"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "mWnt_GP_vs_B_ALL_GP",
            "relation": "~"
          },
          {
            "pair": "B_ALL_GP_vs_mWnt_GP",
            "relation": "~"
          },
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "B_ALL_GP_vs_B16_GP",
            "relation": "<"
          }
        ],
        "regulator_tuples": [
          {
            "evidence_basis": "own_abundance",
            "expr_vs_activity": "concordant",
            "gating_step": "early_release",
            "regulator": "Klf2",
            "role": "repressor"
          },
          {
            "evidence_basis": "target_program_behavior",
            "expr_vs_activity": "discordant",
            "gating_step": "late_activation",
            "regulator": "Batf",
            "role": "activator"
          },
          {
            "evidence_basis": "target_program_behavior",
            "expr_vs_activity": "discordant",
            "gating_step": "late_activation",
            "regulator": "Smad2",
            "role": "activator"
          }
        ]
      },
      "nTurns": 103,
      "checkpointSignals": {
        "clustering": true,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": true,
        "nerveJointAnchor": false,
        "multiomeAssembly": true,
        "depthNormalizedAtac": false,
        "regulonInference": true,
        "cd8SubsetIntegration": true,
        "perPatientEnrichment": true,
        "tcrRichnessMetrics": true,
        "orthologProjection": false,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": true,
        "enrichmentFamilyRanking": true,
        "mousePseudobulkDesign": true,
        "cancerEpithelialDenominator": false,
        "geneLevelDe": true,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "trajectoryFile": "scbench-long/trajectories/green_2_tme_context_dependence__gemini-3.5-flash__pi__r3.json",
      "runId": "gemini-3.5-flash__pi__green_2_tme_context_dependence__r3",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-glm-5p2-pi",
      "modelName": "glm-5p2",
      "harness": "pi",
      "task": "green_2_tme_context_dependence",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 1,
      "passed": false,
      "score": 4,
      "durationS": 4566.93,
      "cost": 0.9068819500000002,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "program_map",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "sig_includes_itgae",
            "role": "gate",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Itgae"
            ]
          },
          {
            "name": "declared_a_known_confounder",
            "role": "gate",
            "field": "$.excluded_confounders[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "cf_signature_gut_contaminant",
            "role": "hard_fail",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "generic_signature_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd103_program_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "<"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "per_model_regulatory_state",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "poised_unfired"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "late_activation"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "closing_committed"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_tuples",
            "kind": "list",
            "matchKey": "regulator",
            "k": 1,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "role": {
                    "op": "equals",
                    "values": [
                      "repressor"
                    ]
                  },
                  "gating_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  },
                  "evidence_basis": {
                    "op": "equals",
                    "values": [
                      "target_program_behavior"
                    ]
                  },
                  "expr_vs_activity": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "locus_differential_gate",
            "role": "gate",
            "field": "locus_differential",
            "kind": "value",
            "expected": "uniform_across_models"
          },
          {
            "name": "formation_limited_by_gate",
            "role": "gate",
            "field": "formation_limited_by",
            "kind": "value",
            "expected": "absent_induction"
          },
          {
            "name": "locus_competent_in_naive_gate",
            "role": "gate",
            "field": "locus_competent_in_naive",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "ready_silent_fraction_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 3,
            "expected": [
              {
                "key": "B16_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_KPC_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "activator_chromatin_footprint_gate",
            "role": "gate",
            "field": "activator_chromatin_footprint",
            "kind": "value",
            "expected": "inconsistent_across_formers"
          },
          {
            "name": "exhaustion_comparable_gate",
            "role": "gate",
            "field": "exhaustion_comparable_across_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_nondeterminant_gate",
            "role": "gate",
            "field": "exhaustion_explains_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_separable_program_gate",
            "role": "gate",
            "field": "exhaustion_separable_program",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "exhaustion_counterexample_gate",
            "role": "gate",
            "field": "exhaustion_counterexample_model",
            "kind": "value",
            "expected": "KPC_GP"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "exhaustion_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "cf_inconsistent_present_without_activator",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "program",
            "entity": "present",
            "valueField": "activator_state",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_no_block_but_absent",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "none",
            "valueField": "program",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_late_block_without_poised_locus",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "late_activation",
            "valueField": "locus_state",
            "expected": [
              "closing_committed",
              "active"
            ]
          },
          {
            "name": "cf_inconsistent_early_block_with_brake_released",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "early_release",
            "valueField": "brake_state",
            "expected": [
              "released"
            ]
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: FAIL\n  x list_match: list_match (answer_field='program_map'): FAIL  (score=0.0)\n  + sig_includes_itgae: passed=True, score=1.0\n  x declared_a_known_confounder: passed=False, score=0.0\n  x list_match: list_match (answer_field='generic_signature_relations'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='cd103_program_relations'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='per_model_regulatory_state'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='regulator_tuples'): FAIL  (score=0.0)\n  x locus_differential_gate: passed=False, score=0.0\n  x formation_limited_by_gate: passed=False, score=0.0\n  + locus_competent_in_naive_gate: passed=True, score=1.0\n  x list_match: list_match (answer_field='ready_silent_fraction_relations'): FAIL  (score=0.0)\n  x activator_chromatin_footprint_gate: passed=False, score=0.0\n  x exhaustion_comparable_gate: passed=False, score=0.0\n  + exhaustion_nondeterminant_gate: passed=True, score=1.0\n  + exhaustion_separable_program_gate: passed=True, score=1.0\n  x exhaustion_counterexample_gate: passed=False, score=0.0\n  x list_match: list_match (answer_field='exhaustion_relations'): FAIL  (score=0.0)\n  x hard_fail 'cf_signature_gut_contaminant': TRIGGERED\n  + hard_fail 'cf_inconsistent_present_without_activator': not triggered\n  + hard_fail 'cf_inconsistent_no_block_but_absent': not triggered\n  + hard_fail 'cf_inconsistent_late_block_without_poised_locus': not triggered\n  + hard_fail 'cf_inconsistent_early_block_with_brake_released': not triggered",
      "metrics": {
        "pass_rule": "all",
        "scoring_count": 17,
        "scoring_passed": 4,
        "scoring_total_score": 4,
        "hard_fail_triggered": [
          "cf_signature_gut_contaminant"
        ]
      },
      "fieldScores": {
        "list_match": 0,
        "sig_includes_itgae": 1,
        "declared_a_known_confounder": 0,
        "locus_differential_gate": 0,
        "formation_limited_by_gate": 0,
        "locus_competent_in_naive_gate": 1,
        "activator_chromatin_footprint_gate": 0,
        "exhaustion_comparable_gate": 0,
        "exhaustion_nondeterminant_gate": 1,
        "exhaustion_separable_program_gate": 1,
        "exhaustion_counterexample_gate": 0
      },
      "graderChecks": [
        {
          "name": "program_map",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "sig_includes_itgae",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "declared_a_known_confounder",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "generic_signature_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "cd103_program_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "per_model_regulatory_state",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "regulator_tuples",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "locus_differential_gate",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "formation_limited_by_gate",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "locus_competent_in_naive_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "ready_silent_fraction_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "activator_chromatin_footprint_gate",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "exhaustion_comparable_gate",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "exhaustion_nondeterminant_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "exhaustion_separable_program_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "exhaustion_counterexample_gate",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "exhaustion_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "cf_signature_gut_contaminant",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "cf_inconsistent_present_without_activator",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_no_block_but_absent",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_late_block_without_poised_locus",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_early_block_with_brake_released",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "activator_chromatin_footprint": "consistent_in_formers",
        "cd103_program_relations": [
          {
            "pair": "KPC_GP_vs_B16_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": "~"
          },
          {
            "pair": "KPC_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": "<"
          },
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "mWnt_GP_vs_B_ALL_GP",
            "relation": ">"
          }
        ],
        "core_residency_signature": [
          "Itgae",
          "Cd69",
          "Itga1",
          "Cdh1",
          "Cxcr6",
          "Itgb7",
          "Ccr9",
          "Bcl2",
          "Il7r"
        ],
        "excluded_confounders": [
          "Tox",
          "Pdcd1",
          "Havcr2",
          "Lag3",
          "Ctla4",
          "Tigit",
          "Gzmb",
          "Gzmk",
          "Prf1",
          "Ifng",
          "Nkg7",
          "Klrg1"
        ],
        "exhaustion_comparable_across_split": "yes",
        "exhaustion_counterexample_model": "mWnt_GP",
        "exhaustion_explains_split": "no",
        "exhaustion_relations": [
          {
            "pair": "KPC_GP_vs_B16_GP",
            "relation": "~"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "mWnt_GP_vs_B_ALL_GP",
            "relation": "<"
          }
        ],
        "exhaustion_separable_program": "yes",
        "formation_limited_by": "both",
        "generic_signature_relations": [
          {
            "pair": "KPC_GP_vs_B16_GP",
            "relation": "<"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_B_ALL_GP",
            "relation": "<"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": "~"
          },
          {
            "pair": "mWnt_GP_vs_B_ALL_GP",
            "relation": "<"
          }
        ],
        "locus_competent_in_naive": "yes",
        "locus_differential": "model_specific_element",
        "per_model_regulatory_state": [
          {
            "activator_state": "present",
            "blocking_step": "none",
            "brake_state": "released",
            "locus_state": "active",
            "model": "KPC_GP",
            "program": "present"
          },
          {
            "activator_state": "present",
            "blocking_step": "both_steps",
            "brake_state": "partial",
            "locus_state": "active",
            "model": "mWnt_GP",
            "program": "partial"
          },
          {
            "activator_state": "present",
            "blocking_step": "late_activation",
            "brake_state": "released",
            "locus_state": "repressed",
            "model": "B16_GP",
            "program": "absent"
          },
          {
            "activator_state": "present",
            "blocking_step": "late_activation",
            "brake_state": "partial",
            "locus_state": "poised_unfired",
            "model": "B_ALL_GP",
            "program": "absent"
          }
        ],
        "program_map": [
          {
            "call": "present",
            "model": "KPC_GP"
          },
          {
            "call": "partial",
            "model": "mWnt_GP"
          },
          {
            "call": "absent",
            "model": "B16_GP"
          },
          {
            "call": "absent",
            "model": "B_ALL_GP"
          }
        ],
        "ready_silent_fraction_relations": [
          {
            "pair": "KPC_GP_vs_B16_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": "<"
          },
          {
            "pair": "KPC_GP_vs_B_ALL_GP",
            "relation": "<"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": "<"
          },
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": "<"
          },
          {
            "pair": "mWnt_GP_vs_B_ALL_GP",
            "relation": "~"
          }
        ],
        "regulator_tuples": [
          {
            "evidence_basis": "own_abundance",
            "expr_vs_activity": "concordant",
            "gating_step": "early_release",
            "regulator": "Klf2",
            "role": "repressor"
          },
          {
            "evidence_basis": "own_abundance",
            "expr_vs_activity": "concordant",
            "gating_step": "early_release",
            "regulator": "S1pr1",
            "role": "repressor"
          },
          {
            "evidence_basis": "own_abundance",
            "expr_vs_activity": "discordant",
            "gating_step": "late_activation",
            "regulator": "Bhlhe40",
            "role": "activator"
          },
          {
            "evidence_basis": "own_abundance",
            "expr_vs_activity": "discordant",
            "gating_step": "late_activation",
            "regulator": "Runx3",
            "role": "activator"
          },
          {
            "evidence_basis": "own_abundance",
            "expr_vs_activity": "concordant",
            "gating_step": "none",
            "regulator": "Zfp683",
            "role": "none"
          }
        ]
      },
      "nTurns": 122,
      "checkpointSignals": {
        "clustering": false,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": false,
        "nerveJointAnchor": false,
        "multiomeAssembly": true,
        "depthNormalizedAtac": true,
        "regulonInference": true,
        "cd8SubsetIntegration": true,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": false,
        "orthologProjection": true,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": false,
        "mousePseudobulkDesign": false,
        "cancerEpithelialDenominator": false,
        "geneLevelDe": false,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "trajectoryFile": "scbench-long/trajectories/green_2_tme_context_dependence__glm-5p2__pi__r1.json",
      "runId": "glm-5p2__pi__green_2_tme_context_dependence__r1",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-glm-5p2-pi",
      "modelName": "glm-5p2",
      "harness": "pi",
      "task": "green_2_tme_context_dependence",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 2,
      "passed": false,
      "score": 4,
      "durationS": 2528.56,
      "cost": 0.6602183099999998,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "program_map",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "sig_includes_itgae",
            "role": "gate",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Itgae"
            ]
          },
          {
            "name": "declared_a_known_confounder",
            "role": "gate",
            "field": "$.excluded_confounders[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "cf_signature_gut_contaminant",
            "role": "hard_fail",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "generic_signature_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd103_program_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "<"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "per_model_regulatory_state",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "poised_unfired"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "late_activation"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "closing_committed"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_tuples",
            "kind": "list",
            "matchKey": "regulator",
            "k": 1,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "role": {
                    "op": "equals",
                    "values": [
                      "repressor"
                    ]
                  },
                  "gating_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  },
                  "evidence_basis": {
                    "op": "equals",
                    "values": [
                      "target_program_behavior"
                    ]
                  },
                  "expr_vs_activity": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "locus_differential_gate",
            "role": "gate",
            "field": "locus_differential",
            "kind": "value",
            "expected": "uniform_across_models"
          },
          {
            "name": "formation_limited_by_gate",
            "role": "gate",
            "field": "formation_limited_by",
            "kind": "value",
            "expected": "absent_induction"
          },
          {
            "name": "locus_competent_in_naive_gate",
            "role": "gate",
            "field": "locus_competent_in_naive",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "ready_silent_fraction_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 3,
            "expected": [
              {
                "key": "B16_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_KPC_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "activator_chromatin_footprint_gate",
            "role": "gate",
            "field": "activator_chromatin_footprint",
            "kind": "value",
            "expected": "inconsistent_across_formers"
          },
          {
            "name": "exhaustion_comparable_gate",
            "role": "gate",
            "field": "exhaustion_comparable_across_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_nondeterminant_gate",
            "role": "gate",
            "field": "exhaustion_explains_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_separable_program_gate",
            "role": "gate",
            "field": "exhaustion_separable_program",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "exhaustion_counterexample_gate",
            "role": "gate",
            "field": "exhaustion_counterexample_model",
            "kind": "value",
            "expected": "KPC_GP"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "exhaustion_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "cf_inconsistent_present_without_activator",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "program",
            "entity": "present",
            "valueField": "activator_state",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_no_block_but_absent",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "none",
            "valueField": "program",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_late_block_without_poised_locus",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "late_activation",
            "valueField": "locus_state",
            "expected": [
              "closing_committed",
              "active"
            ]
          },
          {
            "name": "cf_inconsistent_early_block_with_brake_released",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "early_release",
            "valueField": "brake_state",
            "expected": [
              "released"
            ]
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: FAIL\n  x list_match: list_match (answer_field='program_map'): FAIL  (score=0.0)\n  + sig_includes_itgae: passed=True, score=1.0\n  x declared_a_known_confounder: passed=False, score=0.0\n  + list_match: list_match (answer_field='generic_signature_relations'): PASS  (score=0.0)\n  x list_match: list_match (answer_field='cd103_program_relations'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='per_model_regulatory_state'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='regulator_tuples'): FAIL  (score=0.0)\n  x locus_differential_gate: passed=False, score=0.0\n  x formation_limited_by_gate: passed=False, score=0.0\n  + locus_competent_in_naive_gate: passed=True, score=1.0\n  x list_match: list_match (answer_field='ready_silent_fraction_relations'): FAIL  (score=0.0)\n  x activator_chromatin_footprint_gate: passed=False, score=0.0\n  x exhaustion_comparable_gate: passed=False, score=0.0\n  + exhaustion_nondeterminant_gate: passed=True, score=1.0\n  + exhaustion_separable_program_gate: passed=True, score=1.0\n  x exhaustion_counterexample_gate: passed=False, score=0.0\n  + list_match: list_match (answer_field='exhaustion_relations'): PASS  (score=0.0)\n  x hard_fail 'cf_signature_gut_contaminant': TRIGGERED\n  + hard_fail 'cf_inconsistent_present_without_activator': not triggered\n  + hard_fail 'cf_inconsistent_no_block_but_absent': not triggered\n  + hard_fail 'cf_inconsistent_late_block_without_poised_locus': not triggered\n  + hard_fail 'cf_inconsistent_early_block_with_brake_released': not triggered",
      "metrics": {
        "pass_rule": "all",
        "scoring_count": 17,
        "scoring_passed": 6,
        "scoring_total_score": 4,
        "hard_fail_triggered": [
          "cf_signature_gut_contaminant"
        ]
      },
      "fieldScores": {
        "list_match": 0,
        "sig_includes_itgae": 1,
        "declared_a_known_confounder": 0,
        "locus_differential_gate": 0,
        "formation_limited_by_gate": 0,
        "locus_competent_in_naive_gate": 1,
        "activator_chromatin_footprint_gate": 0,
        "exhaustion_comparable_gate": 0,
        "exhaustion_nondeterminant_gate": 1,
        "exhaustion_separable_program_gate": 1,
        "exhaustion_counterexample_gate": 0
      },
      "graderChecks": [
        {
          "name": "program_map",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "sig_includes_itgae",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "declared_a_known_confounder",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "generic_signature_relations",
          "kind": "list_match",
          "passed": true,
          "score": 0
        },
        {
          "name": "cd103_program_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "per_model_regulatory_state",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "regulator_tuples",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "locus_differential_gate",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "formation_limited_by_gate",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "locus_competent_in_naive_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "ready_silent_fraction_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "activator_chromatin_footprint_gate",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "exhaustion_comparable_gate",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "exhaustion_nondeterminant_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "exhaustion_separable_program_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "exhaustion_counterexample_gate",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "exhaustion_relations",
          "kind": "list_match",
          "passed": true,
          "score": 0
        },
        {
          "name": "cf_signature_gut_contaminant",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "cf_inconsistent_present_without_activator",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_no_block_but_absent",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_late_block_without_poised_locus",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_early_block_with_brake_released",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "activator_chromatin_footprint": "consistent_in_formers",
        "cd103_program_relations": [
          {
            "pair": "KPC_GP_vs_B16_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "B16_GP_vs_KPC_GP",
            "relation": "<"
          },
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": "~"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": "<"
          },
          {
            "pair": "B_ALL_GP_vs_KPC_GP",
            "relation": "<"
          },
          {
            "pair": "B_ALL_GP_vs_B16_GP",
            "relation": "~"
          },
          {
            "pair": "B_ALL_GP_vs_mWnt_GP",
            "relation": "<"
          },
          {
            "pair": "mWnt_GP_vs_KPC_GP",
            "relation": "<"
          },
          {
            "pair": "mWnt_GP_vs_B16_GP",
            "relation": ">"
          },
          {
            "pair": "mWnt_GP_vs_B_ALL_GP",
            "relation": ">"
          }
        ],
        "core_residency_signature": [
          "Itgae",
          "Itga1",
          "Cd69",
          "Bhlhe40",
          "Runx3",
          "Prdm1",
          "Zfp683",
          "Cxcr6",
          "Btg1",
          "Rgs1",
          "Rgs2",
          "Crip1",
          "Capg",
          "Areg"
        ],
        "excluded_confounders": [
          "Pdcd1",
          "Havcr2",
          "Tox",
          "Lag3",
          "Tigit",
          "Ctla4",
          "Entpd1",
          "Ifng",
          "Gzmb",
          "Prf1",
          "Nkg7"
        ],
        "exhaustion_comparable_across_split": "yes",
        "exhaustion_counterexample_model": "B16_GP",
        "exhaustion_explains_split": "no",
        "exhaustion_relations": [
          {
            "pair": "KPC_GP_vs_B16_GP",
            "relation": "~"
          },
          {
            "pair": "KPC_GP_vs_B_ALL_GP",
            "relation": "~"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "B16_GP_vs_KPC_GP",
            "relation": "~"
          },
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": "~"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "B_ALL_GP_vs_KPC_GP",
            "relation": "~"
          },
          {
            "pair": "B_ALL_GP_vs_B16_GP",
            "relation": "~"
          },
          {
            "pair": "B_ALL_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "mWnt_GP_vs_KPC_GP",
            "relation": "<"
          },
          {
            "pair": "mWnt_GP_vs_B16_GP",
            "relation": "<"
          },
          {
            "pair": "mWnt_GP_vs_B_ALL_GP",
            "relation": "<"
          }
        ],
        "exhaustion_separable_program": "yes",
        "formation_limited_by": "locus_incompetence",
        "generic_signature_relations": [
          {
            "pair": "KPC_GP_vs_B16_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": "~"
          },
          {
            "pair": "B16_GP_vs_KPC_GP",
            "relation": "<"
          },
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": "<"
          },
          {
            "pair": "B_ALL_GP_vs_KPC_GP",
            "relation": "<"
          },
          {
            "pair": "B_ALL_GP_vs_B16_GP",
            "relation": "<"
          },
          {
            "pair": "B_ALL_GP_vs_mWnt_GP",
            "relation": "<"
          },
          {
            "pair": "mWnt_GP_vs_KPC_GP",
            "relation": "~"
          },
          {
            "pair": "mWnt_GP_vs_B16_GP",
            "relation": ">"
          },
          {
            "pair": "mWnt_GP_vs_B_ALL_GP",
            "relation": ">"
          }
        ],
        "locus_competent_in_naive": "yes",
        "locus_differential": "model_specific_element",
        "per_model_regulatory_state": [
          {
            "activator_state": "present",
            "blocking_step": "none",
            "brake_state": "released",
            "locus_state": "active",
            "model": "KPC_GP",
            "program": "present"
          },
          {
            "activator_state": "present",
            "blocking_step": "early_release",
            "brake_state": "partial",
            "locus_state": "poised_unfired",
            "model": "mWnt_GP",
            "program": "partial"
          },
          {
            "activator_state": "present",
            "blocking_step": "signal_independent",
            "brake_state": "released",
            "locus_state": "repressed",
            "model": "B16_GP",
            "program": "partial"
          },
          {
            "activator_state": "present",
            "blocking_step": "both_steps",
            "brake_state": "partial",
            "locus_state": "repressed",
            "model": "B_ALL_GP",
            "program": "absent"
          }
        ],
        "program_map": [
          {
            "call": "present",
            "model": "KPC_GP"
          },
          {
            "call": "partial",
            "model": "mWnt_GP"
          },
          {
            "call": "partial",
            "model": "B16_GP"
          },
          {
            "call": "absent",
            "model": "B_ALL_GP"
          }
        ],
        "ready_silent_fraction_relations": [
          {
            "pair": "KPC_GP_vs_B16_GP",
            "relation": "<"
          },
          {
            "pair": "KPC_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": "<"
          },
          {
            "pair": "B16_GP_vs_KPC_GP",
            "relation": ">"
          },
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": "<"
          },
          {
            "pair": "B_ALL_GP_vs_KPC_GP",
            "relation": "<"
          },
          {
            "pair": "B_ALL_GP_vs_B16_GP",
            "relation": "<"
          },
          {
            "pair": "B_ALL_GP_vs_mWnt_GP",
            "relation": "<"
          },
          {
            "pair": "mWnt_GP_vs_KPC_GP",
            "relation": ">"
          },
          {
            "pair": "mWnt_GP_vs_B16_GP",
            "relation": ">"
          },
          {
            "pair": "mWnt_GP_vs_B_ALL_GP",
            "relation": ">"
          }
        ],
        "regulator_tuples": [
          {
            "evidence_basis": "own_abundance",
            "expr_vs_activity": "concordant",
            "gating_step": "early_release",
            "regulator": "Klf2",
            "role": "repressor"
          },
          {
            "evidence_basis": "target_program_behavior",
            "expr_vs_activity": "discordant",
            "gating_step": "late_activation",
            "regulator": "Runx3",
            "role": "activator"
          },
          {
            "evidence_basis": "target_program_behavior",
            "expr_vs_activity": "discordant",
            "gating_step": "late_activation",
            "regulator": "Bhlhe40",
            "role": "activator"
          },
          {
            "evidence_basis": "target_program_behavior",
            "expr_vs_activity": "discordant",
            "gating_step": "late_activation",
            "regulator": "Prdm1",
            "role": "activator"
          },
          {
            "evidence_basis": "own_abundance",
            "expr_vs_activity": "concordant",
            "gating_step": "late_activation",
            "regulator": "Tgfbr2",
            "role": "activator"
          }
        ]
      },
      "nTurns": 73,
      "checkpointSignals": {
        "clustering": false,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": false,
        "nerveJointAnchor": false,
        "multiomeAssembly": true,
        "depthNormalizedAtac": true,
        "regulonInference": true,
        "cd8SubsetIntegration": true,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": false,
        "orthologProjection": false,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": false,
        "mousePseudobulkDesign": false,
        "cancerEpithelialDenominator": true,
        "geneLevelDe": true,
        "agingSignatureControl": false,
        "senescencePanelScoring": true
      },
      "trajectoryFile": "scbench-long/trajectories/green_2_tme_context_dependence__glm-5p2__pi__r2.json",
      "runId": "glm-5p2__pi__green_2_tme_context_dependence__r2",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-glm-5p2-pi",
      "modelName": "glm-5p2",
      "harness": "pi",
      "task": "green_2_tme_context_dependence",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 3,
      "passed": false,
      "score": 7,
      "durationS": 3683.93,
      "cost": 0.6892327600000001,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "program_map",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "sig_includes_itgae",
            "role": "gate",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Itgae"
            ]
          },
          {
            "name": "declared_a_known_confounder",
            "role": "gate",
            "field": "$.excluded_confounders[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "cf_signature_gut_contaminant",
            "role": "hard_fail",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "generic_signature_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd103_program_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "<"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "per_model_regulatory_state",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "poised_unfired"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "late_activation"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "closing_committed"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_tuples",
            "kind": "list",
            "matchKey": "regulator",
            "k": 1,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "role": {
                    "op": "equals",
                    "values": [
                      "repressor"
                    ]
                  },
                  "gating_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  },
                  "evidence_basis": {
                    "op": "equals",
                    "values": [
                      "target_program_behavior"
                    ]
                  },
                  "expr_vs_activity": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "locus_differential_gate",
            "role": "gate",
            "field": "locus_differential",
            "kind": "value",
            "expected": "uniform_across_models"
          },
          {
            "name": "formation_limited_by_gate",
            "role": "gate",
            "field": "formation_limited_by",
            "kind": "value",
            "expected": "absent_induction"
          },
          {
            "name": "locus_competent_in_naive_gate",
            "role": "gate",
            "field": "locus_competent_in_naive",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "ready_silent_fraction_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 3,
            "expected": [
              {
                "key": "B16_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_KPC_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "activator_chromatin_footprint_gate",
            "role": "gate",
            "field": "activator_chromatin_footprint",
            "kind": "value",
            "expected": "inconsistent_across_formers"
          },
          {
            "name": "exhaustion_comparable_gate",
            "role": "gate",
            "field": "exhaustion_comparable_across_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_nondeterminant_gate",
            "role": "gate",
            "field": "exhaustion_explains_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_separable_program_gate",
            "role": "gate",
            "field": "exhaustion_separable_program",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "exhaustion_counterexample_gate",
            "role": "gate",
            "field": "exhaustion_counterexample_model",
            "kind": "value",
            "expected": "KPC_GP"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "exhaustion_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "cf_inconsistent_present_without_activator",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "program",
            "entity": "present",
            "valueField": "activator_state",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_no_block_but_absent",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "none",
            "valueField": "program",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_late_block_without_poised_locus",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "late_activation",
            "valueField": "locus_state",
            "expected": [
              "closing_committed",
              "active"
            ]
          },
          {
            "name": "cf_inconsistent_early_block_with_brake_released",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "early_release",
            "valueField": "brake_state",
            "expected": [
              "released"
            ]
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: FAIL\n  x list_match: list_match (answer_field='program_map'): FAIL  (score=0.0)\n  + sig_includes_itgae: passed=True, score=1.0\n  x declared_a_known_confounder: passed=False, score=0.0\n  + list_match: list_match (answer_field='generic_signature_relations'): PASS  (score=0.0)\n  x list_match: list_match (answer_field='cd103_program_relations'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='per_model_regulatory_state'): FAIL  (score=0.0)\n  + list_match: list_match (answer_field='regulator_tuples'): PASS  (score=0.0)\n  + locus_differential_gate: passed=True, score=1.0\n  + formation_limited_by_gate: passed=True, score=1.0\n  + locus_competent_in_naive_gate: passed=True, score=1.0\n  x list_match: list_match (answer_field='ready_silent_fraction_relations'): FAIL  (score=0.0)\n  x activator_chromatin_footprint_gate: passed=False, score=0.0\n  x exhaustion_comparable_gate: passed=False, score=0.0\n  + exhaustion_nondeterminant_gate: passed=True, score=1.0\n  + exhaustion_separable_program_gate: passed=True, score=1.0\n  + exhaustion_counterexample_gate: passed=True, score=1.0\n  x list_match: list_match (answer_field='exhaustion_relations'): FAIL  (score=0.0)\n  x hard_fail 'cf_signature_gut_contaminant': TRIGGERED\n  + hard_fail 'cf_inconsistent_present_without_activator': not triggered\n  + hard_fail 'cf_inconsistent_no_block_but_absent': not triggered\n  + hard_fail 'cf_inconsistent_late_block_without_poised_locus': not triggered\n  + hard_fail 'cf_inconsistent_early_block_with_brake_released': not triggered",
      "metrics": {
        "pass_rule": "all",
        "scoring_count": 17,
        "scoring_passed": 9,
        "scoring_total_score": 7,
        "hard_fail_triggered": [
          "cf_signature_gut_contaminant"
        ]
      },
      "fieldScores": {
        "list_match": 0,
        "sig_includes_itgae": 1,
        "declared_a_known_confounder": 0,
        "locus_differential_gate": 1,
        "formation_limited_by_gate": 1,
        "locus_competent_in_naive_gate": 1,
        "activator_chromatin_footprint_gate": 0,
        "exhaustion_comparable_gate": 0,
        "exhaustion_nondeterminant_gate": 1,
        "exhaustion_separable_program_gate": 1,
        "exhaustion_counterexample_gate": 1
      },
      "graderChecks": [
        {
          "name": "program_map",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "sig_includes_itgae",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "declared_a_known_confounder",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "generic_signature_relations",
          "kind": "list_match",
          "passed": true,
          "score": 0
        },
        {
          "name": "cd103_program_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "per_model_regulatory_state",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "regulator_tuples",
          "kind": "list_match",
          "passed": true,
          "score": 0
        },
        {
          "name": "locus_differential_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "formation_limited_by_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "locus_competent_in_naive_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "ready_silent_fraction_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "activator_chromatin_footprint_gate",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "exhaustion_comparable_gate",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "exhaustion_nondeterminant_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "exhaustion_separable_program_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "exhaustion_counterexample_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "exhaustion_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "cf_signature_gut_contaminant",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "cf_inconsistent_present_without_activator",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_no_block_but_absent",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_late_block_without_poised_locus",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_early_block_with_brake_released",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "activator_chromatin_footprint": "consistent_in_formers",
        "cd103_program_relations": [
          {
            "pair": "KPC_GP_vs_B16_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": "~"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": "<"
          },
          {
            "pair": "B_ALL_GP_vs_mWnt_GP",
            "relation": "<"
          }
        ],
        "core_residency_signature": [
          "Itgae",
          "Itga1",
          "Cd69",
          "Cxcr6",
          "Rgs1",
          "Rora",
          "Bhlhe40",
          "Runx3"
        ],
        "excluded_confounders": [
          "Pdcd1",
          "Ctla4",
          "Havcr2",
          "Lag3",
          "Tigit",
          "Tox",
          "Entpd1",
          "Gzmb",
          "Gzmk",
          "Prf1",
          "Nkg7",
          "Ccl5",
          "Ifng",
          "Cdkn2a"
        ],
        "exhaustion_comparable_across_split": "yes",
        "exhaustion_counterexample_model": "KPC_GP",
        "exhaustion_explains_split": "no",
        "exhaustion_relations": [
          {
            "pair": "KPC_GP_vs_B16_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": "~"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "B_ALL_GP_vs_mWnt_GP",
            "relation": ">"
          }
        ],
        "exhaustion_separable_program": "yes",
        "formation_limited_by": "absent_induction",
        "generic_signature_relations": [
          {
            "pair": "KPC_GP_vs_B16_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": "~"
          },
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": "<"
          },
          {
            "pair": "B_ALL_GP_vs_mWnt_GP",
            "relation": "<"
          }
        ],
        "locus_competent_in_naive": "yes",
        "locus_differential": "uniform_across_models",
        "per_model_regulatory_state": [
          {
            "activator_state": "present",
            "blocking_step": "none",
            "brake_state": "released",
            "locus_state": "active",
            "model": "KPC_GP",
            "program": "present"
          },
          {
            "activator_state": "partial",
            "blocking_step": "both_steps",
            "brake_state": "partial",
            "locus_state": "poised_unfired",
            "model": "mWnt_GP",
            "program": "partial"
          },
          {
            "activator_state": "partial",
            "blocking_step": "late_activation",
            "brake_state": "released",
            "locus_state": "poised_unfired",
            "model": "B16_GP",
            "program": "absent"
          },
          {
            "activator_state": "partial",
            "blocking_step": "both_steps",
            "brake_state": "partial",
            "locus_state": "poised_unfired",
            "model": "B_ALL_GP",
            "program": "absent"
          }
        ],
        "program_map": [
          {
            "call": "present",
            "model": "KPC_GP"
          },
          {
            "call": "partial",
            "model": "mWnt_GP"
          },
          {
            "call": "absent",
            "model": "B16_GP"
          },
          {
            "call": "absent",
            "model": "B_ALL_GP"
          }
        ],
        "ready_silent_fraction_relations": [
          {
            "pair": "KPC_GP_vs_B16_GP",
            "relation": "<"
          },
          {
            "pair": "KPC_GP_vs_B_ALL_GP",
            "relation": "<"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": "<"
          },
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": "<"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": "<"
          },
          {
            "pair": "B_ALL_GP_vs_mWnt_GP",
            "relation": "<"
          }
        ],
        "regulator_tuples": [
          {
            "evidence_basis": "target_program_behavior",
            "expr_vs_activity": "concordant",
            "gating_step": "late_activation",
            "regulator": "Foxo1",
            "role": "activator"
          },
          {
            "evidence_basis": "target_program_behavior",
            "expr_vs_activity": "concordant",
            "gating_step": "late_activation",
            "regulator": "Rora",
            "role": "activator"
          },
          {
            "evidence_basis": "target_program_behavior",
            "expr_vs_activity": "concordant",
            "gating_step": "late_activation",
            "regulator": "Stat5b",
            "role": "activator"
          },
          {
            "evidence_basis": "target_program_behavior",
            "expr_vs_activity": "concordant",
            "gating_step": "late_activation",
            "regulator": "Runx3",
            "role": "activator"
          },
          {
            "evidence_basis": "target_program_behavior",
            "expr_vs_activity": "discordant",
            "gating_step": "late_activation",
            "regulator": "Bhlhe40",
            "role": "activator"
          },
          {
            "evidence_basis": "target_program_behavior",
            "expr_vs_activity": "discordant",
            "gating_step": "early_release",
            "regulator": "Bach2",
            "role": "repressor"
          },
          {
            "evidence_basis": "target_program_behavior",
            "expr_vs_activity": "concordant",
            "gating_step": "early_release",
            "regulator": "Klf2",
            "role": "repressor"
          },
          {
            "evidence_basis": "target_program_behavior",
            "expr_vs_activity": "concordant",
            "gating_step": "early_release",
            "regulator": "Eomes",
            "role": "repressor"
          },
          {
            "evidence_basis": "target_program_behavior",
            "expr_vs_activity": "concordant",
            "gating_step": "late_activation",
            "regulator": "Id2",
            "role": "repressor"
          }
        ]
      },
      "nTurns": 80,
      "checkpointSignals": {
        "clustering": false,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": true,
        "nerveJointAnchor": false,
        "multiomeAssembly": true,
        "depthNormalizedAtac": true,
        "regulonInference": true,
        "cd8SubsetIntegration": true,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": false,
        "orthologProjection": false,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": true,
        "enrichmentFamilyRanking": false,
        "mousePseudobulkDesign": true,
        "cancerEpithelialDenominator": true,
        "geneLevelDe": true,
        "agingSignatureControl": false,
        "senescencePanelScoring": true
      },
      "trajectoryFile": "scbench-long/trajectories/green_2_tme_context_dependence__glm-5p2__pi__r3.json",
      "runId": "glm-5p2__pi__green_2_tme_context_dependence__r3",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-gpt-5-4-codex",
      "modelName": "gpt-5.4",
      "harness": "openai-codex",
      "task": "green_2_tme_context_dependence",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 1,
      "passed": false,
      "score": 8,
      "durationS": 1657.59,
      "cost": 2.8994590000000002,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "program_map",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "sig_includes_itgae",
            "role": "gate",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Itgae"
            ]
          },
          {
            "name": "declared_a_known_confounder",
            "role": "gate",
            "field": "$.excluded_confounders[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "cf_signature_gut_contaminant",
            "role": "hard_fail",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "generic_signature_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd103_program_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "<"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "per_model_regulatory_state",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "poised_unfired"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "late_activation"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "closing_committed"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_tuples",
            "kind": "list",
            "matchKey": "regulator",
            "k": 1,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "role": {
                    "op": "equals",
                    "values": [
                      "repressor"
                    ]
                  },
                  "gating_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  },
                  "evidence_basis": {
                    "op": "equals",
                    "values": [
                      "target_program_behavior"
                    ]
                  },
                  "expr_vs_activity": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "locus_differential_gate",
            "role": "gate",
            "field": "locus_differential",
            "kind": "value",
            "expected": "uniform_across_models"
          },
          {
            "name": "formation_limited_by_gate",
            "role": "gate",
            "field": "formation_limited_by",
            "kind": "value",
            "expected": "absent_induction"
          },
          {
            "name": "locus_competent_in_naive_gate",
            "role": "gate",
            "field": "locus_competent_in_naive",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "ready_silent_fraction_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 3,
            "expected": [
              {
                "key": "B16_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_KPC_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "activator_chromatin_footprint_gate",
            "role": "gate",
            "field": "activator_chromatin_footprint",
            "kind": "value",
            "expected": "inconsistent_across_formers"
          },
          {
            "name": "exhaustion_comparable_gate",
            "role": "gate",
            "field": "exhaustion_comparable_across_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_nondeterminant_gate",
            "role": "gate",
            "field": "exhaustion_explains_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_separable_program_gate",
            "role": "gate",
            "field": "exhaustion_separable_program",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "exhaustion_counterexample_gate",
            "role": "gate",
            "field": "exhaustion_counterexample_model",
            "kind": "value",
            "expected": "KPC_GP"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "exhaustion_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "cf_inconsistent_present_without_activator",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "program",
            "entity": "present",
            "valueField": "activator_state",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_no_block_but_absent",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "none",
            "valueField": "program",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_late_block_without_poised_locus",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "late_activation",
            "valueField": "locus_state",
            "expected": [
              "closing_committed",
              "active"
            ]
          },
          {
            "name": "cf_inconsistent_early_block_with_brake_released",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "early_release",
            "valueField": "brake_state",
            "expected": [
              "released"
            ]
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: FAIL\n  x list_match: list_match (answer_field='program_map'): FAIL  (score=0.0)\n  + sig_includes_itgae: passed=True, score=1.0\n  x declared_a_known_confounder: passed=False, score=0.0\n  x list_match: list_match (answer_field='generic_signature_relations'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='cd103_program_relations'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='per_model_regulatory_state'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='regulator_tuples'): FAIL  (score=0.0)\n  + locus_differential_gate: passed=True, score=1.0\n  + formation_limited_by_gate: passed=True, score=1.0\n  + locus_competent_in_naive_gate: passed=True, score=1.0\n  x list_match: list_match (answer_field='ready_silent_fraction_relations'): FAIL  (score=0.0)\n  + activator_chromatin_footprint_gate: passed=True, score=1.0\n  x exhaustion_comparable_gate: passed=False, score=0.0\n  + exhaustion_nondeterminant_gate: passed=True, score=1.0\n  + exhaustion_separable_program_gate: passed=True, score=1.0\n  + exhaustion_counterexample_gate: passed=True, score=1.0\n  x list_match: list_match (answer_field='exhaustion_relations'): FAIL  (score=0.0)\n  x hard_fail 'cf_signature_gut_contaminant': TRIGGERED\n  + hard_fail 'cf_inconsistent_present_without_activator': not triggered\n  + hard_fail 'cf_inconsistent_no_block_but_absent': not triggered\n  + hard_fail 'cf_inconsistent_late_block_without_poised_locus': not triggered\n  + hard_fail 'cf_inconsistent_early_block_with_brake_released': not triggered",
      "metrics": {
        "pass_rule": "all",
        "scoring_count": 17,
        "scoring_passed": 8,
        "scoring_total_score": 8,
        "hard_fail_triggered": [
          "cf_signature_gut_contaminant"
        ]
      },
      "fieldScores": {
        "list_match": 0,
        "sig_includes_itgae": 1,
        "declared_a_known_confounder": 0,
        "locus_differential_gate": 1,
        "formation_limited_by_gate": 1,
        "locus_competent_in_naive_gate": 1,
        "activator_chromatin_footprint_gate": 1,
        "exhaustion_comparable_gate": 0,
        "exhaustion_nondeterminant_gate": 1,
        "exhaustion_separable_program_gate": 1,
        "exhaustion_counterexample_gate": 1
      },
      "graderChecks": [
        {
          "name": "program_map",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "sig_includes_itgae",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "declared_a_known_confounder",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "generic_signature_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "cd103_program_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "per_model_regulatory_state",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "regulator_tuples",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "locus_differential_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "formation_limited_by_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "locus_competent_in_naive_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "ready_silent_fraction_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "activator_chromatin_footprint_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "exhaustion_comparable_gate",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "exhaustion_nondeterminant_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "exhaustion_separable_program_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "exhaustion_counterexample_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "exhaustion_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "cf_signature_gut_contaminant",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "cf_inconsistent_present_without_activator",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_no_block_but_absent",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_late_block_without_poised_locus",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_early_block_with_brake_released",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "activator_chromatin_footprint": "inconsistent_across_formers",
        "cd103_program_relations": [
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": "~"
          },
          {
            "pair": "B16_GP_vs_KPC_GP",
            "relation": "<"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": "<"
          },
          {
            "pair": "B_ALL_GP_vs_KPC_GP",
            "relation": "<"
          },
          {
            "pair": "B_ALL_GP_vs_mWnt_GP",
            "relation": "<"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": ">"
          }
        ],
        "core_residency_signature": [
          "Itgae",
          "Itga1",
          "Chn2",
          "Nedd9",
          "Abcb1a"
        ],
        "excluded_confounders": [
          "Tox",
          "Ctla4",
          "Nr4a2",
          "Gzmb",
          "Xcl1",
          "Rgs1"
        ],
        "exhaustion_comparable_across_split": "yes",
        "exhaustion_counterexample_model": "KPC_GP",
        "exhaustion_explains_split": "no",
        "exhaustion_relations": [
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": "~"
          },
          {
            "pair": "B16_GP_vs_KPC_GP",
            "relation": "~"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "B_ALL_GP_vs_KPC_GP",
            "relation": ">"
          },
          {
            "pair": "B_ALL_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": ">"
          }
        ],
        "exhaustion_separable_program": "yes",
        "formation_limited_by": "absent_induction",
        "generic_signature_relations": [
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": "~"
          },
          {
            "pair": "B16_GP_vs_KPC_GP",
            "relation": "~"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "B_ALL_GP_vs_KPC_GP",
            "relation": "<"
          },
          {
            "pair": "B_ALL_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": ">"
          }
        ],
        "locus_competent_in_naive": "yes",
        "locus_differential": "uniform_across_models",
        "per_model_regulatory_state": [
          {
            "activator_state": "absent",
            "blocking_step": "late_activation",
            "brake_state": "released",
            "locus_state": "poised_unfired",
            "model": "B16_GP",
            "program": "absent"
          },
          {
            "activator_state": "absent",
            "blocking_step": "both_steps",
            "brake_state": "partial",
            "locus_state": "poised_unfired",
            "model": "B_ALL_GP",
            "program": "absent"
          },
          {
            "activator_state": "present",
            "blocking_step": "none",
            "brake_state": "released",
            "locus_state": "active",
            "model": "KPC_GP",
            "program": "present"
          },
          {
            "activator_state": "partial",
            "blocking_step": "both_steps",
            "brake_state": "partial",
            "locus_state": "poised_unfired",
            "model": "mWnt_GP",
            "program": "partial"
          }
        ],
        "program_map": [
          {
            "call": "absent",
            "model": "B16_GP"
          },
          {
            "call": "absent",
            "model": "B_ALL_GP"
          },
          {
            "call": "present",
            "model": "KPC_GP"
          },
          {
            "call": "partial",
            "model": "mWnt_GP"
          }
        ],
        "ready_silent_fraction_relations": [
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": "<"
          },
          {
            "pair": "B16_GP_vs_KPC_GP",
            "relation": ">"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": "<"
          },
          {
            "pair": "B_ALL_GP_vs_KPC_GP",
            "relation": ">"
          },
          {
            "pair": "B_ALL_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": "<"
          }
        ],
        "regulator_tuples": [
          {
            "evidence_basis": "own_abundance",
            "expr_vs_activity": "concordant",
            "gating_step": "early_release",
            "regulator": "Klf2",
            "role": "repressor"
          },
          {
            "evidence_basis": "own_abundance",
            "expr_vs_activity": "concordant",
            "gating_step": "early_release",
            "regulator": "Tcf7",
            "role": "repressor"
          },
          {
            "evidence_basis": "own_abundance",
            "expr_vs_activity": "discordant",
            "gating_step": "late_activation",
            "regulator": "Runx3",
            "role": "activator"
          },
          {
            "evidence_basis": "own_abundance",
            "expr_vs_activity": "concordant",
            "gating_step": "late_activation",
            "regulator": "Ahr",
            "role": "activator"
          },
          {
            "evidence_basis": "target_program_behavior",
            "expr_vs_activity": "discordant",
            "gating_step": "none",
            "regulator": "Tox",
            "role": "none"
          }
        ]
      },
      "nTurns": 1,
      "checkpointSignals": {
        "clustering": false,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": true,
        "nerveJointAnchor": false,
        "multiomeAssembly": true,
        "depthNormalizedAtac": true,
        "regulonInference": true,
        "cd8SubsetIntegration": true,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": false,
        "orthologProjection": false,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": false,
        "mousePseudobulkDesign": false,
        "cancerEpithelialDenominator": false,
        "geneLevelDe": false,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "trajectoryFile": "scbench-long/trajectories/green_2_tme_context_dependence__gpt-5.4__openai-codex__r1.json",
      "runId": "gpt-5.4__openai-codex__green_2_tme_context_dependence__r1",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-gpt-5-4-codex",
      "modelName": "gpt-5.4",
      "harness": "openai-codex",
      "task": "green_2_tme_context_dependence",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 2,
      "passed": false,
      "score": 5,
      "durationS": 627.29,
      "cost": 2.0618115,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "program_map",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "sig_includes_itgae",
            "role": "gate",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Itgae"
            ]
          },
          {
            "name": "declared_a_known_confounder",
            "role": "gate",
            "field": "$.excluded_confounders[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "cf_signature_gut_contaminant",
            "role": "hard_fail",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "generic_signature_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd103_program_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "<"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "per_model_regulatory_state",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "poised_unfired"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "late_activation"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "closing_committed"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_tuples",
            "kind": "list",
            "matchKey": "regulator",
            "k": 1,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "role": {
                    "op": "equals",
                    "values": [
                      "repressor"
                    ]
                  },
                  "gating_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  },
                  "evidence_basis": {
                    "op": "equals",
                    "values": [
                      "target_program_behavior"
                    ]
                  },
                  "expr_vs_activity": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "locus_differential_gate",
            "role": "gate",
            "field": "locus_differential",
            "kind": "value",
            "expected": "uniform_across_models"
          },
          {
            "name": "formation_limited_by_gate",
            "role": "gate",
            "field": "formation_limited_by",
            "kind": "value",
            "expected": "absent_induction"
          },
          {
            "name": "locus_competent_in_naive_gate",
            "role": "gate",
            "field": "locus_competent_in_naive",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "ready_silent_fraction_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 3,
            "expected": [
              {
                "key": "B16_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_KPC_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "activator_chromatin_footprint_gate",
            "role": "gate",
            "field": "activator_chromatin_footprint",
            "kind": "value",
            "expected": "inconsistent_across_formers"
          },
          {
            "name": "exhaustion_comparable_gate",
            "role": "gate",
            "field": "exhaustion_comparable_across_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_nondeterminant_gate",
            "role": "gate",
            "field": "exhaustion_explains_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_separable_program_gate",
            "role": "gate",
            "field": "exhaustion_separable_program",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "exhaustion_counterexample_gate",
            "role": "gate",
            "field": "exhaustion_counterexample_model",
            "kind": "value",
            "expected": "KPC_GP"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "exhaustion_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "cf_inconsistent_present_without_activator",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "program",
            "entity": "present",
            "valueField": "activator_state",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_no_block_but_absent",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "none",
            "valueField": "program",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_late_block_without_poised_locus",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "late_activation",
            "valueField": "locus_state",
            "expected": [
              "closing_committed",
              "active"
            ]
          },
          {
            "name": "cf_inconsistent_early_block_with_brake_released",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "early_release",
            "valueField": "brake_state",
            "expected": [
              "released"
            ]
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: FAIL\n  x list_match: list_match (answer_field='program_map'): FAIL  (score=0.0)\n  + sig_includes_itgae: passed=True, score=1.0\n  x declared_a_known_confounder: passed=False, score=0.0\n  x list_match: list_match (answer_field='generic_signature_relations'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='cd103_program_relations'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='per_model_regulatory_state'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='regulator_tuples'): FAIL  (score=0.0)\n  x locus_differential_gate: passed=False, score=0.0\n  + formation_limited_by_gate: passed=True, score=1.0\n  + locus_competent_in_naive_gate: passed=True, score=1.0\n  x list_match: list_match (answer_field='ready_silent_fraction_relations'): FAIL  (score=0.0)\n  x activator_chromatin_footprint_gate: passed=False, score=0.0\n  x exhaustion_comparable_gate: passed=False, score=0.0\n  + exhaustion_nondeterminant_gate: passed=True, score=1.0\n  + exhaustion_separable_program_gate: passed=True, score=1.0\n  x exhaustion_counterexample_gate: passed=False, score=0.0\n  + list_match: list_match (answer_field='exhaustion_relations'): PASS  (score=0.0)\n  x hard_fail 'cf_signature_gut_contaminant': TRIGGERED\n  + hard_fail 'cf_inconsistent_present_without_activator': not triggered\n  + hard_fail 'cf_inconsistent_no_block_but_absent': not triggered\n  x hard_fail 'cf_inconsistent_late_block_without_poised_locus': TRIGGERED\n  + hard_fail 'cf_inconsistent_early_block_with_brake_released': not triggered",
      "metrics": {
        "pass_rule": "all",
        "scoring_count": 17,
        "scoring_passed": 6,
        "scoring_total_score": 5,
        "hard_fail_triggered": [
          "cf_signature_gut_contaminant",
          "cf_inconsistent_late_block_without_poised_locus"
        ]
      },
      "fieldScores": {
        "list_match": 0,
        "sig_includes_itgae": 1,
        "declared_a_known_confounder": 0,
        "locus_differential_gate": 0,
        "formation_limited_by_gate": 1,
        "locus_competent_in_naive_gate": 1,
        "activator_chromatin_footprint_gate": 0,
        "exhaustion_comparable_gate": 0,
        "exhaustion_nondeterminant_gate": 1,
        "exhaustion_separable_program_gate": 1,
        "exhaustion_counterexample_gate": 0
      },
      "graderChecks": [
        {
          "name": "program_map",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "sig_includes_itgae",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "declared_a_known_confounder",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "generic_signature_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "cd103_program_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "per_model_regulatory_state",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "regulator_tuples",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "locus_differential_gate",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "formation_limited_by_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "locus_competent_in_naive_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "ready_silent_fraction_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "activator_chromatin_footprint_gate",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "exhaustion_comparable_gate",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "exhaustion_nondeterminant_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "exhaustion_separable_program_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "exhaustion_counterexample_gate",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "exhaustion_relations",
          "kind": "list_match",
          "passed": true,
          "score": 0
        },
        {
          "name": "cf_signature_gut_contaminant",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "cf_inconsistent_present_without_activator",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_no_block_but_absent",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_late_block_without_poised_locus",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "cf_inconsistent_early_block_with_brake_released",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "activator_chromatin_footprint": "consistent_in_formers",
        "cd103_program_relations": [
          {
            "pair": "B16_GP_vs_KPC_GP",
            "relation": "<"
          },
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": "~"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": "<"
          },
          {
            "pair": "KPC_GP_vs_B16_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "B_ALL_GP_vs_B16_GP",
            "relation": "~"
          },
          {
            "pair": "B_ALL_GP_vs_KPC_GP",
            "relation": "<"
          },
          {
            "pair": "B_ALL_GP_vs_mWnt_GP",
            "relation": "<"
          },
          {
            "pair": "mWnt_GP_vs_B16_GP",
            "relation": ">"
          },
          {
            "pair": "mWnt_GP_vs_KPC_GP",
            "relation": "<"
          },
          {
            "pair": "mWnt_GP_vs_B_ALL_GP",
            "relation": ">"
          }
        ],
        "core_residency_signature": [
          "Itgae",
          "Itga1",
          "Nt5e",
          "Rgs1",
          "Chn2"
        ],
        "excluded_confounders": [
          "Cxcr6",
          "Cd69",
          "Xcl1",
          "Gzmb",
          "Ifng",
          "Lag3",
          "Havcr2",
          "Ctla4",
          "Bhlhe40",
          "Nr4a2"
        ],
        "exhaustion_comparable_across_split": "yes",
        "exhaustion_counterexample_model": "B16_GP",
        "exhaustion_explains_split": "no",
        "exhaustion_relations": [
          {
            "pair": "B16_GP_vs_KPC_GP",
            "relation": "~"
          },
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": "~"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_B16_GP",
            "relation": "~"
          },
          {
            "pair": "KPC_GP_vs_B_ALL_GP",
            "relation": "~"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "B_ALL_GP_vs_B16_GP",
            "relation": "~"
          },
          {
            "pair": "B_ALL_GP_vs_KPC_GP",
            "relation": "~"
          },
          {
            "pair": "B_ALL_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "mWnt_GP_vs_B16_GP",
            "relation": "<"
          },
          {
            "pair": "mWnt_GP_vs_KPC_GP",
            "relation": "<"
          },
          {
            "pair": "mWnt_GP_vs_B_ALL_GP",
            "relation": "<"
          }
        ],
        "exhaustion_separable_program": "yes",
        "formation_limited_by": "absent_induction",
        "generic_signature_relations": [
          {
            "pair": "B16_GP_vs_KPC_GP",
            "relation": "~"
          },
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_B16_GP",
            "relation": "~"
          },
          {
            "pair": "KPC_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "B_ALL_GP_vs_B16_GP",
            "relation": "<"
          },
          {
            "pair": "B_ALL_GP_vs_KPC_GP",
            "relation": "<"
          },
          {
            "pair": "B_ALL_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "mWnt_GP_vs_B16_GP",
            "relation": "<"
          },
          {
            "pair": "mWnt_GP_vs_KPC_GP",
            "relation": "<"
          },
          {
            "pair": "mWnt_GP_vs_B_ALL_GP",
            "relation": "<"
          }
        ],
        "locus_competent_in_naive": "yes",
        "locus_differential": "model_specific_element",
        "per_model_regulatory_state": [
          {
            "activator_state": "absent",
            "blocking_step": "late_activation",
            "brake_state": "released",
            "locus_state": "closing_committed",
            "model": "B16_GP",
            "program": "absent"
          },
          {
            "activator_state": "present",
            "blocking_step": "none",
            "brake_state": "released",
            "locus_state": "active",
            "model": "KPC_GP",
            "program": "present"
          },
          {
            "activator_state": "absent",
            "blocking_step": "late_activation",
            "brake_state": "partial",
            "locus_state": "poised_unfired",
            "model": "B_ALL_GP",
            "program": "absent"
          },
          {
            "activator_state": "partial",
            "blocking_step": "late_activation",
            "brake_state": "partial",
            "locus_state": "poised_unfired",
            "model": "mWnt_GP",
            "program": "partial"
          }
        ],
        "program_map": [
          {
            "call": "absent",
            "model": "B16_GP"
          },
          {
            "call": "present",
            "model": "KPC_GP"
          },
          {
            "call": "absent",
            "model": "B_ALL_GP"
          },
          {
            "call": "partial",
            "model": "mWnt_GP"
          }
        ],
        "ready_silent_fraction_relations": [
          {
            "pair": "B16_GP_vs_KPC_GP",
            "relation": "~"
          },
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": "<"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": "<"
          },
          {
            "pair": "KPC_GP_vs_B16_GP",
            "relation": "~"
          },
          {
            "pair": "KPC_GP_vs_B_ALL_GP",
            "relation": "<"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": "<"
          },
          {
            "pair": "B_ALL_GP_vs_B16_GP",
            "relation": ">"
          },
          {
            "pair": "B_ALL_GP_vs_KPC_GP",
            "relation": ">"
          },
          {
            "pair": "B_ALL_GP_vs_mWnt_GP",
            "relation": "~"
          },
          {
            "pair": "mWnt_GP_vs_B16_GP",
            "relation": ">"
          },
          {
            "pair": "mWnt_GP_vs_KPC_GP",
            "relation": ">"
          },
          {
            "pair": "mWnt_GP_vs_B_ALL_GP",
            "relation": "~"
          }
        ],
        "regulator_tuples": [
          {
            "evidence_basis": "own_abundance",
            "expr_vs_activity": "concordant",
            "gating_step": "early_release",
            "regulator": "Klf2",
            "role": "repressor"
          },
          {
            "evidence_basis": "own_abundance",
            "expr_vs_activity": "concordant",
            "gating_step": "early_release",
            "regulator": "Tcf7",
            "role": "repressor"
          },
          {
            "evidence_basis": "target_program_behavior",
            "expr_vs_activity": "discordant",
            "gating_step": "late_activation",
            "regulator": "Runx3",
            "role": "activator"
          },
          {
            "evidence_basis": "target_program_behavior",
            "expr_vs_activity": "discordant",
            "gating_step": "late_activation",
            "regulator": "Smad3",
            "role": "activator"
          }
        ]
      },
      "nTurns": 1,
      "checkpointSignals": {
        "clustering": false,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": true,
        "nerveJointAnchor": false,
        "multiomeAssembly": true,
        "depthNormalizedAtac": true,
        "regulonInference": true,
        "cd8SubsetIntegration": true,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": false,
        "orthologProjection": false,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": false,
        "mousePseudobulkDesign": false,
        "cancerEpithelialDenominator": false,
        "geneLevelDe": false,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "trajectoryFile": "scbench-long/trajectories/green_2_tme_context_dependence__gpt-5.4__openai-codex__r2.json",
      "runId": "gpt-5.4__openai-codex__green_2_tme_context_dependence__r2",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-gpt-5-4-codex",
      "modelName": "gpt-5.4",
      "harness": "openai-codex",
      "task": "green_2_tme_context_dependence",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 3,
      "passed": false,
      "score": 7,
      "durationS": 2514.5,
      "cost": 5.0598015,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "program_map",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "sig_includes_itgae",
            "role": "gate",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Itgae"
            ]
          },
          {
            "name": "declared_a_known_confounder",
            "role": "gate",
            "field": "$.excluded_confounders[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "cf_signature_gut_contaminant",
            "role": "hard_fail",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "generic_signature_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd103_program_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "<"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "per_model_regulatory_state",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "poised_unfired"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "late_activation"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "closing_committed"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_tuples",
            "kind": "list",
            "matchKey": "regulator",
            "k": 1,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "role": {
                    "op": "equals",
                    "values": [
                      "repressor"
                    ]
                  },
                  "gating_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  },
                  "evidence_basis": {
                    "op": "equals",
                    "values": [
                      "target_program_behavior"
                    ]
                  },
                  "expr_vs_activity": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "locus_differential_gate",
            "role": "gate",
            "field": "locus_differential",
            "kind": "value",
            "expected": "uniform_across_models"
          },
          {
            "name": "formation_limited_by_gate",
            "role": "gate",
            "field": "formation_limited_by",
            "kind": "value",
            "expected": "absent_induction"
          },
          {
            "name": "locus_competent_in_naive_gate",
            "role": "gate",
            "field": "locus_competent_in_naive",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "ready_silent_fraction_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 3,
            "expected": [
              {
                "key": "B16_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_KPC_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "activator_chromatin_footprint_gate",
            "role": "gate",
            "field": "activator_chromatin_footprint",
            "kind": "value",
            "expected": "inconsistent_across_formers"
          },
          {
            "name": "exhaustion_comparable_gate",
            "role": "gate",
            "field": "exhaustion_comparable_across_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_nondeterminant_gate",
            "role": "gate",
            "field": "exhaustion_explains_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_separable_program_gate",
            "role": "gate",
            "field": "exhaustion_separable_program",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "exhaustion_counterexample_gate",
            "role": "gate",
            "field": "exhaustion_counterexample_model",
            "kind": "value",
            "expected": "KPC_GP"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "exhaustion_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "cf_inconsistent_present_without_activator",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "program",
            "entity": "present",
            "valueField": "activator_state",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_no_block_but_absent",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "none",
            "valueField": "program",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_late_block_without_poised_locus",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "late_activation",
            "valueField": "locus_state",
            "expected": [
              "closing_committed",
              "active"
            ]
          },
          {
            "name": "cf_inconsistent_early_block_with_brake_released",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "early_release",
            "valueField": "brake_state",
            "expected": [
              "released"
            ]
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: FAIL\n  x list_match: list_match (answer_field='program_map'): FAIL  (score=0.0)\n  + sig_includes_itgae: passed=True, score=1.0\n  x declared_a_known_confounder: passed=False, score=0.0\n  x list_match: list_match (answer_field='generic_signature_relations'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='cd103_program_relations'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='per_model_regulatory_state'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='regulator_tuples'): FAIL  (score=0.0)\n  + locus_differential_gate: passed=True, score=1.0\n  + formation_limited_by_gate: passed=True, score=1.0\n  x locus_competent_in_naive_gate: passed=False, score=0.0\n  x list_match: list_match (answer_field='ready_silent_fraction_relations'): FAIL  (score=0.0)\n  x activator_chromatin_footprint_gate: passed=False, score=0.0\n  + exhaustion_comparable_gate: passed=True, score=1.0\n  + exhaustion_nondeterminant_gate: passed=True, score=1.0\n  + exhaustion_separable_program_gate: passed=True, score=1.0\n  + exhaustion_counterexample_gate: passed=True, score=1.0\n  + list_match: list_match (answer_field='exhaustion_relations'): PASS  (score=0.0)\n  x hard_fail 'cf_signature_gut_contaminant': TRIGGERED\n  + hard_fail 'cf_inconsistent_present_without_activator': not triggered\n  + hard_fail 'cf_inconsistent_no_block_but_absent': not triggered\n  + hard_fail 'cf_inconsistent_late_block_without_poised_locus': not triggered\n  + hard_fail 'cf_inconsistent_early_block_with_brake_released': not triggered",
      "metrics": {
        "pass_rule": "all",
        "scoring_count": 17,
        "scoring_passed": 8,
        "scoring_total_score": 7,
        "hard_fail_triggered": [
          "cf_signature_gut_contaminant"
        ]
      },
      "fieldScores": {
        "list_match": 0,
        "sig_includes_itgae": 1,
        "declared_a_known_confounder": 0,
        "locus_differential_gate": 1,
        "formation_limited_by_gate": 1,
        "locus_competent_in_naive_gate": 0,
        "activator_chromatin_footprint_gate": 0,
        "exhaustion_comparable_gate": 1,
        "exhaustion_nondeterminant_gate": 1,
        "exhaustion_separable_program_gate": 1,
        "exhaustion_counterexample_gate": 1
      },
      "graderChecks": [
        {
          "name": "program_map",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "sig_includes_itgae",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "declared_a_known_confounder",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "generic_signature_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "cd103_program_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "per_model_regulatory_state",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "regulator_tuples",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "locus_differential_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "formation_limited_by_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "locus_competent_in_naive_gate",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "ready_silent_fraction_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "activator_chromatin_footprint_gate",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "exhaustion_comparable_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "exhaustion_nondeterminant_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "exhaustion_separable_program_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "exhaustion_counterexample_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "exhaustion_relations",
          "kind": "list_match",
          "passed": true,
          "score": 0
        },
        {
          "name": "cf_signature_gut_contaminant",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "cf_inconsistent_present_without_activator",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_no_block_but_absent",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_late_block_without_poised_locus",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_early_block_with_brake_released",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "activator_chromatin_footprint": "absent",
        "cd103_program_relations": [
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": "~"
          },
          {
            "pair": "B16_GP_vs_KPC_GP",
            "relation": "<"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": "<"
          },
          {
            "pair": "B_ALL_GP_vs_B16_GP",
            "relation": "~"
          },
          {
            "pair": "B_ALL_GP_vs_KPC_GP",
            "relation": "<"
          },
          {
            "pair": "B_ALL_GP_vs_mWnt_GP",
            "relation": "<"
          },
          {
            "pair": "KPC_GP_vs_B16_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "mWnt_GP_vs_B16_GP",
            "relation": ">"
          },
          {
            "pair": "mWnt_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "mWnt_GP_vs_KPC_GP",
            "relation": "<"
          }
        ],
        "core_residency_signature": [
          "Itgae",
          "Itga1",
          "Rgs1"
        ],
        "excluded_confounders": [
          "Cd69",
          "Cxcr6",
          "Xcl1",
          "Gzmb",
          "Ctla4",
          "Pdcd1",
          "Tox"
        ],
        "exhaustion_comparable_across_split": "no",
        "exhaustion_counterexample_model": "KPC_GP",
        "exhaustion_explains_split": "no",
        "exhaustion_relations": [
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": "~"
          },
          {
            "pair": "B16_GP_vs_KPC_GP",
            "relation": "~"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "B_ALL_GP_vs_B16_GP",
            "relation": "~"
          },
          {
            "pair": "B_ALL_GP_vs_KPC_GP",
            "relation": "~"
          },
          {
            "pair": "B_ALL_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_B16_GP",
            "relation": "~"
          },
          {
            "pair": "KPC_GP_vs_B_ALL_GP",
            "relation": "~"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "mWnt_GP_vs_B16_GP",
            "relation": "<"
          },
          {
            "pair": "mWnt_GP_vs_B_ALL_GP",
            "relation": "<"
          },
          {
            "pair": "mWnt_GP_vs_KPC_GP",
            "relation": "<"
          }
        ],
        "exhaustion_separable_program": "yes",
        "formation_limited_by": "absent_induction",
        "generic_signature_relations": [
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "B16_GP_vs_KPC_GP",
            "relation": "~"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "B_ALL_GP_vs_B16_GP",
            "relation": "<"
          },
          {
            "pair": "B_ALL_GP_vs_KPC_GP",
            "relation": "<"
          },
          {
            "pair": "B_ALL_GP_vs_mWnt_GP",
            "relation": "<"
          },
          {
            "pair": "KPC_GP_vs_B16_GP",
            "relation": "~"
          },
          {
            "pair": "KPC_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": "~"
          },
          {
            "pair": "mWnt_GP_vs_B16_GP",
            "relation": "<"
          },
          {
            "pair": "mWnt_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "mWnt_GP_vs_KPC_GP",
            "relation": "~"
          }
        ],
        "locus_competent_in_naive": "no",
        "locus_differential": "uniform_across_models",
        "per_model_regulatory_state": [
          {
            "activator_state": "absent",
            "blocking_step": "late_activation",
            "brake_state": "released",
            "locus_state": "poised_unfired",
            "model": "B16_GP",
            "program": "absent"
          },
          {
            "activator_state": "absent",
            "blocking_step": "both_steps",
            "brake_state": "partial",
            "locus_state": "poised_unfired",
            "model": "B_ALL_GP",
            "program": "absent"
          },
          {
            "activator_state": "present",
            "blocking_step": "none",
            "brake_state": "released",
            "locus_state": "active",
            "model": "KPC_GP",
            "program": "present"
          },
          {
            "activator_state": "partial",
            "blocking_step": "early_release",
            "brake_state": "partial",
            "locus_state": "poised_unfired",
            "model": "mWnt_GP",
            "program": "partial"
          }
        ],
        "program_map": [
          {
            "call": "absent",
            "model": "B16_GP"
          },
          {
            "call": "absent",
            "model": "B_ALL_GP"
          },
          {
            "call": "present",
            "model": "KPC_GP"
          },
          {
            "call": "partial",
            "model": "mWnt_GP"
          }
        ],
        "ready_silent_fraction_relations": [
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": "<"
          },
          {
            "pair": "B16_GP_vs_KPC_GP",
            "relation": ">"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": "<"
          },
          {
            "pair": "B_ALL_GP_vs_B16_GP",
            "relation": ">"
          },
          {
            "pair": "B_ALL_GP_vs_KPC_GP",
            "relation": ">"
          },
          {
            "pair": "B_ALL_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_B16_GP",
            "relation": "<"
          },
          {
            "pair": "KPC_GP_vs_B_ALL_GP",
            "relation": "<"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": "<"
          },
          {
            "pair": "mWnt_GP_vs_B16_GP",
            "relation": ">"
          },
          {
            "pair": "mWnt_GP_vs_B_ALL_GP",
            "relation": "<"
          },
          {
            "pair": "mWnt_GP_vs_KPC_GP",
            "relation": ">"
          }
        ],
        "regulator_tuples": [
          {
            "evidence_basis": "own_abundance",
            "expr_vs_activity": "concordant",
            "gating_step": "early_release",
            "regulator": "Klf2",
            "role": "repressor"
          },
          {
            "evidence_basis": "own_abundance",
            "expr_vs_activity": "concordant",
            "gating_step": "early_release",
            "regulator": "S1pr1",
            "role": "repressor"
          },
          {
            "evidence_basis": "own_abundance",
            "expr_vs_activity": "concordant",
            "gating_step": "late_activation",
            "regulator": "Tgfbr2",
            "role": "activator"
          },
          {
            "evidence_basis": "own_abundance",
            "expr_vs_activity": "concordant",
            "gating_step": "late_activation",
            "regulator": "Smad3",
            "role": "activator"
          },
          {
            "evidence_basis": "target_program_behavior",
            "expr_vs_activity": "discordant",
            "gating_step": "late_activation",
            "regulator": "Runx3",
            "role": "activator"
          }
        ]
      },
      "nTurns": 1,
      "checkpointSignals": {
        "clustering": true,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": true,
        "nerveJointAnchor": false,
        "multiomeAssembly": true,
        "depthNormalizedAtac": false,
        "regulonInference": true,
        "cd8SubsetIntegration": true,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": false,
        "orthologProjection": false,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": true,
        "enrichmentFamilyRanking": true,
        "mousePseudobulkDesign": true,
        "cancerEpithelialDenominator": false,
        "geneLevelDe": true,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "trajectoryFile": "scbench-long/trajectories/green_2_tme_context_dependence__gpt-5.4__openai-codex__r3.json",
      "runId": "gpt-5.4__openai-codex__green_2_tme_context_dependence__r3",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-gpt-5-4-pi",
      "modelName": "gpt-5.4",
      "harness": "pi",
      "task": "green_2_tme_context_dependence",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 1,
      "passed": false,
      "score": 7,
      "durationS": 4054.05,
      "cost": 2.02883,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "program_map",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "sig_includes_itgae",
            "role": "gate",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Itgae"
            ]
          },
          {
            "name": "declared_a_known_confounder",
            "role": "gate",
            "field": "$.excluded_confounders[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "cf_signature_gut_contaminant",
            "role": "hard_fail",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "generic_signature_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd103_program_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "<"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "per_model_regulatory_state",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "poised_unfired"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "late_activation"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "closing_committed"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_tuples",
            "kind": "list",
            "matchKey": "regulator",
            "k": 1,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "role": {
                    "op": "equals",
                    "values": [
                      "repressor"
                    ]
                  },
                  "gating_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  },
                  "evidence_basis": {
                    "op": "equals",
                    "values": [
                      "target_program_behavior"
                    ]
                  },
                  "expr_vs_activity": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "locus_differential_gate",
            "role": "gate",
            "field": "locus_differential",
            "kind": "value",
            "expected": "uniform_across_models"
          },
          {
            "name": "formation_limited_by_gate",
            "role": "gate",
            "field": "formation_limited_by",
            "kind": "value",
            "expected": "absent_induction"
          },
          {
            "name": "locus_competent_in_naive_gate",
            "role": "gate",
            "field": "locus_competent_in_naive",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "ready_silent_fraction_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 3,
            "expected": [
              {
                "key": "B16_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_KPC_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "activator_chromatin_footprint_gate",
            "role": "gate",
            "field": "activator_chromatin_footprint",
            "kind": "value",
            "expected": "inconsistent_across_formers"
          },
          {
            "name": "exhaustion_comparable_gate",
            "role": "gate",
            "field": "exhaustion_comparable_across_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_nondeterminant_gate",
            "role": "gate",
            "field": "exhaustion_explains_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_separable_program_gate",
            "role": "gate",
            "field": "exhaustion_separable_program",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "exhaustion_counterexample_gate",
            "role": "gate",
            "field": "exhaustion_counterexample_model",
            "kind": "value",
            "expected": "KPC_GP"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "exhaustion_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "cf_inconsistent_present_without_activator",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "program",
            "entity": "present",
            "valueField": "activator_state",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_no_block_but_absent",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "none",
            "valueField": "program",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_late_block_without_poised_locus",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "late_activation",
            "valueField": "locus_state",
            "expected": [
              "closing_committed",
              "active"
            ]
          },
          {
            "name": "cf_inconsistent_early_block_with_brake_released",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "early_release",
            "valueField": "brake_state",
            "expected": [
              "released"
            ]
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: FAIL\n  x list_match: list_match (answer_field='program_map'): FAIL  (score=0.0)\n  + sig_includes_itgae: passed=True, score=1.0\n  + declared_a_known_confounder: passed=True, score=1.0\n  x list_match: list_match (answer_field='generic_signature_relations'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='cd103_program_relations'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='per_model_regulatory_state'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='regulator_tuples'): FAIL  (score=0.0)\n  x locus_differential_gate: passed=False, score=0.0\n  + formation_limited_by_gate: passed=True, score=1.0\n  + locus_competent_in_naive_gate: passed=True, score=1.0\n  x list_match: list_match (answer_field='ready_silent_fraction_relations'): FAIL  (score=0.0)\n  x activator_chromatin_footprint_gate: passed=False, score=0.0\n  x exhaustion_comparable_gate: passed=False, score=0.0\n  + exhaustion_nondeterminant_gate: passed=True, score=1.0\n  + exhaustion_separable_program_gate: passed=True, score=1.0\n  + exhaustion_counterexample_gate: passed=True, score=1.0\n  x list_match: list_match (answer_field='exhaustion_relations'): FAIL  (score=0.0)\n  x hard_fail 'cf_signature_gut_contaminant': TRIGGERED\n  + hard_fail 'cf_inconsistent_present_without_activator': not triggered\n  + hard_fail 'cf_inconsistent_no_block_but_absent': not triggered\n  + hard_fail 'cf_inconsistent_late_block_without_poised_locus': not triggered\n  + hard_fail 'cf_inconsistent_early_block_with_brake_released': not triggered",
      "metrics": {
        "pass_rule": "all",
        "scoring_count": 17,
        "scoring_passed": 7,
        "scoring_total_score": 7,
        "hard_fail_triggered": [
          "cf_signature_gut_contaminant"
        ]
      },
      "fieldScores": {
        "list_match": 0,
        "sig_includes_itgae": 1,
        "declared_a_known_confounder": 1,
        "locus_differential_gate": 0,
        "formation_limited_by_gate": 1,
        "locus_competent_in_naive_gate": 1,
        "activator_chromatin_footprint_gate": 0,
        "exhaustion_comparable_gate": 0,
        "exhaustion_nondeterminant_gate": 1,
        "exhaustion_separable_program_gate": 1,
        "exhaustion_counterexample_gate": 1
      },
      "graderChecks": [
        {
          "name": "program_map",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "sig_includes_itgae",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "declared_a_known_confounder",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "generic_signature_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "cd103_program_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "per_model_regulatory_state",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "regulator_tuples",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "locus_differential_gate",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "formation_limited_by_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "locus_competent_in_naive_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "ready_silent_fraction_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "activator_chromatin_footprint_gate",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "exhaustion_comparable_gate",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "exhaustion_nondeterminant_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "exhaustion_separable_program_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "exhaustion_counterexample_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "exhaustion_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "cf_signature_gut_contaminant",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "cf_inconsistent_present_without_activator",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_no_block_but_absent",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_late_block_without_poised_locus",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_early_block_with_brake_released",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "activator_chromatin_footprint": "consistent_in_formers",
        "cd103_program_relations": [
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": "~"
          },
          {
            "pair": "B16_GP_vs_KPC_GP",
            "relation": "<"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": "<"
          },
          {
            "pair": "B_ALL_GP_vs_KPC_GP",
            "relation": "<"
          },
          {
            "pair": "B_ALL_GP_vs_mWnt_GP",
            "relation": "<"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": ">"
          }
        ],
        "core_residency_signature": [
          "Itgae",
          "Nt5e",
          "Cdh1",
          "Ahr",
          "Cd101",
          "P2rx7"
        ],
        "excluded_confounders": [
          "Ccr9",
          "Hic1",
          "Xcl1",
          "Gzmb",
          "Ifng",
          "Tox",
          "Pdcd1",
          "Havcr2"
        ],
        "exhaustion_comparable_across_split": "yes",
        "exhaustion_counterexample_model": "KPC_GP",
        "exhaustion_explains_split": "no",
        "exhaustion_relations": [
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": "~"
          },
          {
            "pair": "B16_GP_vs_KPC_GP",
            "relation": "~"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "B_ALL_GP_vs_KPC_GP",
            "relation": "~"
          },
          {
            "pair": "B_ALL_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": ">"
          }
        ],
        "exhaustion_separable_program": "yes",
        "formation_limited_by": "absent_induction",
        "generic_signature_relations": [
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "B16_GP_vs_KPC_GP",
            "relation": "<"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": "<"
          },
          {
            "pair": "B_ALL_GP_vs_KPC_GP",
            "relation": "<"
          },
          {
            "pair": "B_ALL_GP_vs_mWnt_GP",
            "relation": "<"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": "~"
          }
        ],
        "locus_competent_in_naive": "yes",
        "locus_differential": "model_specific_element",
        "per_model_regulatory_state": [
          {
            "activator_state": "absent",
            "blocking_step": "late_activation",
            "brake_state": "released",
            "locus_state": "poised_unfired",
            "model": "B16_GP",
            "program": "absent"
          },
          {
            "activator_state": "absent",
            "blocking_step": "both_steps",
            "brake_state": "partial",
            "locus_state": "poised_unfired",
            "model": "B_ALL_GP",
            "program": "absent"
          },
          {
            "activator_state": "partial",
            "blocking_step": "late_activation",
            "brake_state": "partial",
            "locus_state": "poised_unfired",
            "model": "mWnt_GP",
            "program": "partial"
          },
          {
            "activator_state": "present",
            "blocking_step": "none",
            "brake_state": "released",
            "locus_state": "active",
            "model": "KPC_GP",
            "program": "present"
          }
        ],
        "program_map": [
          {
            "call": "absent",
            "model": "B16_GP"
          },
          {
            "call": "absent",
            "model": "B_ALL_GP"
          },
          {
            "call": "partial",
            "model": "mWnt_GP"
          },
          {
            "call": "present",
            "model": "KPC_GP"
          }
        ],
        "ready_silent_fraction_relations": [
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": "<"
          },
          {
            "pair": "B16_GP_vs_KPC_GP",
            "relation": ">"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": "<"
          },
          {
            "pair": "B_ALL_GP_vs_KPC_GP",
            "relation": ">"
          },
          {
            "pair": "B_ALL_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": "<"
          }
        ],
        "regulator_tuples": [
          {
            "evidence_basis": "own_abundance",
            "expr_vs_activity": "concordant",
            "gating_step": "early_release",
            "regulator": "Klf2",
            "role": "repressor"
          },
          {
            "evidence_basis": "own_abundance",
            "expr_vs_activity": "concordant",
            "gating_step": "early_release",
            "regulator": "Tcf7",
            "role": "repressor"
          },
          {
            "evidence_basis": "own_abundance",
            "expr_vs_activity": "concordant",
            "gating_step": "late_activation",
            "regulator": "Ahr",
            "role": "activator"
          },
          {
            "evidence_basis": "target_program_behavior",
            "expr_vs_activity": "discordant",
            "gating_step": "late_activation",
            "regulator": "Tgfbr2",
            "role": "activator"
          }
        ]
      },
      "nTurns": 56,
      "checkpointSignals": {
        "clustering": true,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": false,
        "nerveJointAnchor": false,
        "multiomeAssembly": true,
        "depthNormalizedAtac": true,
        "regulonInference": true,
        "cd8SubsetIntegration": true,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": false,
        "orthologProjection": false,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": false,
        "mousePseudobulkDesign": true,
        "cancerEpithelialDenominator": false,
        "geneLevelDe": true,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "trajectoryFile": "scbench-long/trajectories/green_2_tme_context_dependence__gpt-5.4__pi__r1.json",
      "runId": "gpt-5.4__pi__green_2_tme_context_dependence__r1",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-gpt-5-4-pi",
      "modelName": "gpt-5.4",
      "harness": "pi",
      "task": "green_2_tme_context_dependence",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 2,
      "passed": false,
      "score": 8,
      "durationS": 3508.88,
      "cost": 2.1812424999999998,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "program_map",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "sig_includes_itgae",
            "role": "gate",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Itgae"
            ]
          },
          {
            "name": "declared_a_known_confounder",
            "role": "gate",
            "field": "$.excluded_confounders[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "cf_signature_gut_contaminant",
            "role": "hard_fail",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "generic_signature_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd103_program_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "<"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "per_model_regulatory_state",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "poised_unfired"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "late_activation"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "closing_committed"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_tuples",
            "kind": "list",
            "matchKey": "regulator",
            "k": 1,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "role": {
                    "op": "equals",
                    "values": [
                      "repressor"
                    ]
                  },
                  "gating_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  },
                  "evidence_basis": {
                    "op": "equals",
                    "values": [
                      "target_program_behavior"
                    ]
                  },
                  "expr_vs_activity": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "locus_differential_gate",
            "role": "gate",
            "field": "locus_differential",
            "kind": "value",
            "expected": "uniform_across_models"
          },
          {
            "name": "formation_limited_by_gate",
            "role": "gate",
            "field": "formation_limited_by",
            "kind": "value",
            "expected": "absent_induction"
          },
          {
            "name": "locus_competent_in_naive_gate",
            "role": "gate",
            "field": "locus_competent_in_naive",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "ready_silent_fraction_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 3,
            "expected": [
              {
                "key": "B16_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_KPC_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "activator_chromatin_footprint_gate",
            "role": "gate",
            "field": "activator_chromatin_footprint",
            "kind": "value",
            "expected": "inconsistent_across_formers"
          },
          {
            "name": "exhaustion_comparable_gate",
            "role": "gate",
            "field": "exhaustion_comparable_across_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_nondeterminant_gate",
            "role": "gate",
            "field": "exhaustion_explains_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_separable_program_gate",
            "role": "gate",
            "field": "exhaustion_separable_program",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "exhaustion_counterexample_gate",
            "role": "gate",
            "field": "exhaustion_counterexample_model",
            "kind": "value",
            "expected": "KPC_GP"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "exhaustion_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "cf_inconsistent_present_without_activator",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "program",
            "entity": "present",
            "valueField": "activator_state",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_no_block_but_absent",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "none",
            "valueField": "program",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_late_block_without_poised_locus",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "late_activation",
            "valueField": "locus_state",
            "expected": [
              "closing_committed",
              "active"
            ]
          },
          {
            "name": "cf_inconsistent_early_block_with_brake_released",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "early_release",
            "valueField": "brake_state",
            "expected": [
              "released"
            ]
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: FAIL\n  x list_match: list_match (answer_field='program_map'): FAIL  (score=0.0)\n  + sig_includes_itgae: passed=True, score=1.0\n  + declared_a_known_confounder: passed=True, score=1.0\n  x list_match: list_match (answer_field='generic_signature_relations'): FAIL  (score=0.0)\n  + list_match: list_match (answer_field='cd103_program_relations'): PASS  (score=0.0)\n  x list_match: list_match (answer_field='per_model_regulatory_state'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='regulator_tuples'): FAIL  (score=0.0)\n  x locus_differential_gate: passed=False, score=0.0\n  + formation_limited_by_gate: passed=True, score=1.0\n  + locus_competent_in_naive_gate: passed=True, score=1.0\n  x list_match: list_match (answer_field='ready_silent_fraction_relations'): FAIL  (score=0.0)\n  x activator_chromatin_footprint_gate: passed=False, score=0.0\n  + exhaustion_comparable_gate: passed=True, score=1.0\n  + exhaustion_nondeterminant_gate: passed=True, score=1.0\n  + exhaustion_separable_program_gate: passed=True, score=1.0\n  + exhaustion_counterexample_gate: passed=True, score=1.0\n  x list_match: list_match (answer_field='exhaustion_relations'): FAIL  (score=0.0)\n  x hard_fail 'cf_signature_gut_contaminant': TRIGGERED\n  + hard_fail 'cf_inconsistent_present_without_activator': not triggered\n  + hard_fail 'cf_inconsistent_no_block_but_absent': not triggered\n  + hard_fail 'cf_inconsistent_late_block_without_poised_locus': not triggered\n  + hard_fail 'cf_inconsistent_early_block_with_brake_released': not triggered",
      "metrics": {
        "pass_rule": "all",
        "scoring_count": 17,
        "scoring_passed": 9,
        "scoring_total_score": 8,
        "hard_fail_triggered": [
          "cf_signature_gut_contaminant"
        ]
      },
      "fieldScores": {
        "list_match": 0,
        "sig_includes_itgae": 1,
        "declared_a_known_confounder": 1,
        "locus_differential_gate": 0,
        "formation_limited_by_gate": 1,
        "locus_competent_in_naive_gate": 1,
        "activator_chromatin_footprint_gate": 0,
        "exhaustion_comparable_gate": 1,
        "exhaustion_nondeterminant_gate": 1,
        "exhaustion_separable_program_gate": 1,
        "exhaustion_counterexample_gate": 1
      },
      "graderChecks": [
        {
          "name": "program_map",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "sig_includes_itgae",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "declared_a_known_confounder",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "generic_signature_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "cd103_program_relations",
          "kind": "list_match",
          "passed": true,
          "score": 0
        },
        {
          "name": "per_model_regulatory_state",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "regulator_tuples",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "locus_differential_gate",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "formation_limited_by_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "locus_competent_in_naive_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "ready_silent_fraction_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "activator_chromatin_footprint_gate",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "exhaustion_comparable_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "exhaustion_nondeterminant_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "exhaustion_separable_program_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "exhaustion_counterexample_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "exhaustion_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "cf_signature_gut_contaminant",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "cf_inconsistent_present_without_activator",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_no_block_but_absent",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_late_block_without_poised_locus",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_early_block_with_brake_released",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "activator_chromatin_footprint": "consistent_in_formers",
        "cd103_program_relations": [
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": "<"
          },
          {
            "pair": "B16_GP_vs_KPC_GP",
            "relation": "<"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": "<"
          },
          {
            "pair": "B_ALL_GP_vs_KPC_GP",
            "relation": "<"
          },
          {
            "pair": "B_ALL_GP_vs_mWnt_GP",
            "relation": "<"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": ">"
          }
        ],
        "core_residency_signature": [
          "Itgae",
          "Itga1",
          "Ahr",
          "Cd101",
          "P2rx7"
        ],
        "excluded_confounders": [
          "Ccr9",
          "Pdcd1",
          "Tox",
          "Lag3",
          "Havcr2",
          "Gzmb",
          "Eomes"
        ],
        "exhaustion_comparable_across_split": "no",
        "exhaustion_counterexample_model": "KPC_GP",
        "exhaustion_explains_split": "no",
        "exhaustion_relations": [
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": "~"
          },
          {
            "pair": "B16_GP_vs_KPC_GP",
            "relation": "~"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "B_ALL_GP_vs_KPC_GP",
            "relation": "~"
          },
          {
            "pair": "B_ALL_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": ">"
          }
        ],
        "exhaustion_separable_program": "yes",
        "formation_limited_by": "absent_induction",
        "generic_signature_relations": [
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": "~"
          },
          {
            "pair": "B16_GP_vs_KPC_GP",
            "relation": "<"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": "<"
          },
          {
            "pair": "B_ALL_GP_vs_KPC_GP",
            "relation": "<"
          },
          {
            "pair": "B_ALL_GP_vs_mWnt_GP",
            "relation": "<"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": "~"
          }
        ],
        "locus_competent_in_naive": "yes",
        "locus_differential": "model_specific_element",
        "per_model_regulatory_state": [
          {
            "activator_state": "absent",
            "blocking_step": "late_activation",
            "brake_state": "released",
            "locus_state": "poised_unfired",
            "model": "B16_GP",
            "program": "absent"
          },
          {
            "activator_state": "absent",
            "blocking_step": "both_steps",
            "brake_state": "partial",
            "locus_state": "poised_unfired",
            "model": "B_ALL_GP",
            "program": "absent"
          },
          {
            "activator_state": "present",
            "blocking_step": "none",
            "brake_state": "released",
            "locus_state": "active",
            "model": "KPC_GP",
            "program": "present"
          },
          {
            "activator_state": "partial",
            "blocking_step": "late_activation",
            "brake_state": "partial",
            "locus_state": "poised_unfired",
            "model": "mWnt_GP",
            "program": "partial"
          }
        ],
        "program_map": [
          {
            "call": "absent",
            "model": "B16_GP"
          },
          {
            "call": "absent",
            "model": "B_ALL_GP"
          },
          {
            "call": "present",
            "model": "KPC_GP"
          },
          {
            "call": "partial",
            "model": "mWnt_GP"
          }
        ],
        "ready_silent_fraction_relations": [
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": "<"
          },
          {
            "pair": "B16_GP_vs_KPC_GP",
            "relation": "~"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": "<"
          },
          {
            "pair": "B_ALL_GP_vs_KPC_GP",
            "relation": ">"
          },
          {
            "pair": "B_ALL_GP_vs_mWnt_GP",
            "relation": "~"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": "<"
          }
        ],
        "regulator_tuples": [
          {
            "evidence_basis": "own_abundance",
            "expr_vs_activity": "concordant",
            "gating_step": "early_release",
            "regulator": "Klf2",
            "role": "repressor"
          },
          {
            "evidence_basis": "own_abundance",
            "expr_vs_activity": "concordant",
            "gating_step": "early_release",
            "regulator": "Tcf7",
            "role": "repressor"
          },
          {
            "evidence_basis": "own_abundance",
            "expr_vs_activity": "concordant",
            "gating_step": "late_activation",
            "regulator": "Ahr",
            "role": "activator"
          },
          {
            "evidence_basis": "target_program_behavior",
            "expr_vs_activity": "discordant",
            "gating_step": "late_activation",
            "regulator": "Runx3",
            "role": "activator"
          }
        ]
      },
      "nTurns": 60,
      "checkpointSignals": {
        "clustering": true,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": true,
        "nerveJointAnchor": false,
        "multiomeAssembly": true,
        "depthNormalizedAtac": true,
        "regulonInference": true,
        "cd8SubsetIntegration": true,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": false,
        "orthologProjection": false,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": true,
        "enrichmentFamilyRanking": true,
        "mousePseudobulkDesign": true,
        "cancerEpithelialDenominator": false,
        "geneLevelDe": true,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "trajectoryFile": "scbench-long/trajectories/green_2_tme_context_dependence__gpt-5.4__pi__r2.json",
      "runId": "gpt-5.4__pi__green_2_tme_context_dependence__r2",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-gpt-5-4-pi",
      "modelName": "gpt-5.4",
      "harness": "pi",
      "task": "green_2_tme_context_dependence",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 3,
      "passed": false,
      "score": 7,
      "durationS": 1669.1,
      "cost": 1.5724580000000001,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "program_map",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "sig_includes_itgae",
            "role": "gate",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Itgae"
            ]
          },
          {
            "name": "declared_a_known_confounder",
            "role": "gate",
            "field": "$.excluded_confounders[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "cf_signature_gut_contaminant",
            "role": "hard_fail",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "generic_signature_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd103_program_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "<"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "per_model_regulatory_state",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "poised_unfired"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "late_activation"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "closing_committed"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_tuples",
            "kind": "list",
            "matchKey": "regulator",
            "k": 1,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "role": {
                    "op": "equals",
                    "values": [
                      "repressor"
                    ]
                  },
                  "gating_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  },
                  "evidence_basis": {
                    "op": "equals",
                    "values": [
                      "target_program_behavior"
                    ]
                  },
                  "expr_vs_activity": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "locus_differential_gate",
            "role": "gate",
            "field": "locus_differential",
            "kind": "value",
            "expected": "uniform_across_models"
          },
          {
            "name": "formation_limited_by_gate",
            "role": "gate",
            "field": "formation_limited_by",
            "kind": "value",
            "expected": "absent_induction"
          },
          {
            "name": "locus_competent_in_naive_gate",
            "role": "gate",
            "field": "locus_competent_in_naive",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "ready_silent_fraction_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 3,
            "expected": [
              {
                "key": "B16_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_KPC_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "activator_chromatin_footprint_gate",
            "role": "gate",
            "field": "activator_chromatin_footprint",
            "kind": "value",
            "expected": "inconsistent_across_formers"
          },
          {
            "name": "exhaustion_comparable_gate",
            "role": "gate",
            "field": "exhaustion_comparable_across_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_nondeterminant_gate",
            "role": "gate",
            "field": "exhaustion_explains_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_separable_program_gate",
            "role": "gate",
            "field": "exhaustion_separable_program",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "exhaustion_counterexample_gate",
            "role": "gate",
            "field": "exhaustion_counterexample_model",
            "kind": "value",
            "expected": "KPC_GP"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "exhaustion_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "cf_inconsistent_present_without_activator",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "program",
            "entity": "present",
            "valueField": "activator_state",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_no_block_but_absent",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "none",
            "valueField": "program",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_late_block_without_poised_locus",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "late_activation",
            "valueField": "locus_state",
            "expected": [
              "closing_committed",
              "active"
            ]
          },
          {
            "name": "cf_inconsistent_early_block_with_brake_released",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "early_release",
            "valueField": "brake_state",
            "expected": [
              "released"
            ]
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: FAIL\n  x list_match: list_match (answer_field='program_map'): FAIL  (score=0.0)\n  + sig_includes_itgae: passed=True, score=1.0\n  x declared_a_known_confounder: passed=False, score=0.0\n  x list_match: list_match (answer_field='generic_signature_relations'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='cd103_program_relations'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='per_model_regulatory_state'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='regulator_tuples'): FAIL  (score=0.0)\n  + locus_differential_gate: passed=True, score=1.0\n  x formation_limited_by_gate: passed=False, score=0.0\n  + locus_competent_in_naive_gate: passed=True, score=1.0\n  x list_match: list_match (answer_field='ready_silent_fraction_relations'): FAIL  (score=0.0)\n  x activator_chromatin_footprint_gate: passed=False, score=0.0\n  + exhaustion_comparable_gate: passed=True, score=1.0\n  + exhaustion_nondeterminant_gate: passed=True, score=1.0\n  + exhaustion_separable_program_gate: passed=True, score=1.0\n  + exhaustion_counterexample_gate: passed=True, score=1.0\n  x list_match: list_match (answer_field='exhaustion_relations'): FAIL  (score=0.0)\n  x hard_fail 'cf_signature_gut_contaminant': TRIGGERED\n  + hard_fail 'cf_inconsistent_present_without_activator': not triggered\n  + hard_fail 'cf_inconsistent_no_block_but_absent': not triggered\n  + hard_fail 'cf_inconsistent_late_block_without_poised_locus': not triggered\n  + hard_fail 'cf_inconsistent_early_block_with_brake_released': not triggered",
      "metrics": {
        "pass_rule": "all",
        "scoring_count": 17,
        "scoring_passed": 7,
        "scoring_total_score": 7,
        "hard_fail_triggered": [
          "cf_signature_gut_contaminant"
        ]
      },
      "fieldScores": {
        "list_match": 0,
        "sig_includes_itgae": 1,
        "declared_a_known_confounder": 0,
        "locus_differential_gate": 1,
        "formation_limited_by_gate": 0,
        "locus_competent_in_naive_gate": 1,
        "activator_chromatin_footprint_gate": 0,
        "exhaustion_comparable_gate": 1,
        "exhaustion_nondeterminant_gate": 1,
        "exhaustion_separable_program_gate": 1,
        "exhaustion_counterexample_gate": 1
      },
      "graderChecks": [
        {
          "name": "program_map",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "sig_includes_itgae",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "declared_a_known_confounder",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "generic_signature_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "cd103_program_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "per_model_regulatory_state",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "regulator_tuples",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "locus_differential_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "formation_limited_by_gate",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "locus_competent_in_naive_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "ready_silent_fraction_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "activator_chromatin_footprint_gate",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "exhaustion_comparable_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "exhaustion_nondeterminant_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "exhaustion_separable_program_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "exhaustion_counterexample_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "exhaustion_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "cf_signature_gut_contaminant",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "cf_inconsistent_present_without_activator",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_no_block_but_absent",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_late_block_without_poised_locus",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_early_block_with_brake_released",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "activator_chromatin_footprint": "consistent_in_formers",
        "cd103_program_relations": [
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": "~"
          },
          {
            "pair": "B16_GP_vs_KPC_GP",
            "relation": "<"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": "<"
          },
          {
            "pair": "B_ALL_GP_vs_KPC_GP",
            "relation": "<"
          },
          {
            "pair": "B_ALL_GP_vs_mWnt_GP",
            "relation": "<"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": ">"
          }
        ],
        "core_residency_signature": [
          "Itgae",
          "Itga1",
          "Nt5e",
          "Pmepa1",
          "Ldlrad4",
          "Skil"
        ],
        "excluded_confounders": [
          "Tox",
          "Pdcd1",
          "Gzmb",
          "Ctla4",
          "Nr4a2",
          "Rgs1",
          "Xcl1"
        ],
        "exhaustion_comparable_across_split": "no",
        "exhaustion_counterexample_model": "KPC_GP",
        "exhaustion_explains_split": "no",
        "exhaustion_relations": [
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": "~"
          },
          {
            "pair": "B16_GP_vs_KPC_GP",
            "relation": "~"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "B_ALL_GP_vs_KPC_GP",
            "relation": "~"
          },
          {
            "pair": "B_ALL_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": ">"
          }
        ],
        "exhaustion_separable_program": "yes",
        "formation_limited_by": "both",
        "generic_signature_relations": [
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "B16_GP_vs_KPC_GP",
            "relation": "<"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": "<"
          },
          {
            "pair": "B_ALL_GP_vs_KPC_GP",
            "relation": "<"
          },
          {
            "pair": "B_ALL_GP_vs_mWnt_GP",
            "relation": "<"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": "<"
          }
        ],
        "locus_competent_in_naive": "yes",
        "locus_differential": "uniform_across_models",
        "per_model_regulatory_state": [
          {
            "activator_state": "absent",
            "blocking_step": "late_activation",
            "brake_state": "released",
            "locus_state": "repressed",
            "model": "B16_GP",
            "program": "absent"
          },
          {
            "activator_state": "absent",
            "blocking_step": "both_steps",
            "brake_state": "partial",
            "locus_state": "poised_unfired",
            "model": "B_ALL_GP",
            "program": "absent"
          },
          {
            "activator_state": "present",
            "blocking_step": "none",
            "brake_state": "released",
            "locus_state": "active",
            "model": "KPC_GP",
            "program": "present"
          },
          {
            "activator_state": "partial",
            "blocking_step": "late_activation",
            "brake_state": "partial",
            "locus_state": "poised_unfired",
            "model": "mWnt_GP",
            "program": "partial"
          }
        ],
        "program_map": [
          {
            "call": "absent",
            "model": "B16_GP"
          },
          {
            "call": "absent",
            "model": "B_ALL_GP"
          },
          {
            "call": "present",
            "model": "KPC_GP"
          },
          {
            "call": "partial",
            "model": "mWnt_GP"
          }
        ],
        "ready_silent_fraction_relations": [
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": "<"
          },
          {
            "pair": "B16_GP_vs_KPC_GP",
            "relation": "~"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": "<"
          },
          {
            "pair": "B_ALL_GP_vs_KPC_GP",
            "relation": ">"
          },
          {
            "pair": "B_ALL_GP_vs_mWnt_GP",
            "relation": "~"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": "<"
          }
        ],
        "regulator_tuples": [
          {
            "evidence_basis": "own_abundance",
            "expr_vs_activity": "concordant",
            "gating_step": "early_release",
            "regulator": "Klf2",
            "role": "repressor"
          },
          {
            "evidence_basis": "target_program_behavior",
            "expr_vs_activity": "discordant",
            "gating_step": "late_activation",
            "regulator": "Smad3",
            "role": "activator"
          },
          {
            "evidence_basis": "target_program_behavior",
            "expr_vs_activity": "discordant",
            "gating_step": "late_activation",
            "regulator": "Runx3",
            "role": "activator"
          }
        ]
      },
      "nTurns": 49,
      "checkpointSignals": {
        "clustering": false,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": false,
        "nerveJointAnchor": false,
        "multiomeAssembly": true,
        "depthNormalizedAtac": true,
        "regulonInference": true,
        "cd8SubsetIntegration": true,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": false,
        "orthologProjection": false,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": true,
        "enrichmentFamilyRanking": false,
        "mousePseudobulkDesign": true,
        "cancerEpithelialDenominator": true,
        "geneLevelDe": true,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "trajectoryFile": "scbench-long/trajectories/green_2_tme_context_dependence__gpt-5.4__pi__r3.json",
      "runId": "gpt-5.4__pi__green_2_tme_context_dependence__r3",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-gpt-5-5-codex",
      "modelName": "gpt-5.5",
      "harness": "openai-codex",
      "task": "green_2_tme_context_dependence",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 1,
      "passed": false,
      "score": 8,
      "durationS": 1667.47,
      "cost": 4.0376080000000005,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "program_map",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "sig_includes_itgae",
            "role": "gate",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Itgae"
            ]
          },
          {
            "name": "declared_a_known_confounder",
            "role": "gate",
            "field": "$.excluded_confounders[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "cf_signature_gut_contaminant",
            "role": "hard_fail",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "generic_signature_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd103_program_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "<"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "per_model_regulatory_state",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "poised_unfired"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "late_activation"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "closing_committed"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_tuples",
            "kind": "list",
            "matchKey": "regulator",
            "k": 1,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "role": {
                    "op": "equals",
                    "values": [
                      "repressor"
                    ]
                  },
                  "gating_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  },
                  "evidence_basis": {
                    "op": "equals",
                    "values": [
                      "target_program_behavior"
                    ]
                  },
                  "expr_vs_activity": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "locus_differential_gate",
            "role": "gate",
            "field": "locus_differential",
            "kind": "value",
            "expected": "uniform_across_models"
          },
          {
            "name": "formation_limited_by_gate",
            "role": "gate",
            "field": "formation_limited_by",
            "kind": "value",
            "expected": "absent_induction"
          },
          {
            "name": "locus_competent_in_naive_gate",
            "role": "gate",
            "field": "locus_competent_in_naive",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "ready_silent_fraction_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 3,
            "expected": [
              {
                "key": "B16_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_KPC_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "activator_chromatin_footprint_gate",
            "role": "gate",
            "field": "activator_chromatin_footprint",
            "kind": "value",
            "expected": "inconsistent_across_formers"
          },
          {
            "name": "exhaustion_comparable_gate",
            "role": "gate",
            "field": "exhaustion_comparable_across_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_nondeterminant_gate",
            "role": "gate",
            "field": "exhaustion_explains_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_separable_program_gate",
            "role": "gate",
            "field": "exhaustion_separable_program",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "exhaustion_counterexample_gate",
            "role": "gate",
            "field": "exhaustion_counterexample_model",
            "kind": "value",
            "expected": "KPC_GP"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "exhaustion_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "cf_inconsistent_present_without_activator",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "program",
            "entity": "present",
            "valueField": "activator_state",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_no_block_but_absent",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "none",
            "valueField": "program",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_late_block_without_poised_locus",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "late_activation",
            "valueField": "locus_state",
            "expected": [
              "closing_committed",
              "active"
            ]
          },
          {
            "name": "cf_inconsistent_early_block_with_brake_released",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "early_release",
            "valueField": "brake_state",
            "expected": [
              "released"
            ]
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: FAIL\n  x list_match: list_match (answer_field='program_map'): FAIL  (score=0.0)\n  + sig_includes_itgae: passed=True, score=1.0\n  x declared_a_known_confounder: passed=False, score=0.0\n  x list_match: list_match (answer_field='generic_signature_relations'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='cd103_program_relations'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='per_model_regulatory_state'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='regulator_tuples'): FAIL  (score=0.0)\n  + locus_differential_gate: passed=True, score=1.0\n  + formation_limited_by_gate: passed=True, score=1.0\n  + locus_competent_in_naive_gate: passed=True, score=1.0\n  x list_match: list_match (answer_field='ready_silent_fraction_relations'): FAIL  (score=0.0)\n  x activator_chromatin_footprint_gate: passed=False, score=0.0\n  + exhaustion_comparable_gate: passed=True, score=1.0\n  + exhaustion_nondeterminant_gate: passed=True, score=1.0\n  + exhaustion_separable_program_gate: passed=True, score=1.0\n  + exhaustion_counterexample_gate: passed=True, score=1.0\n  x list_match: list_match (answer_field='exhaustion_relations'): FAIL  (score=0.0)\n  x hard_fail 'cf_signature_gut_contaminant': TRIGGERED\n  + hard_fail 'cf_inconsistent_present_without_activator': not triggered\n  + hard_fail 'cf_inconsistent_no_block_but_absent': not triggered\n  + hard_fail 'cf_inconsistent_late_block_without_poised_locus': not triggered\n  + hard_fail 'cf_inconsistent_early_block_with_brake_released': not triggered",
      "metrics": {
        "pass_rule": "all",
        "scoring_count": 17,
        "scoring_passed": 8,
        "scoring_total_score": 8,
        "hard_fail_triggered": [
          "cf_signature_gut_contaminant"
        ]
      },
      "fieldScores": {
        "list_match": 0,
        "sig_includes_itgae": 1,
        "declared_a_known_confounder": 0,
        "locus_differential_gate": 1,
        "formation_limited_by_gate": 1,
        "locus_competent_in_naive_gate": 1,
        "activator_chromatin_footprint_gate": 0,
        "exhaustion_comparable_gate": 1,
        "exhaustion_nondeterminant_gate": 1,
        "exhaustion_separable_program_gate": 1,
        "exhaustion_counterexample_gate": 1
      },
      "graderChecks": [
        {
          "name": "program_map",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "sig_includes_itgae",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "declared_a_known_confounder",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "generic_signature_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "cd103_program_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "per_model_regulatory_state",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "regulator_tuples",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "locus_differential_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "formation_limited_by_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "locus_competent_in_naive_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "ready_silent_fraction_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "activator_chromatin_footprint_gate",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "exhaustion_comparable_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "exhaustion_nondeterminant_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "exhaustion_separable_program_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "exhaustion_counterexample_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "exhaustion_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "cf_signature_gut_contaminant",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "cf_inconsistent_present_without_activator",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_no_block_but_absent",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_late_block_without_poised_locus",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_early_block_with_brake_released",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "activator_chromatin_footprint": "consistent_in_formers",
        "cd103_program_relations": [
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": "~"
          },
          {
            "pair": "B16_GP_vs_KPC_GP",
            "relation": "<"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": "<"
          },
          {
            "pair": "B_ALL_GP_vs_KPC_GP",
            "relation": "<"
          },
          {
            "pair": "B_ALL_GP_vs_mWnt_GP",
            "relation": "<"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": ">"
          }
        ],
        "core_residency_signature": [
          "Itgae",
          "Itga1",
          "Cd69",
          "Cxcr6"
        ],
        "excluded_confounders": [
          "Tox",
          "Pdcd1",
          "Lag3",
          "Havcr2",
          "Entpd1",
          "Tigit"
        ],
        "exhaustion_comparable_across_split": "no",
        "exhaustion_counterexample_model": "KPC_GP",
        "exhaustion_explains_split": "no",
        "exhaustion_relations": [
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": "<"
          },
          {
            "pair": "B16_GP_vs_KPC_GP",
            "relation": "~"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "B_ALL_GP_vs_KPC_GP",
            "relation": ">"
          },
          {
            "pair": "B_ALL_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": ">"
          }
        ],
        "exhaustion_separable_program": "yes",
        "formation_limited_by": "absent_induction",
        "generic_signature_relations": [
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "B16_GP_vs_KPC_GP",
            "relation": "<"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": "<"
          },
          {
            "pair": "B_ALL_GP_vs_KPC_GP",
            "relation": "<"
          },
          {
            "pair": "B_ALL_GP_vs_mWnt_GP",
            "relation": "<"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": "~"
          }
        ],
        "locus_competent_in_naive": "yes",
        "locus_differential": "uniform_across_models",
        "per_model_regulatory_state": [
          {
            "activator_state": "partial",
            "blocking_step": "late_activation",
            "brake_state": "released",
            "locus_state": "poised_unfired",
            "model": "B16_GP",
            "program": "absent"
          },
          {
            "activator_state": "partial",
            "blocking_step": "both_steps",
            "brake_state": "partial",
            "locus_state": "poised_unfired",
            "model": "B_ALL_GP",
            "program": "absent"
          },
          {
            "activator_state": "present",
            "blocking_step": "none",
            "brake_state": "released",
            "locus_state": "active",
            "model": "KPC_GP",
            "program": "present"
          },
          {
            "activator_state": "partial",
            "blocking_step": "both_steps",
            "brake_state": "engaged",
            "locus_state": "active",
            "model": "mWnt_GP",
            "program": "partial"
          }
        ],
        "program_map": [
          {
            "call": "absent",
            "model": "B16_GP"
          },
          {
            "call": "absent",
            "model": "B_ALL_GP"
          },
          {
            "call": "present",
            "model": "KPC_GP"
          },
          {
            "call": "partial",
            "model": "mWnt_GP"
          }
        ],
        "ready_silent_fraction_relations": [
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": "<"
          },
          {
            "pair": "B16_GP_vs_KPC_GP",
            "relation": ">"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": "<"
          },
          {
            "pair": "B_ALL_GP_vs_KPC_GP",
            "relation": ">"
          },
          {
            "pair": "B_ALL_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": "<"
          }
        ],
        "regulator_tuples": [
          {
            "evidence_basis": "own_abundance",
            "expr_vs_activity": "concordant",
            "gating_step": "early_release",
            "regulator": "Klf2",
            "role": "repressor"
          },
          {
            "evidence_basis": "own_abundance",
            "expr_vs_activity": "concordant",
            "gating_step": "early_release",
            "regulator": "S1pr1",
            "role": "repressor"
          },
          {
            "evidence_basis": "own_abundance",
            "expr_vs_activity": "concordant",
            "gating_step": "early_release",
            "regulator": "Tcf7",
            "role": "repressor"
          },
          {
            "evidence_basis": "target_program_behavior",
            "expr_vs_activity": "discordant",
            "gating_step": "late_activation",
            "regulator": "Runx3",
            "role": "activator"
          },
          {
            "evidence_basis": "target_program_behavior",
            "expr_vs_activity": "discordant",
            "gating_step": "late_activation",
            "regulator": "Bhlhe40",
            "role": "activator"
          },
          {
            "evidence_basis": "target_program_behavior",
            "expr_vs_activity": "discordant",
            "gating_step": "late_activation",
            "regulator": "Nr4a3",
            "role": "activator"
          },
          {
            "evidence_basis": "target_program_behavior",
            "expr_vs_activity": "discordant",
            "gating_step": "none",
            "regulator": "Tox",
            "role": "none"
          }
        ]
      },
      "nTurns": 1,
      "checkpointSignals": {
        "clustering": false,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": true,
        "nerveJointAnchor": false,
        "multiomeAssembly": true,
        "depthNormalizedAtac": true,
        "regulonInference": true,
        "cd8SubsetIntegration": true,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": false,
        "orthologProjection": false,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": false,
        "mousePseudobulkDesign": false,
        "cancerEpithelialDenominator": false,
        "geneLevelDe": false,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "trajectoryFile": "scbench-long/trajectories/green_2_tme_context_dependence__gpt-5.5__openai-codex__r1.json",
      "runId": "gpt-5.5__openai-codex__green_2_tme_context_dependence__r1",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-gpt-5-5-codex",
      "modelName": "gpt-5.5",
      "harness": "openai-codex",
      "task": "green_2_tme_context_dependence",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 2,
      "passed": false,
      "score": 6,
      "durationS": 2509.32,
      "cost": 3.497576,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "program_map",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "sig_includes_itgae",
            "role": "gate",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Itgae"
            ]
          },
          {
            "name": "declared_a_known_confounder",
            "role": "gate",
            "field": "$.excluded_confounders[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "cf_signature_gut_contaminant",
            "role": "hard_fail",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "generic_signature_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd103_program_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "<"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "per_model_regulatory_state",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "poised_unfired"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "late_activation"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "closing_committed"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_tuples",
            "kind": "list",
            "matchKey": "regulator",
            "k": 1,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "role": {
                    "op": "equals",
                    "values": [
                      "repressor"
                    ]
                  },
                  "gating_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  },
                  "evidence_basis": {
                    "op": "equals",
                    "values": [
                      "target_program_behavior"
                    ]
                  },
                  "expr_vs_activity": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "locus_differential_gate",
            "role": "gate",
            "field": "locus_differential",
            "kind": "value",
            "expected": "uniform_across_models"
          },
          {
            "name": "formation_limited_by_gate",
            "role": "gate",
            "field": "formation_limited_by",
            "kind": "value",
            "expected": "absent_induction"
          },
          {
            "name": "locus_competent_in_naive_gate",
            "role": "gate",
            "field": "locus_competent_in_naive",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "ready_silent_fraction_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 3,
            "expected": [
              {
                "key": "B16_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_KPC_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "activator_chromatin_footprint_gate",
            "role": "gate",
            "field": "activator_chromatin_footprint",
            "kind": "value",
            "expected": "inconsistent_across_formers"
          },
          {
            "name": "exhaustion_comparable_gate",
            "role": "gate",
            "field": "exhaustion_comparable_across_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_nondeterminant_gate",
            "role": "gate",
            "field": "exhaustion_explains_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_separable_program_gate",
            "role": "gate",
            "field": "exhaustion_separable_program",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "exhaustion_counterexample_gate",
            "role": "gate",
            "field": "exhaustion_counterexample_model",
            "kind": "value",
            "expected": "KPC_GP"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "exhaustion_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "cf_inconsistent_present_without_activator",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "program",
            "entity": "present",
            "valueField": "activator_state",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_no_block_but_absent",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "none",
            "valueField": "program",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_late_block_without_poised_locus",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "late_activation",
            "valueField": "locus_state",
            "expected": [
              "closing_committed",
              "active"
            ]
          },
          {
            "name": "cf_inconsistent_early_block_with_brake_released",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "early_release",
            "valueField": "brake_state",
            "expected": [
              "released"
            ]
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: FAIL\n  x list_match: list_match (answer_field='program_map'): FAIL  (score=0.0)\n  + sig_includes_itgae: passed=True, score=1.0\n  x declared_a_known_confounder: passed=False, score=0.0\n  x list_match: list_match (answer_field='generic_signature_relations'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='cd103_program_relations'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='per_model_regulatory_state'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='regulator_tuples'): FAIL  (score=0.0)\n  + locus_differential_gate: passed=True, score=1.0\n  + formation_limited_by_gate: passed=True, score=1.0\n  + locus_competent_in_naive_gate: passed=True, score=1.0\n  x list_match: list_match (answer_field='ready_silent_fraction_relations'): FAIL  (score=0.0)\n  x activator_chromatin_footprint_gate: passed=False, score=0.0\n  x exhaustion_comparable_gate: passed=False, score=0.0\n  + exhaustion_nondeterminant_gate: passed=True, score=1.0\n  + exhaustion_separable_program_gate: passed=True, score=1.0\n  x exhaustion_counterexample_gate: passed=False, score=0.0\n  x list_match: list_match (answer_field='exhaustion_relations'): FAIL  (score=0.0)\n  x hard_fail 'cf_signature_gut_contaminant': TRIGGERED\n  + hard_fail 'cf_inconsistent_present_without_activator': not triggered\n  + hard_fail 'cf_inconsistent_no_block_but_absent': not triggered\n  + hard_fail 'cf_inconsistent_late_block_without_poised_locus': not triggered\n  + hard_fail 'cf_inconsistent_early_block_with_brake_released': not triggered",
      "metrics": {
        "pass_rule": "all",
        "scoring_count": 17,
        "scoring_passed": 6,
        "scoring_total_score": 6,
        "hard_fail_triggered": [
          "cf_signature_gut_contaminant"
        ]
      },
      "fieldScores": {
        "list_match": 0,
        "sig_includes_itgae": 1,
        "declared_a_known_confounder": 0,
        "locus_differential_gate": 1,
        "formation_limited_by_gate": 1,
        "locus_competent_in_naive_gate": 1,
        "activator_chromatin_footprint_gate": 0,
        "exhaustion_comparable_gate": 0,
        "exhaustion_nondeterminant_gate": 1,
        "exhaustion_separable_program_gate": 1,
        "exhaustion_counterexample_gate": 0
      },
      "graderChecks": [
        {
          "name": "program_map",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "sig_includes_itgae",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "declared_a_known_confounder",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "generic_signature_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "cd103_program_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "per_model_regulatory_state",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "regulator_tuples",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "locus_differential_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "formation_limited_by_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "locus_competent_in_naive_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "ready_silent_fraction_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "activator_chromatin_footprint_gate",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "exhaustion_comparable_gate",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "exhaustion_nondeterminant_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "exhaustion_separable_program_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "exhaustion_counterexample_gate",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "exhaustion_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "cf_signature_gut_contaminant",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "cf_inconsistent_present_without_activator",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_no_block_but_absent",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_late_block_without_poised_locus",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_early_block_with_brake_released",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "activator_chromatin_footprint": "consistent_in_formers",
        "cd103_program_relations": [
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": "~"
          },
          {
            "pair": "B16_GP_vs_KPC_GP",
            "relation": "<"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": "<"
          },
          {
            "pair": "B_ALL_GP_vs_KPC_GP",
            "relation": "<"
          },
          {
            "pair": "B_ALL_GP_vs_mWnt_GP",
            "relation": "<"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": ">"
          }
        ],
        "core_residency_signature": [
          "Itgae",
          "Itga1",
          "Cd69",
          "Cxcr6",
          "Rgs1",
          "Ahr",
          "Runx3",
          "Bhlhe40",
          "Cd101",
          "Cdh1",
          "Ccr9",
          "Cd160"
        ],
        "excluded_confounders": [
          "Pdcd1",
          "Lag3",
          "Havcr2",
          "Tox",
          "Ctla4",
          "Entpd1",
          "Gzmb",
          "Gzma",
          "Prf1",
          "Nkg7",
          "Ifng",
          "Ccl3",
          "Ccl4",
          "Ccl5",
          "Mki67"
        ],
        "exhaustion_comparable_across_split": "yes",
        "exhaustion_counterexample_model": "B16_GP",
        "exhaustion_explains_split": "no",
        "exhaustion_relations": [
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": "~"
          },
          {
            "pair": "B16_GP_vs_KPC_GP",
            "relation": "~"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "B_ALL_GP_vs_KPC_GP",
            "relation": "<"
          },
          {
            "pair": "B_ALL_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": ">"
          }
        ],
        "exhaustion_separable_program": "yes",
        "formation_limited_by": "absent_induction",
        "generic_signature_relations": [
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": "<"
          },
          {
            "pair": "B16_GP_vs_KPC_GP",
            "relation": "<"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "B_ALL_GP_vs_KPC_GP",
            "relation": "~"
          },
          {
            "pair": "B_ALL_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": ">"
          }
        ],
        "locus_competent_in_naive": "yes",
        "locus_differential": "uniform_across_models",
        "per_model_regulatory_state": [
          {
            "activator_state": "absent",
            "blocking_step": "late_activation",
            "brake_state": "released",
            "locus_state": "poised_unfired",
            "model": "B16_GP",
            "program": "absent"
          },
          {
            "activator_state": "partial",
            "blocking_step": "both_steps",
            "brake_state": "engaged",
            "locus_state": "poised_unfired",
            "model": "B_ALL_GP",
            "program": "absent"
          },
          {
            "activator_state": "present",
            "blocking_step": "none",
            "brake_state": "released",
            "locus_state": "active",
            "model": "KPC_GP",
            "program": "present"
          },
          {
            "activator_state": "present",
            "blocking_step": "early_release",
            "brake_state": "engaged",
            "locus_state": "poised_unfired",
            "model": "mWnt_GP",
            "program": "partial"
          }
        ],
        "program_map": [
          {
            "call": "absent",
            "model": "B16_GP"
          },
          {
            "call": "absent",
            "model": "B_ALL_GP"
          },
          {
            "call": "present",
            "model": "KPC_GP"
          },
          {
            "call": "partial",
            "model": "mWnt_GP"
          }
        ],
        "ready_silent_fraction_relations": [
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": "<"
          },
          {
            "pair": "B16_GP_vs_KPC_GP",
            "relation": ">"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": "<"
          },
          {
            "pair": "B_ALL_GP_vs_KPC_GP",
            "relation": ">"
          },
          {
            "pair": "B_ALL_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": "<"
          }
        ],
        "regulator_tuples": [
          {
            "evidence_basis": "own_abundance",
            "expr_vs_activity": "concordant",
            "gating_step": "early_release",
            "regulator": "Klf2",
            "role": "repressor"
          },
          {
            "evidence_basis": "own_abundance",
            "expr_vs_activity": "concordant",
            "gating_step": "early_release",
            "regulator": "S1pr1",
            "role": "repressor"
          },
          {
            "evidence_basis": "own_abundance",
            "expr_vs_activity": "concordant",
            "gating_step": "early_release",
            "regulator": "Tcf7",
            "role": "repressor"
          },
          {
            "evidence_basis": "own_abundance",
            "expr_vs_activity": "concordant",
            "gating_step": "late_activation",
            "regulator": "Ahr",
            "role": "activator"
          },
          {
            "evidence_basis": "own_abundance",
            "expr_vs_activity": "concordant",
            "gating_step": "late_activation",
            "regulator": "Smad3",
            "role": "activator"
          },
          {
            "evidence_basis": "target_program_behavior",
            "expr_vs_activity": "discordant",
            "gating_step": "late_activation",
            "regulator": "Runx3",
            "role": "activator"
          }
        ]
      },
      "nTurns": 1,
      "checkpointSignals": {
        "clustering": false,
        "markerAnnotation": false,
        "scoreBasedAnnotation": false,
        "normalization": true,
        "nerveJointAnchor": false,
        "multiomeAssembly": true,
        "depthNormalizedAtac": true,
        "regulonInference": true,
        "cd8SubsetIntegration": true,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": false,
        "orthologProjection": false,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": false,
        "mousePseudobulkDesign": false,
        "cancerEpithelialDenominator": false,
        "geneLevelDe": false,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "trajectoryFile": "scbench-long/trajectories/green_2_tme_context_dependence__gpt-5.5__openai-codex__r2.json",
      "runId": "gpt-5.5__openai-codex__green_2_tme_context_dependence__r2",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-gpt-5-5-codex",
      "modelName": "gpt-5.5",
      "harness": "openai-codex",
      "task": "green_2_tme_context_dependence",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 3,
      "passed": false,
      "score": 7,
      "durationS": 1646.17,
      "cost": 3.448083,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "program_map",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "sig_includes_itgae",
            "role": "gate",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Itgae"
            ]
          },
          {
            "name": "declared_a_known_confounder",
            "role": "gate",
            "field": "$.excluded_confounders[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "cf_signature_gut_contaminant",
            "role": "hard_fail",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "generic_signature_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd103_program_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "<"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "per_model_regulatory_state",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "poised_unfired"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "late_activation"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "closing_committed"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_tuples",
            "kind": "list",
            "matchKey": "regulator",
            "k": 1,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "role": {
                    "op": "equals",
                    "values": [
                      "repressor"
                    ]
                  },
                  "gating_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  },
                  "evidence_basis": {
                    "op": "equals",
                    "values": [
                      "target_program_behavior"
                    ]
                  },
                  "expr_vs_activity": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "locus_differential_gate",
            "role": "gate",
            "field": "locus_differential",
            "kind": "value",
            "expected": "uniform_across_models"
          },
          {
            "name": "formation_limited_by_gate",
            "role": "gate",
            "field": "formation_limited_by",
            "kind": "value",
            "expected": "absent_induction"
          },
          {
            "name": "locus_competent_in_naive_gate",
            "role": "gate",
            "field": "locus_competent_in_naive",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "ready_silent_fraction_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 3,
            "expected": [
              {
                "key": "B16_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_KPC_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "activator_chromatin_footprint_gate",
            "role": "gate",
            "field": "activator_chromatin_footprint",
            "kind": "value",
            "expected": "inconsistent_across_formers"
          },
          {
            "name": "exhaustion_comparable_gate",
            "role": "gate",
            "field": "exhaustion_comparable_across_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_nondeterminant_gate",
            "role": "gate",
            "field": "exhaustion_explains_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_separable_program_gate",
            "role": "gate",
            "field": "exhaustion_separable_program",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "exhaustion_counterexample_gate",
            "role": "gate",
            "field": "exhaustion_counterexample_model",
            "kind": "value",
            "expected": "KPC_GP"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "exhaustion_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "cf_inconsistent_present_without_activator",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "program",
            "entity": "present",
            "valueField": "activator_state",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_no_block_but_absent",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "none",
            "valueField": "program",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_late_block_without_poised_locus",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "late_activation",
            "valueField": "locus_state",
            "expected": [
              "closing_committed",
              "active"
            ]
          },
          {
            "name": "cf_inconsistent_early_block_with_brake_released",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "early_release",
            "valueField": "brake_state",
            "expected": [
              "released"
            ]
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: FAIL\n  x list_match: list_match (answer_field='program_map'): FAIL  (score=0.0)\n  + sig_includes_itgae: passed=True, score=1.0\n  x declared_a_known_confounder: passed=False, score=0.0\n  x list_match: list_match (answer_field='generic_signature_relations'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='cd103_program_relations'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='per_model_regulatory_state'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='regulator_tuples'): FAIL  (score=0.0)\n  + locus_differential_gate: passed=True, score=1.0\n  + formation_limited_by_gate: passed=True, score=1.0\n  + locus_competent_in_naive_gate: passed=True, score=1.0\n  x list_match: list_match (answer_field='ready_silent_fraction_relations'): FAIL  (score=0.0)\n  x activator_chromatin_footprint_gate: passed=False, score=0.0\n  x exhaustion_comparable_gate: passed=False, score=0.0\n  + exhaustion_nondeterminant_gate: passed=True, score=1.0\n  + exhaustion_separable_program_gate: passed=True, score=1.0\n  + exhaustion_counterexample_gate: passed=True, score=1.0\n  x list_match: list_match (answer_field='exhaustion_relations'): FAIL  (score=0.0)\n  x hard_fail 'cf_signature_gut_contaminant': TRIGGERED\n  + hard_fail 'cf_inconsistent_present_without_activator': not triggered\n  + hard_fail 'cf_inconsistent_no_block_but_absent': not triggered\n  + hard_fail 'cf_inconsistent_late_block_without_poised_locus': not triggered\n  + hard_fail 'cf_inconsistent_early_block_with_brake_released': not triggered",
      "metrics": {
        "pass_rule": "all",
        "scoring_count": 17,
        "scoring_passed": 7,
        "scoring_total_score": 7,
        "hard_fail_triggered": [
          "cf_signature_gut_contaminant"
        ]
      },
      "fieldScores": {
        "list_match": 0,
        "sig_includes_itgae": 1,
        "declared_a_known_confounder": 0,
        "locus_differential_gate": 1,
        "formation_limited_by_gate": 1,
        "locus_competent_in_naive_gate": 1,
        "activator_chromatin_footprint_gate": 0,
        "exhaustion_comparable_gate": 0,
        "exhaustion_nondeterminant_gate": 1,
        "exhaustion_separable_program_gate": 1,
        "exhaustion_counterexample_gate": 1
      },
      "graderChecks": [
        {
          "name": "program_map",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "sig_includes_itgae",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "declared_a_known_confounder",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "generic_signature_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "cd103_program_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "per_model_regulatory_state",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "regulator_tuples",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "locus_differential_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "formation_limited_by_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "locus_competent_in_naive_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "ready_silent_fraction_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "activator_chromatin_footprint_gate",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "exhaustion_comparable_gate",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "exhaustion_nondeterminant_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "exhaustion_separable_program_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "exhaustion_counterexample_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "exhaustion_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "cf_signature_gut_contaminant",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "cf_inconsistent_present_without_activator",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_no_block_but_absent",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_late_block_without_poised_locus",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_early_block_with_brake_released",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "activator_chromatin_footprint": "consistent_in_formers",
        "cd103_program_relations": [
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": "~"
          },
          {
            "pair": "B16_GP_vs_KPC_GP",
            "relation": "<"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": "<"
          },
          {
            "pair": "B_ALL_GP_vs_KPC_GP",
            "relation": "<"
          },
          {
            "pair": "B_ALL_GP_vs_mWnt_GP",
            "relation": "<"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": ">"
          }
        ],
        "core_residency_signature": [
          "Itgae",
          "Itga1",
          "Cd69",
          "Cxcr6",
          "Rgs1"
        ],
        "excluded_confounders": [
          "Ccl5",
          "Nkg7",
          "Gzmb",
          "Gzma",
          "Prf1",
          "Ifng",
          "Xcl1",
          "Pdcd1",
          "Tox",
          "Lag3",
          "Havcr2",
          "Entpd1",
          "Ctla4"
        ],
        "exhaustion_comparable_across_split": "yes",
        "exhaustion_counterexample_model": "KPC_GP",
        "exhaustion_explains_split": "no",
        "exhaustion_relations": [
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": "~"
          },
          {
            "pair": "B16_GP_vs_KPC_GP",
            "relation": "~"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "B_ALL_GP_vs_KPC_GP",
            "relation": "~"
          },
          {
            "pair": "B_ALL_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": ">"
          }
        ],
        "exhaustion_separable_program": "yes",
        "formation_limited_by": "absent_induction",
        "generic_signature_relations": [
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "B16_GP_vs_KPC_GP",
            "relation": "<"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": "<"
          },
          {
            "pair": "B_ALL_GP_vs_KPC_GP",
            "relation": "<"
          },
          {
            "pair": "B_ALL_GP_vs_mWnt_GP",
            "relation": "<"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": "<"
          }
        ],
        "locus_competent_in_naive": "yes",
        "locus_differential": "uniform_across_models",
        "per_model_regulatory_state": [
          {
            "activator_state": "absent",
            "blocking_step": "late_activation",
            "brake_state": "released",
            "locus_state": "poised_unfired",
            "model": "B16_GP",
            "program": "absent"
          },
          {
            "activator_state": "absent",
            "blocking_step": "both_steps",
            "brake_state": "engaged",
            "locus_state": "poised_unfired",
            "model": "B_ALL_GP",
            "program": "absent"
          },
          {
            "activator_state": "present",
            "blocking_step": "none",
            "brake_state": "released",
            "locus_state": "active",
            "model": "KPC_GP",
            "program": "present"
          },
          {
            "activator_state": "present",
            "blocking_step": "early_release",
            "brake_state": "partial",
            "locus_state": "poised_unfired",
            "model": "mWnt_GP",
            "program": "partial"
          }
        ],
        "program_map": [
          {
            "call": "absent",
            "model": "B16_GP"
          },
          {
            "call": "absent",
            "model": "B_ALL_GP"
          },
          {
            "call": "present",
            "model": "KPC_GP"
          },
          {
            "call": "partial",
            "model": "mWnt_GP"
          }
        ],
        "ready_silent_fraction_relations": [
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": "<"
          },
          {
            "pair": "B16_GP_vs_KPC_GP",
            "relation": ">"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": "<"
          },
          {
            "pair": "B_ALL_GP_vs_KPC_GP",
            "relation": ">"
          },
          {
            "pair": "B_ALL_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": "<"
          }
        ],
        "regulator_tuples": [
          {
            "evidence_basis": "own_abundance",
            "expr_vs_activity": "concordant",
            "gating_step": "early_release",
            "regulator": "Klf2",
            "role": "repressor"
          },
          {
            "evidence_basis": "own_abundance",
            "expr_vs_activity": "concordant",
            "gating_step": "early_release",
            "regulator": "S1pr1",
            "role": "repressor"
          },
          {
            "evidence_basis": "own_abundance",
            "expr_vs_activity": "concordant",
            "gating_step": "early_release",
            "regulator": "Zeb2",
            "role": "repressor"
          },
          {
            "evidence_basis": "own_abundance",
            "expr_vs_activity": "concordant",
            "gating_step": "late_activation",
            "regulator": "Ahr",
            "role": "activator"
          },
          {
            "evidence_basis": "own_abundance",
            "expr_vs_activity": "concordant",
            "gating_step": "late_activation",
            "regulator": "Tgfbr2",
            "role": "activator"
          },
          {
            "evidence_basis": "target_program_behavior",
            "expr_vs_activity": "discordant",
            "gating_step": "late_activation",
            "regulator": "Runx3",
            "role": "activator"
          },
          {
            "evidence_basis": "target_program_behavior",
            "expr_vs_activity": "discordant",
            "gating_step": "none",
            "regulator": "Pdcd1",
            "role": "none"
          },
          {
            "evidence_basis": "target_program_behavior",
            "expr_vs_activity": "discordant",
            "gating_step": "none",
            "regulator": "Tox",
            "role": "none"
          }
        ]
      },
      "nTurns": 1,
      "checkpointSignals": {
        "clustering": false,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": true,
        "nerveJointAnchor": false,
        "multiomeAssembly": true,
        "depthNormalizedAtac": true,
        "regulonInference": true,
        "cd8SubsetIntegration": true,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": false,
        "orthologProjection": false,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": true,
        "mousePseudobulkDesign": false,
        "cancerEpithelialDenominator": false,
        "geneLevelDe": false,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "trajectoryFile": "scbench-long/trajectories/green_2_tme_context_dependence__gpt-5.5__openai-codex__r3.json",
      "runId": "gpt-5.5__openai-codex__green_2_tme_context_dependence__r3",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-gpt-5-5-pi",
      "modelName": "gpt-5.5",
      "harness": "pi",
      "task": "green_2_tme_context_dependence",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 1,
      "passed": false,
      "score": 6,
      "durationS": 2725.51,
      "cost": 2.5449159999999997,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "program_map",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "sig_includes_itgae",
            "role": "gate",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Itgae"
            ]
          },
          {
            "name": "declared_a_known_confounder",
            "role": "gate",
            "field": "$.excluded_confounders[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "cf_signature_gut_contaminant",
            "role": "hard_fail",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "generic_signature_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd103_program_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "<"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "per_model_regulatory_state",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "poised_unfired"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "late_activation"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "closing_committed"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_tuples",
            "kind": "list",
            "matchKey": "regulator",
            "k": 1,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "role": {
                    "op": "equals",
                    "values": [
                      "repressor"
                    ]
                  },
                  "gating_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  },
                  "evidence_basis": {
                    "op": "equals",
                    "values": [
                      "target_program_behavior"
                    ]
                  },
                  "expr_vs_activity": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "locus_differential_gate",
            "role": "gate",
            "field": "locus_differential",
            "kind": "value",
            "expected": "uniform_across_models"
          },
          {
            "name": "formation_limited_by_gate",
            "role": "gate",
            "field": "formation_limited_by",
            "kind": "value",
            "expected": "absent_induction"
          },
          {
            "name": "locus_competent_in_naive_gate",
            "role": "gate",
            "field": "locus_competent_in_naive",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "ready_silent_fraction_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 3,
            "expected": [
              {
                "key": "B16_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_KPC_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "activator_chromatin_footprint_gate",
            "role": "gate",
            "field": "activator_chromatin_footprint",
            "kind": "value",
            "expected": "inconsistent_across_formers"
          },
          {
            "name": "exhaustion_comparable_gate",
            "role": "gate",
            "field": "exhaustion_comparable_across_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_nondeterminant_gate",
            "role": "gate",
            "field": "exhaustion_explains_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_separable_program_gate",
            "role": "gate",
            "field": "exhaustion_separable_program",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "exhaustion_counterexample_gate",
            "role": "gate",
            "field": "exhaustion_counterexample_model",
            "kind": "value",
            "expected": "KPC_GP"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "exhaustion_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "cf_inconsistent_present_without_activator",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "program",
            "entity": "present",
            "valueField": "activator_state",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_no_block_but_absent",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "none",
            "valueField": "program",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_late_block_without_poised_locus",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "late_activation",
            "valueField": "locus_state",
            "expected": [
              "closing_committed",
              "active"
            ]
          },
          {
            "name": "cf_inconsistent_early_block_with_brake_released",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "early_release",
            "valueField": "brake_state",
            "expected": [
              "released"
            ]
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: FAIL\n  x list_match: list_match (answer_field='program_map'): FAIL  (score=0.0)\n  + sig_includes_itgae: passed=True, score=1.0\n  x declared_a_known_confounder: passed=False, score=0.0\n  x list_match: list_match (answer_field='generic_signature_relations'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='cd103_program_relations'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='per_model_regulatory_state'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='regulator_tuples'): FAIL  (score=0.0)\n  + locus_differential_gate: passed=True, score=1.0\n  + formation_limited_by_gate: passed=True, score=1.0\n  + locus_competent_in_naive_gate: passed=True, score=1.0\n  x list_match: list_match (answer_field='ready_silent_fraction_relations'): FAIL  (score=0.0)\n  x activator_chromatin_footprint_gate: passed=False, score=0.0\n  x exhaustion_comparable_gate: passed=False, score=0.0\n  + exhaustion_nondeterminant_gate: passed=True, score=1.0\n  + exhaustion_separable_program_gate: passed=True, score=1.0\n  x exhaustion_counterexample_gate: passed=False, score=0.0\n  x list_match: list_match (answer_field='exhaustion_relations'): FAIL  (score=0.0)\n  x hard_fail 'cf_signature_gut_contaminant': TRIGGERED\n  + hard_fail 'cf_inconsistent_present_without_activator': not triggered\n  + hard_fail 'cf_inconsistent_no_block_but_absent': not triggered\n  + hard_fail 'cf_inconsistent_late_block_without_poised_locus': not triggered\n  + hard_fail 'cf_inconsistent_early_block_with_brake_released': not triggered",
      "metrics": {
        "pass_rule": "all",
        "scoring_count": 17,
        "scoring_passed": 6,
        "scoring_total_score": 6,
        "hard_fail_triggered": [
          "cf_signature_gut_contaminant"
        ]
      },
      "fieldScores": {
        "list_match": 0,
        "sig_includes_itgae": 1,
        "declared_a_known_confounder": 0,
        "locus_differential_gate": 1,
        "formation_limited_by_gate": 1,
        "locus_competent_in_naive_gate": 1,
        "activator_chromatin_footprint_gate": 0,
        "exhaustion_comparable_gate": 0,
        "exhaustion_nondeterminant_gate": 1,
        "exhaustion_separable_program_gate": 1,
        "exhaustion_counterexample_gate": 0
      },
      "graderChecks": [
        {
          "name": "program_map",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "sig_includes_itgae",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "declared_a_known_confounder",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "generic_signature_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "cd103_program_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "per_model_regulatory_state",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "regulator_tuples",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "locus_differential_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "formation_limited_by_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "locus_competent_in_naive_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "ready_silent_fraction_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "activator_chromatin_footprint_gate",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "exhaustion_comparable_gate",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "exhaustion_nondeterminant_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "exhaustion_separable_program_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "exhaustion_counterexample_gate",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "exhaustion_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "cf_signature_gut_contaminant",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "cf_inconsistent_present_without_activator",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_no_block_but_absent",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_late_block_without_poised_locus",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_early_block_with_brake_released",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "activator_chromatin_footprint": "consistent_in_formers",
        "cd103_program_relations": [
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": "~"
          },
          {
            "pair": "B16_GP_vs_KPC_GP",
            "relation": "<"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": "<"
          },
          {
            "pair": "B_ALL_GP_vs_KPC_GP",
            "relation": "<"
          },
          {
            "pair": "B_ALL_GP_vs_mWnt_GP",
            "relation": "<"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": ">"
          }
        ],
        "core_residency_signature": [
          "Itgae",
          "Itga1",
          "Rgs1",
          "Nt5e",
          "Pmepa1",
          "Smad7"
        ],
        "excluded_confounders": [
          "Cd69",
          "Cxcr6",
          "Tox",
          "Pdcd1",
          "Lag3",
          "Havcr2",
          "Tigit",
          "Entpd1",
          "Ctla4",
          "Gzmb",
          "Ifng",
          "Prf1",
          "Nkg7",
          "Xcl1",
          "Ccl5"
        ],
        "exhaustion_comparable_across_split": "yes",
        "exhaustion_counterexample_model": "B16_GP",
        "exhaustion_explains_split": "no",
        "exhaustion_relations": [
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": "~"
          },
          {
            "pair": "B16_GP_vs_KPC_GP",
            "relation": "~"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "B_ALL_GP_vs_KPC_GP",
            "relation": "~"
          },
          {
            "pair": "B_ALL_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": ">"
          }
        ],
        "exhaustion_separable_program": "yes",
        "formation_limited_by": "absent_induction",
        "generic_signature_relations": [
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "B16_GP_vs_KPC_GP",
            "relation": "<"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": "<"
          },
          {
            "pair": "B_ALL_GP_vs_KPC_GP",
            "relation": "<"
          },
          {
            "pair": "B_ALL_GP_vs_mWnt_GP",
            "relation": "<"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": "~"
          }
        ],
        "locus_competent_in_naive": "yes",
        "locus_differential": "uniform_across_models",
        "per_model_regulatory_state": [
          {
            "activator_state": "absent",
            "blocking_step": "late_activation",
            "brake_state": "released",
            "locus_state": "poised_unfired",
            "model": "B16_GP",
            "program": "absent"
          },
          {
            "activator_state": "absent",
            "blocking_step": "both_steps",
            "brake_state": "engaged",
            "locus_state": "poised_unfired",
            "model": "B_ALL_GP",
            "program": "absent"
          },
          {
            "activator_state": "present",
            "blocking_step": "none",
            "brake_state": "released",
            "locus_state": "active",
            "model": "KPC_GP",
            "program": "present"
          },
          {
            "activator_state": "present",
            "blocking_step": "early_release",
            "brake_state": "partial",
            "locus_state": "active",
            "model": "mWnt_GP",
            "program": "partial"
          }
        ],
        "program_map": [
          {
            "call": "absent",
            "model": "B16_GP"
          },
          {
            "call": "absent",
            "model": "B_ALL_GP"
          },
          {
            "call": "present",
            "model": "KPC_GP"
          },
          {
            "call": "partial",
            "model": "mWnt_GP"
          }
        ],
        "ready_silent_fraction_relations": [
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": "~"
          },
          {
            "pair": "B16_GP_vs_KPC_GP",
            "relation": ">"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "B_ALL_GP_vs_KPC_GP",
            "relation": ">"
          },
          {
            "pair": "B_ALL_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": "<"
          }
        ],
        "regulator_tuples": [
          {
            "evidence_basis": "own_abundance",
            "expr_vs_activity": "concordant",
            "gating_step": "early_release",
            "regulator": "Klf2",
            "role": "repressor"
          },
          {
            "evidence_basis": "own_abundance",
            "expr_vs_activity": "concordant",
            "gating_step": "late_activation",
            "regulator": "Ahr",
            "role": "activator"
          },
          {
            "evidence_basis": "target_program_behavior",
            "expr_vs_activity": "discordant",
            "gating_step": "late_activation",
            "regulator": "Runx3",
            "role": "activator"
          },
          {
            "evidence_basis": "own_abundance",
            "expr_vs_activity": "discordant",
            "gating_step": "none",
            "regulator": "Tox",
            "role": "none"
          }
        ]
      },
      "nTurns": 36,
      "checkpointSignals": {
        "clustering": false,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": true,
        "nerveJointAnchor": false,
        "multiomeAssembly": true,
        "depthNormalizedAtac": true,
        "regulonInference": true,
        "cd8SubsetIntegration": true,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": false,
        "orthologProjection": false,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": false,
        "mousePseudobulkDesign": true,
        "cancerEpithelialDenominator": false,
        "geneLevelDe": true,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "trajectoryFile": "scbench-long/trajectories/green_2_tme_context_dependence__gpt-5.5__pi__r1.json",
      "runId": "gpt-5.5__pi__green_2_tme_context_dependence__r1",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-gpt-5-5-pi",
      "modelName": "gpt-5.5",
      "harness": "pi",
      "task": "green_2_tme_context_dependence",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 2,
      "passed": false,
      "score": 7,
      "durationS": 1882.18,
      "cost": 3.0506290000000003,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "program_map",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "sig_includes_itgae",
            "role": "gate",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Itgae"
            ]
          },
          {
            "name": "declared_a_known_confounder",
            "role": "gate",
            "field": "$.excluded_confounders[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "cf_signature_gut_contaminant",
            "role": "hard_fail",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "generic_signature_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd103_program_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "<"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "per_model_regulatory_state",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "poised_unfired"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "late_activation"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "closing_committed"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_tuples",
            "kind": "list",
            "matchKey": "regulator",
            "k": 1,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "role": {
                    "op": "equals",
                    "values": [
                      "repressor"
                    ]
                  },
                  "gating_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  },
                  "evidence_basis": {
                    "op": "equals",
                    "values": [
                      "target_program_behavior"
                    ]
                  },
                  "expr_vs_activity": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "locus_differential_gate",
            "role": "gate",
            "field": "locus_differential",
            "kind": "value",
            "expected": "uniform_across_models"
          },
          {
            "name": "formation_limited_by_gate",
            "role": "gate",
            "field": "formation_limited_by",
            "kind": "value",
            "expected": "absent_induction"
          },
          {
            "name": "locus_competent_in_naive_gate",
            "role": "gate",
            "field": "locus_competent_in_naive",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "ready_silent_fraction_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 3,
            "expected": [
              {
                "key": "B16_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_KPC_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "activator_chromatin_footprint_gate",
            "role": "gate",
            "field": "activator_chromatin_footprint",
            "kind": "value",
            "expected": "inconsistent_across_formers"
          },
          {
            "name": "exhaustion_comparable_gate",
            "role": "gate",
            "field": "exhaustion_comparable_across_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_nondeterminant_gate",
            "role": "gate",
            "field": "exhaustion_explains_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_separable_program_gate",
            "role": "gate",
            "field": "exhaustion_separable_program",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "exhaustion_counterexample_gate",
            "role": "gate",
            "field": "exhaustion_counterexample_model",
            "kind": "value",
            "expected": "KPC_GP"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "exhaustion_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "cf_inconsistent_present_without_activator",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "program",
            "entity": "present",
            "valueField": "activator_state",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_no_block_but_absent",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "none",
            "valueField": "program",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_late_block_without_poised_locus",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "late_activation",
            "valueField": "locus_state",
            "expected": [
              "closing_committed",
              "active"
            ]
          },
          {
            "name": "cf_inconsistent_early_block_with_brake_released",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "early_release",
            "valueField": "brake_state",
            "expected": [
              "released"
            ]
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: FAIL\n  x list_match: list_match (answer_field='program_map'): FAIL  (score=0.0)\n  + sig_includes_itgae: passed=True, score=1.0\n  x declared_a_known_confounder: passed=False, score=0.0\n  x list_match: list_match (answer_field='generic_signature_relations'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='cd103_program_relations'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='per_model_regulatory_state'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='regulator_tuples'): FAIL  (score=0.0)\n  + locus_differential_gate: passed=True, score=1.0\n  + formation_limited_by_gate: passed=True, score=1.0\n  + locus_competent_in_naive_gate: passed=True, score=1.0\n  x list_match: list_match (answer_field='ready_silent_fraction_relations'): FAIL  (score=0.0)\n  x activator_chromatin_footprint_gate: passed=False, score=0.0\n  x exhaustion_comparable_gate: passed=False, score=0.0\n  + exhaustion_nondeterminant_gate: passed=True, score=1.0\n  + exhaustion_separable_program_gate: passed=True, score=1.0\n  + exhaustion_counterexample_gate: passed=True, score=1.0\n  x list_match: list_match (answer_field='exhaustion_relations'): FAIL  (score=0.0)\n  x hard_fail 'cf_signature_gut_contaminant': TRIGGERED\n  + hard_fail 'cf_inconsistent_present_without_activator': not triggered\n  + hard_fail 'cf_inconsistent_no_block_but_absent': not triggered\n  + hard_fail 'cf_inconsistent_late_block_without_poised_locus': not triggered\n  + hard_fail 'cf_inconsistent_early_block_with_brake_released': not triggered",
      "metrics": {
        "pass_rule": "all",
        "scoring_count": 17,
        "scoring_passed": 7,
        "scoring_total_score": 7,
        "hard_fail_triggered": [
          "cf_signature_gut_contaminant"
        ]
      },
      "fieldScores": {
        "list_match": 0,
        "sig_includes_itgae": 1,
        "declared_a_known_confounder": 0,
        "locus_differential_gate": 1,
        "formation_limited_by_gate": 1,
        "locus_competent_in_naive_gate": 1,
        "activator_chromatin_footprint_gate": 0,
        "exhaustion_comparable_gate": 0,
        "exhaustion_nondeterminant_gate": 1,
        "exhaustion_separable_program_gate": 1,
        "exhaustion_counterexample_gate": 1
      },
      "graderChecks": [
        {
          "name": "program_map",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "sig_includes_itgae",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "declared_a_known_confounder",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "generic_signature_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "cd103_program_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "per_model_regulatory_state",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "regulator_tuples",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "locus_differential_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "formation_limited_by_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "locus_competent_in_naive_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "ready_silent_fraction_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "activator_chromatin_footprint_gate",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "exhaustion_comparable_gate",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "exhaustion_nondeterminant_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "exhaustion_separable_program_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "exhaustion_counterexample_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "exhaustion_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "cf_signature_gut_contaminant",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "cf_inconsistent_present_without_activator",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_no_block_but_absent",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_late_block_without_poised_locus",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_early_block_with_brake_released",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "activator_chromatin_footprint": "consistent_in_formers",
        "cd103_program_relations": [
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": "~"
          },
          {
            "pair": "B16_GP_vs_KPC_GP",
            "relation": "<"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": "<"
          },
          {
            "pair": "B_ALL_GP_vs_KPC_GP",
            "relation": "<"
          },
          {
            "pair": "B_ALL_GP_vs_mWnt_GP",
            "relation": "<"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": ">"
          }
        ],
        "core_residency_signature": [
          "Itgae",
          "Itga1",
          "Cdh1",
          "Rgs1",
          "Cxcr6",
          "Cd69",
          "Xcl1"
        ],
        "excluded_confounders": [
          "Tox",
          "Pdcd1",
          "Lag3",
          "Havcr2",
          "Tigit",
          "Entpd1",
          "Ctla4",
          "Gzmb",
          "Gzma",
          "Prf1",
          "Nkg7",
          "Ifng",
          "Ccl5",
          "Mki67",
          "Top2a"
        ],
        "exhaustion_comparable_across_split": "yes",
        "exhaustion_counterexample_model": "KPC_GP",
        "exhaustion_explains_split": "no",
        "exhaustion_relations": [
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": "~"
          },
          {
            "pair": "B16_GP_vs_KPC_GP",
            "relation": "~"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "B_ALL_GP_vs_KPC_GP",
            "relation": "~"
          },
          {
            "pair": "B_ALL_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": ">"
          }
        ],
        "exhaustion_separable_program": "yes",
        "formation_limited_by": "absent_induction",
        "generic_signature_relations": [
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "B16_GP_vs_KPC_GP",
            "relation": "<"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": "<"
          },
          {
            "pair": "B_ALL_GP_vs_KPC_GP",
            "relation": "<"
          },
          {
            "pair": "B_ALL_GP_vs_mWnt_GP",
            "relation": "<"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": "~"
          }
        ],
        "locus_competent_in_naive": "yes",
        "locus_differential": "uniform_across_models",
        "per_model_regulatory_state": [
          {
            "activator_state": "absent",
            "blocking_step": "late_activation",
            "brake_state": "released",
            "locus_state": "poised_unfired",
            "model": "B16_GP",
            "program": "absent"
          },
          {
            "activator_state": "absent",
            "blocking_step": "both_steps",
            "brake_state": "engaged",
            "locus_state": "poised_unfired",
            "model": "B_ALL_GP",
            "program": "absent"
          },
          {
            "activator_state": "present",
            "blocking_step": "none",
            "brake_state": "released",
            "locus_state": "active",
            "model": "KPC_GP",
            "program": "present"
          },
          {
            "activator_state": "present",
            "blocking_step": "early_release",
            "brake_state": "partial",
            "locus_state": "active",
            "model": "mWnt_GP",
            "program": "partial"
          }
        ],
        "program_map": [
          {
            "call": "absent",
            "model": "B16_GP"
          },
          {
            "call": "absent",
            "model": "B_ALL_GP"
          },
          {
            "call": "present",
            "model": "KPC_GP"
          },
          {
            "call": "partial",
            "model": "mWnt_GP"
          }
        ],
        "ready_silent_fraction_relations": [
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": "<"
          },
          {
            "pair": "B16_GP_vs_KPC_GP",
            "relation": ">"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": "<"
          },
          {
            "pair": "B_ALL_GP_vs_KPC_GP",
            "relation": ">"
          },
          {
            "pair": "B_ALL_GP_vs_mWnt_GP",
            "relation": "~"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": "<"
          }
        ],
        "regulator_tuples": [
          {
            "evidence_basis": "own_abundance",
            "expr_vs_activity": "concordant",
            "gating_step": "early_release",
            "regulator": "Klf2",
            "role": "repressor"
          },
          {
            "evidence_basis": "own_abundance",
            "expr_vs_activity": "concordant",
            "gating_step": "early_release",
            "regulator": "S1pr1",
            "role": "repressor"
          },
          {
            "evidence_basis": "own_abundance",
            "expr_vs_activity": "concordant",
            "gating_step": "late_activation",
            "regulator": "Smad3",
            "role": "activator"
          },
          {
            "evidence_basis": "own_abundance",
            "expr_vs_activity": "concordant",
            "gating_step": "late_activation",
            "regulator": "Tgfbr2",
            "role": "activator"
          },
          {
            "evidence_basis": "target_program_behavior",
            "expr_vs_activity": "discordant",
            "gating_step": "late_activation",
            "regulator": "Runx3",
            "role": "activator"
          }
        ]
      },
      "nTurns": 43,
      "checkpointSignals": {
        "clustering": true,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": false,
        "nerveJointAnchor": false,
        "multiomeAssembly": true,
        "depthNormalizedAtac": true,
        "regulonInference": true,
        "cd8SubsetIntegration": true,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": false,
        "orthologProjection": false,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": true,
        "enrichmentFamilyRanking": true,
        "mousePseudobulkDesign": true,
        "cancerEpithelialDenominator": false,
        "geneLevelDe": true,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "trajectoryFile": "scbench-long/trajectories/green_2_tme_context_dependence__gpt-5.5__pi__r2.json",
      "runId": "gpt-5.5__pi__green_2_tme_context_dependence__r2",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-gpt-5-5-pi",
      "modelName": "gpt-5.5",
      "harness": "pi",
      "task": "green_2_tme_context_dependence",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 3,
      "passed": false,
      "score": 6,
      "durationS": 938.4,
      "cost": 2.5480719999999994,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "program_map",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "sig_includes_itgae",
            "role": "gate",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Itgae"
            ]
          },
          {
            "name": "declared_a_known_confounder",
            "role": "gate",
            "field": "$.excluded_confounders[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "cf_signature_gut_contaminant",
            "role": "hard_fail",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "generic_signature_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd103_program_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "<"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "per_model_regulatory_state",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "poised_unfired"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "late_activation"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "closing_committed"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_tuples",
            "kind": "list",
            "matchKey": "regulator",
            "k": 1,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "role": {
                    "op": "equals",
                    "values": [
                      "repressor"
                    ]
                  },
                  "gating_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  },
                  "evidence_basis": {
                    "op": "equals",
                    "values": [
                      "target_program_behavior"
                    ]
                  },
                  "expr_vs_activity": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "locus_differential_gate",
            "role": "gate",
            "field": "locus_differential",
            "kind": "value",
            "expected": "uniform_across_models"
          },
          {
            "name": "formation_limited_by_gate",
            "role": "gate",
            "field": "formation_limited_by",
            "kind": "value",
            "expected": "absent_induction"
          },
          {
            "name": "locus_competent_in_naive_gate",
            "role": "gate",
            "field": "locus_competent_in_naive",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "ready_silent_fraction_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 3,
            "expected": [
              {
                "key": "B16_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_KPC_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "activator_chromatin_footprint_gate",
            "role": "gate",
            "field": "activator_chromatin_footprint",
            "kind": "value",
            "expected": "inconsistent_across_formers"
          },
          {
            "name": "exhaustion_comparable_gate",
            "role": "gate",
            "field": "exhaustion_comparable_across_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_nondeterminant_gate",
            "role": "gate",
            "field": "exhaustion_explains_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_separable_program_gate",
            "role": "gate",
            "field": "exhaustion_separable_program",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "exhaustion_counterexample_gate",
            "role": "gate",
            "field": "exhaustion_counterexample_model",
            "kind": "value",
            "expected": "KPC_GP"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "exhaustion_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "cf_inconsistent_present_without_activator",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "program",
            "entity": "present",
            "valueField": "activator_state",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_no_block_but_absent",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "none",
            "valueField": "program",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_late_block_without_poised_locus",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "late_activation",
            "valueField": "locus_state",
            "expected": [
              "closing_committed",
              "active"
            ]
          },
          {
            "name": "cf_inconsistent_early_block_with_brake_released",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "early_release",
            "valueField": "brake_state",
            "expected": [
              "released"
            ]
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: FAIL\n  x list_match: list_match (answer_field='program_map'): FAIL  (score=0.0)\n  + sig_includes_itgae: passed=True, score=1.0\n  x declared_a_known_confounder: passed=False, score=0.0\n  + list_match: list_match (answer_field='generic_signature_relations'): PASS  (score=0.0)\n  x list_match: list_match (answer_field='cd103_program_relations'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='per_model_regulatory_state'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='regulator_tuples'): FAIL  (score=0.0)\n  + locus_differential_gate: passed=True, score=1.0\n  + formation_limited_by_gate: passed=True, score=1.0\n  + locus_competent_in_naive_gate: passed=True, score=1.0\n  x list_match: list_match (answer_field='ready_silent_fraction_relations'): FAIL  (score=0.0)\n  x activator_chromatin_footprint_gate: passed=False, score=0.0\n  x exhaustion_comparable_gate: passed=False, score=0.0\n  + exhaustion_nondeterminant_gate: passed=True, score=1.0\n  + exhaustion_separable_program_gate: passed=True, score=1.0\n  x exhaustion_counterexample_gate: passed=False, score=0.0\n  x list_match: list_match (answer_field='exhaustion_relations'): FAIL  (score=0.0)\n  x hard_fail 'cf_signature_gut_contaminant': TRIGGERED\n  + hard_fail 'cf_inconsistent_present_without_activator': not triggered\n  + hard_fail 'cf_inconsistent_no_block_but_absent': not triggered\n  + hard_fail 'cf_inconsistent_late_block_without_poised_locus': not triggered\n  + hard_fail 'cf_inconsistent_early_block_with_brake_released': not triggered",
      "metrics": {
        "pass_rule": "all",
        "scoring_count": 17,
        "scoring_passed": 7,
        "scoring_total_score": 6,
        "hard_fail_triggered": [
          "cf_signature_gut_contaminant"
        ]
      },
      "fieldScores": {
        "list_match": 0,
        "sig_includes_itgae": 1,
        "declared_a_known_confounder": 0,
        "locus_differential_gate": 1,
        "formation_limited_by_gate": 1,
        "locus_competent_in_naive_gate": 1,
        "activator_chromatin_footprint_gate": 0,
        "exhaustion_comparable_gate": 0,
        "exhaustion_nondeterminant_gate": 1,
        "exhaustion_separable_program_gate": 1,
        "exhaustion_counterexample_gate": 0
      },
      "graderChecks": [
        {
          "name": "program_map",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "sig_includes_itgae",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "declared_a_known_confounder",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "generic_signature_relations",
          "kind": "list_match",
          "passed": true,
          "score": 0
        },
        {
          "name": "cd103_program_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "per_model_regulatory_state",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "regulator_tuples",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "locus_differential_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "formation_limited_by_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "locus_competent_in_naive_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "ready_silent_fraction_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "activator_chromatin_footprint_gate",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "exhaustion_comparable_gate",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "exhaustion_nondeterminant_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "exhaustion_separable_program_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "exhaustion_counterexample_gate",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "exhaustion_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "cf_signature_gut_contaminant",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "cf_inconsistent_present_without_activator",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_no_block_but_absent",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_late_block_without_poised_locus",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_early_block_with_brake_released",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "activator_chromatin_footprint": "consistent_in_formers",
        "cd103_program_relations": [
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_B16_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "mWnt_GP_vs_B16_GP",
            "relation": ">"
          },
          {
            "pair": "mWnt_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": "~"
          }
        ],
        "core_residency_signature": [
          "Itgae",
          "Itga1",
          "Cd69",
          "Cxcr6",
          "Rgs1",
          "Bhlhe40",
          "Xcl1",
          "Ahr",
          "Nt5e"
        ],
        "excluded_confounders": [
          "Gzmb",
          "Ifng",
          "Prf1",
          "Nkg7",
          "Ccl5",
          "Gzma",
          "Tox",
          "Pdcd1",
          "Lag3",
          "Havcr2",
          "Tigit",
          "Ctla4",
          "Entpd1",
          "Mki67",
          "Top2a"
        ],
        "exhaustion_comparable_across_split": "yes",
        "exhaustion_counterexample_model": "B16_GP",
        "exhaustion_explains_split": "no",
        "exhaustion_relations": [
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_B16_GP",
            "relation": "~"
          },
          {
            "pair": "KPC_GP_vs_B_ALL_GP",
            "relation": "~"
          },
          {
            "pair": "mWnt_GP_vs_B16_GP",
            "relation": "<"
          },
          {
            "pair": "mWnt_GP_vs_B_ALL_GP",
            "relation": "<"
          },
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": "~"
          }
        ],
        "exhaustion_separable_program": "yes",
        "formation_limited_by": "absent_induction",
        "generic_signature_relations": [
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": "~"
          },
          {
            "pair": "KPC_GP_vs_B16_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "mWnt_GP_vs_B16_GP",
            "relation": ">"
          },
          {
            "pair": "mWnt_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": ">"
          }
        ],
        "locus_competent_in_naive": "yes",
        "locus_differential": "uniform_across_models",
        "per_model_regulatory_state": [
          {
            "activator_state": "present",
            "blocking_step": "none",
            "brake_state": "released",
            "locus_state": "active",
            "model": "KPC_GP",
            "program": "present"
          },
          {
            "activator_state": "present",
            "blocking_step": "early_release",
            "brake_state": "partial",
            "locus_state": "poised_unfired",
            "model": "mWnt_GP",
            "program": "partial"
          },
          {
            "activator_state": "absent",
            "blocking_step": "late_activation",
            "brake_state": "released",
            "locus_state": "poised_unfired",
            "model": "B16_GP",
            "program": "absent"
          },
          {
            "activator_state": "absent",
            "blocking_step": "both_steps",
            "brake_state": "partial",
            "locus_state": "poised_unfired",
            "model": "B_ALL_GP",
            "program": "absent"
          }
        ],
        "program_map": [
          {
            "call": "present",
            "model": "KPC_GP"
          },
          {
            "call": "partial",
            "model": "mWnt_GP"
          },
          {
            "call": "absent",
            "model": "B16_GP"
          },
          {
            "call": "absent",
            "model": "B_ALL_GP"
          }
        ],
        "ready_silent_fraction_relations": [
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": "<"
          },
          {
            "pair": "KPC_GP_vs_B16_GP",
            "relation": "<"
          },
          {
            "pair": "KPC_GP_vs_B_ALL_GP",
            "relation": "<"
          },
          {
            "pair": "mWnt_GP_vs_B16_GP",
            "relation": ">"
          },
          {
            "pair": "mWnt_GP_vs_B_ALL_GP",
            "relation": "~"
          },
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": "<"
          }
        ],
        "regulator_tuples": [
          {
            "evidence_basis": "own_abundance",
            "expr_vs_activity": "concordant",
            "gating_step": "early_release",
            "regulator": "Klf2",
            "role": "repressor"
          },
          {
            "evidence_basis": "own_abundance",
            "expr_vs_activity": "concordant",
            "gating_step": "early_release",
            "regulator": "Tcf7",
            "role": "repressor"
          },
          {
            "evidence_basis": "own_abundance",
            "expr_vs_activity": "concordant",
            "gating_step": "late_activation",
            "regulator": "Ahr",
            "role": "activator"
          },
          {
            "evidence_basis": "target_program_behavior",
            "expr_vs_activity": "concordant",
            "gating_step": "late_activation",
            "regulator": "Smad3",
            "role": "activator"
          },
          {
            "evidence_basis": "target_program_behavior",
            "expr_vs_activity": "discordant",
            "gating_step": "late_activation",
            "regulator": "Runx3",
            "role": "activator"
          }
        ]
      },
      "nTurns": 35,
      "checkpointSignals": {
        "clustering": false,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": true,
        "nerveJointAnchor": false,
        "multiomeAssembly": true,
        "depthNormalizedAtac": true,
        "regulonInference": true,
        "cd8SubsetIntegration": true,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": false,
        "orthologProjection": false,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": false,
        "mousePseudobulkDesign": false,
        "cancerEpithelialDenominator": true,
        "geneLevelDe": false,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "trajectoryFile": "scbench-long/trajectories/green_2_tme_context_dependence__gpt-5.5__pi__r3.json",
      "runId": "gpt-5.5__pi__green_2_tme_context_dependence__r3",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-grok-4-20-beta-reasoning-pi",
      "modelName": "grok-4.20-beta-0309-reasoning",
      "harness": "pi",
      "task": "green_2_tme_context_dependence",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 1,
      "passed": false,
      "score": 6,
      "durationS": 154.05,
      "cost": 0.026422300000000003,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "program_map",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "sig_includes_itgae",
            "role": "gate",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Itgae"
            ]
          },
          {
            "name": "declared_a_known_confounder",
            "role": "gate",
            "field": "$.excluded_confounders[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "cf_signature_gut_contaminant",
            "role": "hard_fail",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "generic_signature_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd103_program_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "<"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "per_model_regulatory_state",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "poised_unfired"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "late_activation"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "closing_committed"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_tuples",
            "kind": "list",
            "matchKey": "regulator",
            "k": 1,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "role": {
                    "op": "equals",
                    "values": [
                      "repressor"
                    ]
                  },
                  "gating_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  },
                  "evidence_basis": {
                    "op": "equals",
                    "values": [
                      "target_program_behavior"
                    ]
                  },
                  "expr_vs_activity": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "locus_differential_gate",
            "role": "gate",
            "field": "locus_differential",
            "kind": "value",
            "expected": "uniform_across_models"
          },
          {
            "name": "formation_limited_by_gate",
            "role": "gate",
            "field": "formation_limited_by",
            "kind": "value",
            "expected": "absent_induction"
          },
          {
            "name": "locus_competent_in_naive_gate",
            "role": "gate",
            "field": "locus_competent_in_naive",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "ready_silent_fraction_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 3,
            "expected": [
              {
                "key": "B16_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_KPC_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "activator_chromatin_footprint_gate",
            "role": "gate",
            "field": "activator_chromatin_footprint",
            "kind": "value",
            "expected": "inconsistent_across_formers"
          },
          {
            "name": "exhaustion_comparable_gate",
            "role": "gate",
            "field": "exhaustion_comparable_across_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_nondeterminant_gate",
            "role": "gate",
            "field": "exhaustion_explains_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_separable_program_gate",
            "role": "gate",
            "field": "exhaustion_separable_program",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "exhaustion_counterexample_gate",
            "role": "gate",
            "field": "exhaustion_counterexample_model",
            "kind": "value",
            "expected": "KPC_GP"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "exhaustion_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "cf_inconsistent_present_without_activator",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "program",
            "entity": "present",
            "valueField": "activator_state",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_no_block_but_absent",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "none",
            "valueField": "program",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_late_block_without_poised_locus",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "late_activation",
            "valueField": "locus_state",
            "expected": [
              "closing_committed",
              "active"
            ]
          },
          {
            "name": "cf_inconsistent_early_block_with_brake_released",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "early_release",
            "valueField": "brake_state",
            "expected": [
              "released"
            ]
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: FAIL\n  x list_match: list_match (answer_field='program_map'): FAIL  (score=0.0)\n  + sig_includes_itgae: passed=True, score=1.0\n  x declared_a_known_confounder: passed=False, score=0.0\n  x list_match: list_match (answer_field='generic_signature_relations'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='cd103_program_relations'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='per_model_regulatory_state'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='regulator_tuples'): FAIL  (score=0.0)\n  x locus_differential_gate: passed=False, score=0.0\n  + formation_limited_by_gate: passed=True, score=1.0\n  + locus_competent_in_naive_gate: passed=True, score=1.0\n  x list_match: list_match (answer_field='ready_silent_fraction_relations'): FAIL  (score=0.0)\n  x activator_chromatin_footprint_gate: passed=False, score=0.0\n  + exhaustion_comparable_gate: passed=True, score=1.0\n  + exhaustion_nondeterminant_gate: passed=True, score=1.0\n  + exhaustion_separable_program_gate: passed=True, score=1.0\n  x exhaustion_counterexample_gate: passed=False, score=0.0\n  x list_match: list_match (answer_field='exhaustion_relations'): FAIL  (score=0.0)\n  + hard_fail 'cf_signature_gut_contaminant': not triggered\n  + hard_fail 'cf_inconsistent_present_without_activator': not triggered\n  + hard_fail 'cf_inconsistent_no_block_but_absent': not triggered\n  + hard_fail 'cf_inconsistent_late_block_without_poised_locus': not triggered\n  + hard_fail 'cf_inconsistent_early_block_with_brake_released': not triggered",
      "metrics": {
        "pass_rule": "all",
        "scoring_count": 17,
        "scoring_passed": 6,
        "scoring_total_score": 6,
        "hard_fail_triggered": []
      },
      "fieldScores": {
        "list_match": 0,
        "sig_includes_itgae": 1,
        "declared_a_known_confounder": 0,
        "locus_differential_gate": 0,
        "formation_limited_by_gate": 1,
        "locus_competent_in_naive_gate": 1,
        "activator_chromatin_footprint_gate": 0,
        "exhaustion_comparable_gate": 1,
        "exhaustion_nondeterminant_gate": 1,
        "exhaustion_separable_program_gate": 1,
        "exhaustion_counterexample_gate": 0
      },
      "graderChecks": [
        {
          "name": "program_map",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "sig_includes_itgae",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "declared_a_known_confounder",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "generic_signature_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "cd103_program_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "per_model_regulatory_state",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "regulator_tuples",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "locus_differential_gate",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "formation_limited_by_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "locus_competent_in_naive_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "ready_silent_fraction_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "activator_chromatin_footprint_gate",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "exhaustion_comparable_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "exhaustion_nondeterminant_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "exhaustion_separable_program_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "exhaustion_counterexample_gate",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "exhaustion_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "cf_signature_gut_contaminant",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_present_without_activator",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_no_block_but_absent",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_late_block_without_poised_locus",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_early_block_with_brake_released",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "activator_chromatin_footprint": "consistent_in_formers",
        "cd103_program_relations": [
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": "~"
          },
          {
            "pair": "KPC_GP_vs_B16_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "mWnt_GP_vs_B16_GP",
            "relation": ">"
          },
          {
            "pair": "mWnt_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": "~"
          }
        ],
        "core_residency_signature": [
          "Itgae",
          "Cd69",
          "Cxcr6"
        ],
        "excluded_confounders": [
          "Tox",
          "Pdcd1"
        ],
        "exhaustion_comparable_across_split": "no",
        "exhaustion_counterexample_model": "B16_GP",
        "exhaustion_explains_split": "no",
        "exhaustion_relations": [
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": "~"
          },
          {
            "pair": "KPC_GP_vs_B16_GP",
            "relation": "<"
          },
          {
            "pair": "KPC_GP_vs_B_ALL_GP",
            "relation": "<"
          },
          {
            "pair": "mWnt_GP_vs_B16_GP",
            "relation": "<"
          },
          {
            "pair": "mWnt_GP_vs_B_ALL_GP",
            "relation": "<"
          },
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": "~"
          }
        ],
        "exhaustion_separable_program": "yes",
        "formation_limited_by": "absent_induction",
        "generic_signature_relations": [
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": "~"
          },
          {
            "pair": "KPC_GP_vs_B16_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "mWnt_GP_vs_B16_GP",
            "relation": ">"
          },
          {
            "pair": "mWnt_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": "~"
          }
        ],
        "locus_competent_in_naive": "yes",
        "locus_differential": "model_specific_element",
        "per_model_regulatory_state": [
          {
            "activator_state": "present",
            "blocking_step": "none",
            "brake_state": "released",
            "locus_state": "active",
            "model": "KPC_GP",
            "program": "present"
          },
          {
            "activator_state": "present",
            "blocking_step": "late_activation",
            "brake_state": "partial",
            "locus_state": "poised_unfired",
            "model": "mWnt_GP",
            "program": "partial"
          },
          {
            "activator_state": "absent",
            "blocking_step": "both_steps",
            "brake_state": "engaged",
            "locus_state": "repressed",
            "model": "B16_GP",
            "program": "absent"
          },
          {
            "activator_state": "absent",
            "blocking_step": "both_steps",
            "brake_state": "engaged",
            "locus_state": "repressed",
            "model": "B_ALL_GP",
            "program": "absent"
          }
        ],
        "program_map": [
          {
            "call": "present",
            "model": "KPC_GP"
          },
          {
            "call": "partial",
            "model": "mWnt_GP"
          },
          {
            "call": "absent",
            "model": "B16_GP"
          },
          {
            "call": "absent",
            "model": "B_ALL_GP"
          }
        ],
        "ready_silent_fraction_relations": [
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": "~"
          },
          {
            "pair": "KPC_GP_vs_B16_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "mWnt_GP_vs_B16_GP",
            "relation": ">"
          },
          {
            "pair": "mWnt_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": "~"
          }
        ],
        "regulator_tuples": [
          {
            "evidence_basis": "target_program_behavior",
            "expr_vs_activity": "concordant",
            "gating_step": "late_activation",
            "regulator": "Runx3",
            "role": "activator"
          },
          {
            "evidence_basis": "own_abundance",
            "expr_vs_activity": "concordant",
            "gating_step": "early_release",
            "regulator": "Nr4a1",
            "role": "repressor"
          },
          {
            "evidence_basis": "target_program_behavior",
            "expr_vs_activity": "discordant",
            "gating_step": "indeterminate",
            "regulator": "Tox",
            "role": "dual"
          }
        ]
      },
      "nTurns": 10,
      "checkpointSignals": {
        "clustering": false,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": false,
        "nerveJointAnchor": false,
        "multiomeAssembly": true,
        "depthNormalizedAtac": false,
        "regulonInference": true,
        "cd8SubsetIntegration": true,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": false,
        "orthologProjection": false,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": false,
        "mousePseudobulkDesign": false,
        "cancerEpithelialDenominator": false,
        "geneLevelDe": false,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "trajectoryFile": "scbench-long/trajectories/green_2_tme_context_dependence__grok-4.20-beta-0309-reasoning__pi__r1.json",
      "runId": "grok-4.20-beta-0309-reasoning__pi__green_2_tme_context_dependence__r1",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-grok-4-20-beta-reasoning-pi",
      "modelName": "grok-4.20-beta-0309-reasoning",
      "harness": "pi",
      "task": "green_2_tme_context_dependence",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 2,
      "passed": false,
      "score": 5,
      "durationS": 301.33,
      "cost": 0.0506083,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "program_map",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "sig_includes_itgae",
            "role": "gate",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Itgae"
            ]
          },
          {
            "name": "declared_a_known_confounder",
            "role": "gate",
            "field": "$.excluded_confounders[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "cf_signature_gut_contaminant",
            "role": "hard_fail",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "generic_signature_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd103_program_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "<"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "per_model_regulatory_state",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "poised_unfired"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "late_activation"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "closing_committed"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_tuples",
            "kind": "list",
            "matchKey": "regulator",
            "k": 1,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "role": {
                    "op": "equals",
                    "values": [
                      "repressor"
                    ]
                  },
                  "gating_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  },
                  "evidence_basis": {
                    "op": "equals",
                    "values": [
                      "target_program_behavior"
                    ]
                  },
                  "expr_vs_activity": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "locus_differential_gate",
            "role": "gate",
            "field": "locus_differential",
            "kind": "value",
            "expected": "uniform_across_models"
          },
          {
            "name": "formation_limited_by_gate",
            "role": "gate",
            "field": "formation_limited_by",
            "kind": "value",
            "expected": "absent_induction"
          },
          {
            "name": "locus_competent_in_naive_gate",
            "role": "gate",
            "field": "locus_competent_in_naive",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "ready_silent_fraction_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 3,
            "expected": [
              {
                "key": "B16_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_KPC_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "activator_chromatin_footprint_gate",
            "role": "gate",
            "field": "activator_chromatin_footprint",
            "kind": "value",
            "expected": "inconsistent_across_formers"
          },
          {
            "name": "exhaustion_comparable_gate",
            "role": "gate",
            "field": "exhaustion_comparable_across_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_nondeterminant_gate",
            "role": "gate",
            "field": "exhaustion_explains_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_separable_program_gate",
            "role": "gate",
            "field": "exhaustion_separable_program",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "exhaustion_counterexample_gate",
            "role": "gate",
            "field": "exhaustion_counterexample_model",
            "kind": "value",
            "expected": "KPC_GP"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "exhaustion_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "cf_inconsistent_present_without_activator",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "program",
            "entity": "present",
            "valueField": "activator_state",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_no_block_but_absent",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "none",
            "valueField": "program",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_late_block_without_poised_locus",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "late_activation",
            "valueField": "locus_state",
            "expected": [
              "closing_committed",
              "active"
            ]
          },
          {
            "name": "cf_inconsistent_early_block_with_brake_released",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "early_release",
            "valueField": "brake_state",
            "expected": [
              "released"
            ]
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: FAIL\n  x list_match: list_match (answer_field='program_map'): FAIL  (score=0.0)\n  + sig_includes_itgae: passed=True, score=1.0\n  x declared_a_known_confounder: passed=False, score=0.0\n  x list_match: list_match (answer_field='generic_signature_relations'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='cd103_program_relations'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='per_model_regulatory_state'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='regulator_tuples'): FAIL  (score=0.0)\n  x locus_differential_gate: passed=False, score=0.0\n  + formation_limited_by_gate: passed=True, score=1.0\n  + locus_competent_in_naive_gate: passed=True, score=1.0\n  x list_match: list_match (answer_field='ready_silent_fraction_relations'): FAIL  (score=0.0)\n  x activator_chromatin_footprint_gate: passed=False, score=0.0\n  x exhaustion_comparable_gate: passed=False, score=0.0\n  + exhaustion_nondeterminant_gate: passed=True, score=1.0\n  + exhaustion_separable_program_gate: passed=True, score=1.0\n  x exhaustion_counterexample_gate: passed=False, score=0.0\n  x list_match: list_match (answer_field='exhaustion_relations'): FAIL  (score=0.0)\n  + hard_fail 'cf_signature_gut_contaminant': not triggered\n  + hard_fail 'cf_inconsistent_present_without_activator': not triggered\n  + hard_fail 'cf_inconsistent_no_block_but_absent': not triggered\n  + hard_fail 'cf_inconsistent_late_block_without_poised_locus': not triggered\n  + hard_fail 'cf_inconsistent_early_block_with_brake_released': not triggered",
      "metrics": {
        "pass_rule": "all",
        "scoring_count": 17,
        "scoring_passed": 5,
        "scoring_total_score": 5,
        "hard_fail_triggered": []
      },
      "fieldScores": {
        "list_match": 0,
        "sig_includes_itgae": 1,
        "declared_a_known_confounder": 0,
        "locus_differential_gate": 0,
        "formation_limited_by_gate": 1,
        "locus_competent_in_naive_gate": 1,
        "activator_chromatin_footprint_gate": 0,
        "exhaustion_comparable_gate": 0,
        "exhaustion_nondeterminant_gate": 1,
        "exhaustion_separable_program_gate": 1,
        "exhaustion_counterexample_gate": 0
      },
      "graderChecks": [
        {
          "name": "program_map",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "sig_includes_itgae",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "declared_a_known_confounder",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "generic_signature_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "cd103_program_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "per_model_regulatory_state",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "regulator_tuples",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "locus_differential_gate",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "formation_limited_by_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "locus_competent_in_naive_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "ready_silent_fraction_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "activator_chromatin_footprint_gate",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "exhaustion_comparable_gate",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "exhaustion_nondeterminant_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "exhaustion_separable_program_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "exhaustion_counterexample_gate",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "exhaustion_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "cf_signature_gut_contaminant",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_present_without_activator",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_no_block_but_absent",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_late_block_without_poised_locus",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_early_block_with_brake_released",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "activator_chromatin_footprint": "consistent_in_formers",
        "cd103_program_relations": [
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_B16_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "mWnt_GP_vs_B16_GP",
            "relation": ">"
          },
          {
            "pair": "mWnt_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": "~"
          }
        ],
        "core_residency_signature": [
          "Itgae"
        ],
        "excluded_confounders": [
          "Tox"
        ],
        "exhaustion_comparable_across_split": "yes",
        "exhaustion_counterexample_model": "B_ALL_GP",
        "exhaustion_explains_split": "no",
        "exhaustion_relations": [
          {
            "pair": "B_ALL_GP_vs_KPC_GP",
            "relation": ">"
          },
          {
            "pair": "B_ALL_GP_vs_B16_GP",
            "relation": ">"
          },
          {
            "pair": "B_ALL_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_B16_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": ">"
          }
        ],
        "exhaustion_separable_program": "yes",
        "formation_limited_by": "absent_induction",
        "generic_signature_relations": [
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": "~"
          },
          {
            "pair": "KPC_GP_vs_B16_GP",
            "relation": "~"
          },
          {
            "pair": "KPC_GP_vs_B_ALL_GP",
            "relation": "~"
          },
          {
            "pair": "mWnt_GP_vs_B16_GP",
            "relation": "~"
          },
          {
            "pair": "mWnt_GP_vs_B_ALL_GP",
            "relation": "~"
          },
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": "~"
          }
        ],
        "locus_competent_in_naive": "yes",
        "locus_differential": "model_specific_element",
        "per_model_regulatory_state": [
          {
            "activator_state": "present",
            "blocking_step": "none",
            "brake_state": "released",
            "locus_state": "active",
            "model": "KPC_GP",
            "program": "present"
          },
          {
            "activator_state": "partial",
            "blocking_step": "late_activation",
            "brake_state": "partial",
            "locus_state": "poised_unfired",
            "model": "mWnt_GP",
            "program": "partial"
          },
          {
            "activator_state": "absent",
            "blocking_step": "both_steps",
            "brake_state": "engaged",
            "locus_state": "repressed",
            "model": "B16_GP",
            "program": "absent"
          },
          {
            "activator_state": "absent",
            "blocking_step": "both_steps",
            "brake_state": "engaged",
            "locus_state": "repressed",
            "model": "B_ALL_GP",
            "program": "absent"
          }
        ],
        "program_map": [
          {
            "call": "present",
            "model": "KPC_GP"
          },
          {
            "call": "partial",
            "model": "mWnt_GP"
          },
          {
            "call": "absent",
            "model": "B16_GP"
          },
          {
            "call": "absent",
            "model": "B_ALL_GP"
          }
        ],
        "ready_silent_fraction_relations": [
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": "~"
          },
          {
            "pair": "KPC_GP_vs_B16_GP",
            "relation": "~"
          },
          {
            "pair": "KPC_GP_vs_B_ALL_GP",
            "relation": "~"
          },
          {
            "pair": "mWnt_GP_vs_B16_GP",
            "relation": "~"
          },
          {
            "pair": "mWnt_GP_vs_B_ALL_GP",
            "relation": "~"
          },
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": "~"
          }
        ],
        "regulator_tuples": [
          {
            "evidence_basis": "target_program_behavior",
            "expr_vs_activity": "concordant",
            "gating_step": "late_activation",
            "regulator": "Runx3",
            "role": "activator"
          },
          {
            "evidence_basis": "own_abundance",
            "expr_vs_activity": "discordant",
            "gating_step": "early_release",
            "regulator": "Prdm1",
            "role": "repressor"
          },
          {
            "evidence_basis": "indeterminate",
            "expr_vs_activity": "indeterminate",
            "gating_step": "indeterminate",
            "regulator": "Nr4a1",
            "role": "dual"
          }
        ]
      },
      "nTurns": 20,
      "checkpointSignals": {
        "clustering": true,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": false,
        "nerveJointAnchor": false,
        "multiomeAssembly": true,
        "depthNormalizedAtac": false,
        "regulonInference": true,
        "cd8SubsetIntegration": true,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": false,
        "orthologProjection": false,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": false,
        "mousePseudobulkDesign": false,
        "cancerEpithelialDenominator": false,
        "geneLevelDe": false,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "trajectoryFile": "scbench-long/trajectories/green_2_tme_context_dependence__grok-4.20-beta-0309-reasoning__pi__r2.json",
      "runId": "grok-4.20-beta-0309-reasoning__pi__green_2_tme_context_dependence__r2",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-grok-4-20-beta-reasoning-pi",
      "modelName": "grok-4.20-beta-0309-reasoning",
      "harness": "pi",
      "task": "green_2_tme_context_dependence",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 3,
      "passed": false,
      "score": 5,
      "durationS": 156.04,
      "cost": 0.05231125,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "program_map",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "sig_includes_itgae",
            "role": "gate",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Itgae"
            ]
          },
          {
            "name": "declared_a_known_confounder",
            "role": "gate",
            "field": "$.excluded_confounders[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "cf_signature_gut_contaminant",
            "role": "hard_fail",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "generic_signature_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd103_program_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "<"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "per_model_regulatory_state",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "poised_unfired"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "late_activation"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "closing_committed"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_tuples",
            "kind": "list",
            "matchKey": "regulator",
            "k": 1,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "role": {
                    "op": "equals",
                    "values": [
                      "repressor"
                    ]
                  },
                  "gating_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  },
                  "evidence_basis": {
                    "op": "equals",
                    "values": [
                      "target_program_behavior"
                    ]
                  },
                  "expr_vs_activity": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "locus_differential_gate",
            "role": "gate",
            "field": "locus_differential",
            "kind": "value",
            "expected": "uniform_across_models"
          },
          {
            "name": "formation_limited_by_gate",
            "role": "gate",
            "field": "formation_limited_by",
            "kind": "value",
            "expected": "absent_induction"
          },
          {
            "name": "locus_competent_in_naive_gate",
            "role": "gate",
            "field": "locus_competent_in_naive",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "ready_silent_fraction_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 3,
            "expected": [
              {
                "key": "B16_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_KPC_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "activator_chromatin_footprint_gate",
            "role": "gate",
            "field": "activator_chromatin_footprint",
            "kind": "value",
            "expected": "inconsistent_across_formers"
          },
          {
            "name": "exhaustion_comparable_gate",
            "role": "gate",
            "field": "exhaustion_comparable_across_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_nondeterminant_gate",
            "role": "gate",
            "field": "exhaustion_explains_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_separable_program_gate",
            "role": "gate",
            "field": "exhaustion_separable_program",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "exhaustion_counterexample_gate",
            "role": "gate",
            "field": "exhaustion_counterexample_model",
            "kind": "value",
            "expected": "KPC_GP"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "exhaustion_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "cf_inconsistent_present_without_activator",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "program",
            "entity": "present",
            "valueField": "activator_state",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_no_block_but_absent",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "none",
            "valueField": "program",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_late_block_without_poised_locus",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "late_activation",
            "valueField": "locus_state",
            "expected": [
              "closing_committed",
              "active"
            ]
          },
          {
            "name": "cf_inconsistent_early_block_with_brake_released",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "early_release",
            "valueField": "brake_state",
            "expected": [
              "released"
            ]
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: FAIL\n  x list_match: list_match (answer_field='program_map'): FAIL  (score=0.0)\n  + sig_includes_itgae: passed=True, score=1.0\n  x declared_a_known_confounder: passed=False, score=0.0\n  x list_match: list_match (answer_field='generic_signature_relations'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='cd103_program_relations'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='per_model_regulatory_state'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='regulator_tuples'): FAIL  (score=0.0)\n  x locus_differential_gate: passed=False, score=0.0\n  + formation_limited_by_gate: passed=True, score=1.0\n  + locus_competent_in_naive_gate: passed=True, score=1.0\n  x list_match: list_match (answer_field='ready_silent_fraction_relations'): FAIL  (score=0.0)\n  x activator_chromatin_footprint_gate: passed=False, score=0.0\n  x exhaustion_comparable_gate: passed=False, score=0.0\n  + exhaustion_nondeterminant_gate: passed=True, score=1.0\n  + exhaustion_separable_program_gate: passed=True, score=1.0\n  x exhaustion_counterexample_gate: passed=False, score=0.0\n  x list_match: list_match (answer_field='exhaustion_relations'): FAIL  (score=0.0)\n  + hard_fail 'cf_signature_gut_contaminant': not triggered\n  + hard_fail 'cf_inconsistent_present_without_activator': not triggered\n  + hard_fail 'cf_inconsistent_no_block_but_absent': not triggered\n  + hard_fail 'cf_inconsistent_late_block_without_poised_locus': not triggered\n  + hard_fail 'cf_inconsistent_early_block_with_brake_released': not triggered",
      "metrics": {
        "pass_rule": "all",
        "scoring_count": 17,
        "scoring_passed": 5,
        "scoring_total_score": 5,
        "hard_fail_triggered": []
      },
      "fieldScores": {
        "list_match": 0,
        "sig_includes_itgae": 1,
        "declared_a_known_confounder": 0,
        "locus_differential_gate": 0,
        "formation_limited_by_gate": 1,
        "locus_competent_in_naive_gate": 1,
        "activator_chromatin_footprint_gate": 0,
        "exhaustion_comparable_gate": 0,
        "exhaustion_nondeterminant_gate": 1,
        "exhaustion_separable_program_gate": 1,
        "exhaustion_counterexample_gate": 0
      },
      "graderChecks": [
        {
          "name": "program_map",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "sig_includes_itgae",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "declared_a_known_confounder",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "generic_signature_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "cd103_program_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "per_model_regulatory_state",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "regulator_tuples",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "locus_differential_gate",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "formation_limited_by_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "locus_competent_in_naive_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "ready_silent_fraction_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "activator_chromatin_footprint_gate",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "exhaustion_comparable_gate",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "exhaustion_nondeterminant_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "exhaustion_separable_program_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "exhaustion_counterexample_gate",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "exhaustion_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "cf_signature_gut_contaminant",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_present_without_activator",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_no_block_but_absent",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_late_block_without_poised_locus",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_early_block_with_brake_released",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "activator_chromatin_footprint": "consistent_in_formers",
        "cd103_program_relations": [
          {
            "pair": "KPC_GP_vs_B16_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "mWnt_GP_vs_B16_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "mWnt_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": "~"
          }
        ],
        "core_residency_signature": [
          "Itgae",
          "Cd69",
          "Cxcr6",
          "Zfp683"
        ],
        "excluded_confounders": [
          "Pdcd1",
          "Tox",
          "Havcr2"
        ],
        "exhaustion_comparable_across_split": "yes",
        "exhaustion_counterexample_model": "B16_GP",
        "exhaustion_explains_split": "no",
        "exhaustion_relations": [
          {
            "pair": "B_ALL_GP_vs_KPC_GP",
            "relation": ">"
          },
          {
            "pair": "B16_GP_vs_KPC_GP",
            "relation": ">"
          },
          {
            "pair": "B_ALL_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "B_ALL_GP_vs_B16_GP",
            "relation": "~"
          },
          {
            "pair": "mWnt_GP_vs_KPC_GP",
            "relation": "<"
          }
        ],
        "exhaustion_separable_program": "yes",
        "formation_limited_by": "absent_induction",
        "generic_signature_relations": [
          {
            "pair": "KPC_GP_vs_B16_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "mWnt_GP_vs_B16_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "mWnt_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": "~"
          }
        ],
        "locus_competent_in_naive": "yes",
        "locus_differential": "model_specific_element",
        "per_model_regulatory_state": [
          {
            "activator_state": "present",
            "blocking_step": "none",
            "brake_state": "released",
            "locus_state": "active",
            "model": "KPC_GP",
            "program": "present"
          },
          {
            "activator_state": "partial",
            "blocking_step": "late_activation",
            "brake_state": "partial",
            "locus_state": "poised_unfired",
            "model": "mWnt_GP",
            "program": "partial"
          },
          {
            "activator_state": "absent",
            "blocking_step": "both_steps",
            "brake_state": "engaged",
            "locus_state": "repressed",
            "model": "B16_GP",
            "program": "absent"
          },
          {
            "activator_state": "absent",
            "blocking_step": "both_steps",
            "brake_state": "engaged",
            "locus_state": "repressed",
            "model": "B_ALL_GP",
            "program": "absent"
          }
        ],
        "program_map": [
          {
            "call": "present",
            "model": "KPC_GP"
          },
          {
            "call": "partial",
            "model": "mWnt_GP"
          },
          {
            "call": "absent",
            "model": "B16_GP"
          },
          {
            "call": "absent",
            "model": "B_ALL_GP"
          }
        ],
        "ready_silent_fraction_relations": [
          {
            "pair": "KPC_GP_vs_B16_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "mWnt_GP_vs_B16_GP",
            "relation": "~"
          },
          {
            "pair": "KPC_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "mWnt_GP_vs_B_ALL_GP",
            "relation": "~"
          },
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": "~"
          }
        ],
        "regulator_tuples": [
          {
            "evidence_basis": "target_program_behavior",
            "expr_vs_activity": "concordant",
            "gating_step": "late_activation",
            "regulator": "Runx3",
            "role": "activator"
          },
          {
            "evidence_basis": "own_abundance",
            "expr_vs_activity": "discordant",
            "gating_step": "early_release",
            "regulator": "Tox",
            "role": "repressor"
          },
          {
            "evidence_basis": "target_program_behavior",
            "expr_vs_activity": "concordant",
            "gating_step": "none",
            "regulator": "Prdm1",
            "role": "dual"
          }
        ]
      },
      "nTurns": 19,
      "checkpointSignals": {
        "clustering": false,
        "markerAnnotation": false,
        "scoreBasedAnnotation": false,
        "normalization": false,
        "nerveJointAnchor": false,
        "multiomeAssembly": true,
        "depthNormalizedAtac": false,
        "regulonInference": true,
        "cd8SubsetIntegration": true,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": false,
        "orthologProjection": false,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": false,
        "mousePseudobulkDesign": false,
        "cancerEpithelialDenominator": false,
        "geneLevelDe": false,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "trajectoryFile": "scbench-long/trajectories/green_2_tme_context_dependence__grok-4.20-beta-0309-reasoning__pi__r3.json",
      "runId": "grok-4.20-beta-0309-reasoning__pi__green_2_tme_context_dependence__r3",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-grok-4-3-pi",
      "modelName": "grok-4.3",
      "harness": "pi",
      "task": "green_2_tme_context_dependence",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 1,
      "passed": false,
      "score": 5,
      "durationS": 48.09,
      "cost": 0.01039195,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "program_map",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "sig_includes_itgae",
            "role": "gate",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Itgae"
            ]
          },
          {
            "name": "declared_a_known_confounder",
            "role": "gate",
            "field": "$.excluded_confounders[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "cf_signature_gut_contaminant",
            "role": "hard_fail",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "generic_signature_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd103_program_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "<"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "per_model_regulatory_state",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "poised_unfired"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "late_activation"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "closing_committed"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_tuples",
            "kind": "list",
            "matchKey": "regulator",
            "k": 1,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "role": {
                    "op": "equals",
                    "values": [
                      "repressor"
                    ]
                  },
                  "gating_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  },
                  "evidence_basis": {
                    "op": "equals",
                    "values": [
                      "target_program_behavior"
                    ]
                  },
                  "expr_vs_activity": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "locus_differential_gate",
            "role": "gate",
            "field": "locus_differential",
            "kind": "value",
            "expected": "uniform_across_models"
          },
          {
            "name": "formation_limited_by_gate",
            "role": "gate",
            "field": "formation_limited_by",
            "kind": "value",
            "expected": "absent_induction"
          },
          {
            "name": "locus_competent_in_naive_gate",
            "role": "gate",
            "field": "locus_competent_in_naive",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "ready_silent_fraction_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 3,
            "expected": [
              {
                "key": "B16_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_KPC_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "activator_chromatin_footprint_gate",
            "role": "gate",
            "field": "activator_chromatin_footprint",
            "kind": "value",
            "expected": "inconsistent_across_formers"
          },
          {
            "name": "exhaustion_comparable_gate",
            "role": "gate",
            "field": "exhaustion_comparable_across_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_nondeterminant_gate",
            "role": "gate",
            "field": "exhaustion_explains_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_separable_program_gate",
            "role": "gate",
            "field": "exhaustion_separable_program",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "exhaustion_counterexample_gate",
            "role": "gate",
            "field": "exhaustion_counterexample_model",
            "kind": "value",
            "expected": "KPC_GP"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "exhaustion_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "cf_inconsistent_present_without_activator",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "program",
            "entity": "present",
            "valueField": "activator_state",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_no_block_but_absent",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "none",
            "valueField": "program",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_late_block_without_poised_locus",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "late_activation",
            "valueField": "locus_state",
            "expected": [
              "closing_committed",
              "active"
            ]
          },
          {
            "name": "cf_inconsistent_early_block_with_brake_released",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "early_release",
            "valueField": "brake_state",
            "expected": [
              "released"
            ]
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: FAIL\n  x list_match: list_match (answer_field='program_map'): FAIL  (score=0.0)\n  + sig_includes_itgae: passed=True, score=1.0\n  x declared_a_known_confounder: passed=False, score=0.0\n  x list_match: list_match (answer_field='generic_signature_relations'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='cd103_program_relations'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='per_model_regulatory_state'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='regulator_tuples'): FAIL  (score=0.0)\n  x locus_differential_gate: passed=False, score=0.0\n  + formation_limited_by_gate: passed=True, score=1.0\n  + locus_competent_in_naive_gate: passed=True, score=1.0\n  x list_match: list_match (answer_field='ready_silent_fraction_relations'): FAIL  (score=0.0)\n  x activator_chromatin_footprint_gate: passed=False, score=0.0\n  x exhaustion_comparable_gate: passed=False, score=0.0\n  + exhaustion_nondeterminant_gate: passed=True, score=1.0\n  + exhaustion_separable_program_gate: passed=True, score=1.0\n  x exhaustion_counterexample_gate: passed=False, score=0.0\n  x list_match: list_match (answer_field='exhaustion_relations'): FAIL  (score=0.0)\n  + hard_fail 'cf_signature_gut_contaminant': not triggered\n  + hard_fail 'cf_inconsistent_present_without_activator': not triggered\n  + hard_fail 'cf_inconsistent_no_block_but_absent': not triggered\n  + hard_fail 'cf_inconsistent_late_block_without_poised_locus': not triggered\n  + hard_fail 'cf_inconsistent_early_block_with_brake_released': not triggered",
      "metrics": {
        "pass_rule": "all",
        "scoring_count": 17,
        "scoring_passed": 5,
        "scoring_total_score": 5,
        "hard_fail_triggered": []
      },
      "fieldScores": {
        "list_match": 0,
        "sig_includes_itgae": 1,
        "declared_a_known_confounder": 0,
        "locus_differential_gate": 0,
        "formation_limited_by_gate": 1,
        "locus_competent_in_naive_gate": 1,
        "activator_chromatin_footprint_gate": 0,
        "exhaustion_comparable_gate": 0,
        "exhaustion_nondeterminant_gate": 1,
        "exhaustion_separable_program_gate": 1,
        "exhaustion_counterexample_gate": 0
      },
      "graderChecks": [
        {
          "name": "program_map",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "sig_includes_itgae",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "declared_a_known_confounder",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "generic_signature_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "cd103_program_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "per_model_regulatory_state",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "regulator_tuples",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "locus_differential_gate",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "formation_limited_by_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "locus_competent_in_naive_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "ready_silent_fraction_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "activator_chromatin_footprint_gate",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "exhaustion_comparable_gate",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "exhaustion_nondeterminant_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "exhaustion_separable_program_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "exhaustion_counterexample_gate",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "exhaustion_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "cf_signature_gut_contaminant",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_present_without_activator",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_no_block_but_absent",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_late_block_without_poised_locus",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_early_block_with_brake_released",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "activator_chromatin_footprint": "consistent_in_formers",
        "cd103_program_relations": [
          {
            "pair": "B16_GP_vs_KPC_GP",
            "relation": "~"
          },
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "B_ALL_GP_vs_mWnt_GP",
            "relation": "~"
          }
        ],
        "core_residency_signature": [
          "Itgae",
          "Cd69",
          "Runx3",
          "Batf",
          "Prdm1"
        ],
        "excluded_confounders": [
          "Pdcd1",
          "Tcf7",
          "Tox"
        ],
        "exhaustion_comparable_across_split": "yes",
        "exhaustion_counterexample_model": "B16_GP",
        "exhaustion_explains_split": "no",
        "exhaustion_relations": [
          {
            "pair": "B16_GP_vs_KPC_GP",
            "relation": "~"
          },
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": "~"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": "~"
          },
          {
            "pair": "KPC_GP_vs_B_ALL_GP",
            "relation": "~"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": "~"
          },
          {
            "pair": "B_ALL_GP_vs_mWnt_GP",
            "relation": "~"
          }
        ],
        "exhaustion_separable_program": "yes",
        "formation_limited_by": "absent_induction",
        "generic_signature_relations": [
          {
            "pair": "B16_GP_vs_KPC_GP",
            "relation": "~"
          },
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "B_ALL_GP_vs_mWnt_GP",
            "relation": "~"
          }
        ],
        "locus_competent_in_naive": "yes",
        "locus_differential": "model_specific_element",
        "per_model_regulatory_state": [
          {
            "activator_state": "present",
            "blocking_step": "none",
            "brake_state": "released",
            "locus_state": "active",
            "model": "B16_GP",
            "program": "present"
          },
          {
            "activator_state": "present",
            "blocking_step": "none",
            "brake_state": "released",
            "locus_state": "active",
            "model": "KPC_GP",
            "program": "present"
          },
          {
            "activator_state": "partial",
            "blocking_step": "late_activation",
            "brake_state": "partial",
            "locus_state": "poised_unfired",
            "model": "B_ALL_GP",
            "program": "partial"
          },
          {
            "activator_state": "absent",
            "blocking_step": "both_steps",
            "brake_state": "engaged",
            "locus_state": "repressed",
            "model": "mWnt_GP",
            "program": "absent"
          }
        ],
        "program_map": [
          {
            "call": "present",
            "model": "B16_GP"
          },
          {
            "call": "present",
            "model": "KPC_GP"
          },
          {
            "call": "partial",
            "model": "B_ALL_GP"
          },
          {
            "call": "absent",
            "model": "mWnt_GP"
          }
        ],
        "ready_silent_fraction_relations": [
          {
            "pair": "B16_GP_vs_KPC_GP",
            "relation": "~"
          },
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "B_ALL_GP_vs_mWnt_GP",
            "relation": "~"
          }
        ],
        "regulator_tuples": [
          {
            "evidence_basis": "target_program_behavior",
            "expr_vs_activity": "concordant",
            "gating_step": "late_activation",
            "regulator": "Runx3",
            "role": "activator"
          },
          {
            "evidence_basis": "target_program_behavior",
            "expr_vs_activity": "concordant",
            "gating_step": "late_activation",
            "regulator": "Batf",
            "role": "activator"
          },
          {
            "evidence_basis": "own_abundance",
            "expr_vs_activity": "discordant",
            "gating_step": "early_release",
            "regulator": "Prdm1",
            "role": "repressor"
          }
        ]
      },
      "nTurns": 6,
      "checkpointSignals": {
        "clustering": false,
        "markerAnnotation": false,
        "scoreBasedAnnotation": false,
        "normalization": false,
        "nerveJointAnchor": false,
        "multiomeAssembly": true,
        "depthNormalizedAtac": false,
        "regulonInference": true,
        "cd8SubsetIntegration": true,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": false,
        "orthologProjection": false,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": false,
        "mousePseudobulkDesign": false,
        "cancerEpithelialDenominator": false,
        "geneLevelDe": false,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "trajectoryFile": "scbench-long/trajectories/green_2_tme_context_dependence__grok-4.3__pi__r1.json",
      "runId": "grok-4.3__pi__green_2_tme_context_dependence__r1",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-grok-4-3-pi",
      "modelName": "grok-4.3",
      "harness": "pi",
      "task": "green_2_tme_context_dependence",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 2,
      "passed": false,
      "score": 6,
      "durationS": 29.01,
      "cost": 0.013951900000000001,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "program_map",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "sig_includes_itgae",
            "role": "gate",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Itgae"
            ]
          },
          {
            "name": "declared_a_known_confounder",
            "role": "gate",
            "field": "$.excluded_confounders[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "cf_signature_gut_contaminant",
            "role": "hard_fail",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "generic_signature_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd103_program_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "<"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "per_model_regulatory_state",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "poised_unfired"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "late_activation"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "closing_committed"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_tuples",
            "kind": "list",
            "matchKey": "regulator",
            "k": 1,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "role": {
                    "op": "equals",
                    "values": [
                      "repressor"
                    ]
                  },
                  "gating_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  },
                  "evidence_basis": {
                    "op": "equals",
                    "values": [
                      "target_program_behavior"
                    ]
                  },
                  "expr_vs_activity": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "locus_differential_gate",
            "role": "gate",
            "field": "locus_differential",
            "kind": "value",
            "expected": "uniform_across_models"
          },
          {
            "name": "formation_limited_by_gate",
            "role": "gate",
            "field": "formation_limited_by",
            "kind": "value",
            "expected": "absent_induction"
          },
          {
            "name": "locus_competent_in_naive_gate",
            "role": "gate",
            "field": "locus_competent_in_naive",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "ready_silent_fraction_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 3,
            "expected": [
              {
                "key": "B16_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_KPC_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "activator_chromatin_footprint_gate",
            "role": "gate",
            "field": "activator_chromatin_footprint",
            "kind": "value",
            "expected": "inconsistent_across_formers"
          },
          {
            "name": "exhaustion_comparable_gate",
            "role": "gate",
            "field": "exhaustion_comparable_across_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_nondeterminant_gate",
            "role": "gate",
            "field": "exhaustion_explains_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_separable_program_gate",
            "role": "gate",
            "field": "exhaustion_separable_program",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "exhaustion_counterexample_gate",
            "role": "gate",
            "field": "exhaustion_counterexample_model",
            "kind": "value",
            "expected": "KPC_GP"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "exhaustion_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "cf_inconsistent_present_without_activator",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "program",
            "entity": "present",
            "valueField": "activator_state",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_no_block_but_absent",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "none",
            "valueField": "program",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_late_block_without_poised_locus",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "late_activation",
            "valueField": "locus_state",
            "expected": [
              "closing_committed",
              "active"
            ]
          },
          {
            "name": "cf_inconsistent_early_block_with_brake_released",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "early_release",
            "valueField": "brake_state",
            "expected": [
              "released"
            ]
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: FAIL\n  x list_match: list_match (answer_field='program_map'): FAIL  (score=0.0)\n  + sig_includes_itgae: passed=True, score=1.0\n  x declared_a_known_confounder: passed=False, score=0.0\n  x list_match: list_match (answer_field='generic_signature_relations'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='cd103_program_relations'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='per_model_regulatory_state'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='regulator_tuples'): FAIL  (score=0.0)\n  + locus_differential_gate: passed=True, score=1.0\n  + formation_limited_by_gate: passed=True, score=1.0\n  + locus_competent_in_naive_gate: passed=True, score=1.0\n  x list_match: list_match (answer_field='ready_silent_fraction_relations'): FAIL  (score=0.0)\n  x activator_chromatin_footprint_gate: passed=False, score=0.0\n  x exhaustion_comparable_gate: passed=False, score=0.0\n  + exhaustion_nondeterminant_gate: passed=True, score=1.0\n  + exhaustion_separable_program_gate: passed=True, score=1.0\n  x exhaustion_counterexample_gate: passed=False, score=0.0\n  x list_match: list_match (answer_field='exhaustion_relations'): FAIL  (score=0.0)\n  x hard_fail 'cf_signature_gut_contaminant': TRIGGERED\n  + hard_fail 'cf_inconsistent_present_without_activator': not triggered\n  + hard_fail 'cf_inconsistent_no_block_but_absent': not triggered\n  + hard_fail 'cf_inconsistent_late_block_without_poised_locus': not triggered\n  + hard_fail 'cf_inconsistent_early_block_with_brake_released': not triggered",
      "metrics": {
        "pass_rule": "all",
        "scoring_count": 17,
        "scoring_passed": 6,
        "scoring_total_score": 6,
        "hard_fail_triggered": [
          "cf_signature_gut_contaminant"
        ]
      },
      "fieldScores": {
        "list_match": 0,
        "sig_includes_itgae": 1,
        "declared_a_known_confounder": 0,
        "locus_differential_gate": 1,
        "formation_limited_by_gate": 1,
        "locus_competent_in_naive_gate": 1,
        "activator_chromatin_footprint_gate": 0,
        "exhaustion_comparable_gate": 0,
        "exhaustion_nondeterminant_gate": 1,
        "exhaustion_separable_program_gate": 1,
        "exhaustion_counterexample_gate": 0
      },
      "graderChecks": [
        {
          "name": "program_map",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "sig_includes_itgae",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "declared_a_known_confounder",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "generic_signature_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "cd103_program_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "per_model_regulatory_state",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "regulator_tuples",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "locus_differential_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "formation_limited_by_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "locus_competent_in_naive_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "ready_silent_fraction_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "activator_chromatin_footprint_gate",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "exhaustion_comparable_gate",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "exhaustion_nondeterminant_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "exhaustion_separable_program_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "exhaustion_counterexample_gate",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "exhaustion_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "cf_signature_gut_contaminant",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "cf_inconsistent_present_without_activator",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_no_block_but_absent",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_late_block_without_poised_locus",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_early_block_with_brake_released",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "activator_chromatin_footprint": "consistent_in_formers",
        "cd103_program_relations": [
          {
            "pair": "B16_GP_vs_KPC_GP",
            "relation": "~"
          },
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": "~"
          },
          {
            "pair": "KPC_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": "~"
          },
          {
            "pair": "B_ALL_GP_vs_mWnt_GP",
            "relation": "<"
          }
        ],
        "core_residency_signature": [
          "Itgae",
          "Itgb7",
          "Runx3",
          "Hobit",
          "Prdm1",
          "Cd69",
          "Ccr9"
        ],
        "excluded_confounders": [
          "Pdcd1",
          "Tigit",
          "Lag3",
          "Tox"
        ],
        "exhaustion_comparable_across_split": "yes",
        "exhaustion_counterexample_model": "B_ALL_GP",
        "exhaustion_explains_split": "no",
        "exhaustion_relations": [
          {
            "pair": "B16_GP_vs_KPC_GP",
            "relation": "~"
          },
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": "~"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": "~"
          },
          {
            "pair": "KPC_GP_vs_B_ALL_GP",
            "relation": "~"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": "~"
          },
          {
            "pair": "B_ALL_GP_vs_mWnt_GP",
            "relation": "~"
          }
        ],
        "exhaustion_separable_program": "yes",
        "formation_limited_by": "absent_induction",
        "generic_signature_relations": [
          {
            "pair": "B16_GP_vs_KPC_GP",
            "relation": "~"
          },
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": "~"
          },
          {
            "pair": "KPC_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": "~"
          },
          {
            "pair": "B_ALL_GP_vs_mWnt_GP",
            "relation": "<"
          }
        ],
        "locus_competent_in_naive": "yes",
        "locus_differential": "uniform_across_models",
        "per_model_regulatory_state": [
          {
            "activator_state": "present",
            "blocking_step": "none",
            "brake_state": "released",
            "locus_state": "active",
            "model": "B16_GP",
            "program": "present"
          },
          {
            "activator_state": "present",
            "blocking_step": "none",
            "brake_state": "released",
            "locus_state": "active",
            "model": "KPC_GP",
            "program": "present"
          },
          {
            "activator_state": "absent",
            "blocking_step": "both_steps",
            "brake_state": "engaged",
            "locus_state": "repressed",
            "model": "B_ALL_GP",
            "program": "absent"
          },
          {
            "activator_state": "present",
            "blocking_step": "none",
            "brake_state": "released",
            "locus_state": "active",
            "model": "mWnt_GP",
            "program": "present"
          }
        ],
        "program_map": [
          {
            "call": "present",
            "model": "B16_GP"
          },
          {
            "call": "present",
            "model": "KPC_GP"
          },
          {
            "call": "absent",
            "model": "B_ALL_GP"
          },
          {
            "call": "present",
            "model": "mWnt_GP"
          }
        ],
        "ready_silent_fraction_relations": [
          {
            "pair": "B16_GP_vs_KPC_GP",
            "relation": "~"
          },
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": "~"
          },
          {
            "pair": "KPC_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": "~"
          },
          {
            "pair": "B_ALL_GP_vs_mWnt_GP",
            "relation": "<"
          }
        ],
        "regulator_tuples": [
          {
            "evidence_basis": "own_abundance",
            "expr_vs_activity": "concordant",
            "gating_step": "early_release",
            "regulator": "Runx3",
            "role": "activator"
          },
          {
            "evidence_basis": "target_program_behavior",
            "expr_vs_activity": "concordant",
            "gating_step": "late_activation",
            "regulator": "Hobit",
            "role": "activator"
          },
          {
            "evidence_basis": "indeterminate",
            "expr_vs_activity": "discordant",
            "gating_step": "none",
            "regulator": "Prdm1",
            "role": "repressor"
          }
        ]
      },
      "nTurns": 5,
      "checkpointSignals": {
        "clustering": false,
        "markerAnnotation": false,
        "scoreBasedAnnotation": false,
        "normalization": false,
        "nerveJointAnchor": false,
        "multiomeAssembly": true,
        "depthNormalizedAtac": false,
        "regulonInference": true,
        "cd8SubsetIntegration": true,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": false,
        "orthologProjection": false,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": false,
        "mousePseudobulkDesign": false,
        "cancerEpithelialDenominator": false,
        "geneLevelDe": false,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "trajectoryFile": "scbench-long/trajectories/green_2_tme_context_dependence__grok-4.3__pi__r2.json",
      "runId": "grok-4.3__pi__green_2_tme_context_dependence__r2",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-grok-4-3-pi",
      "modelName": "grok-4.3",
      "harness": "pi",
      "task": "green_2_tme_context_dependence",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 3,
      "passed": false,
      "score": 5,
      "durationS": 35.01,
      "cost": 0.010968100000000001,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "program_map",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "sig_includes_itgae",
            "role": "gate",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Itgae"
            ]
          },
          {
            "name": "declared_a_known_confounder",
            "role": "gate",
            "field": "$.excluded_confounders[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "cf_signature_gut_contaminant",
            "role": "hard_fail",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "generic_signature_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd103_program_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "<"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "per_model_regulatory_state",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "poised_unfired"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "late_activation"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "closing_committed"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_tuples",
            "kind": "list",
            "matchKey": "regulator",
            "k": 1,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "role": {
                    "op": "equals",
                    "values": [
                      "repressor"
                    ]
                  },
                  "gating_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  },
                  "evidence_basis": {
                    "op": "equals",
                    "values": [
                      "target_program_behavior"
                    ]
                  },
                  "expr_vs_activity": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "locus_differential_gate",
            "role": "gate",
            "field": "locus_differential",
            "kind": "value",
            "expected": "uniform_across_models"
          },
          {
            "name": "formation_limited_by_gate",
            "role": "gate",
            "field": "formation_limited_by",
            "kind": "value",
            "expected": "absent_induction"
          },
          {
            "name": "locus_competent_in_naive_gate",
            "role": "gate",
            "field": "locus_competent_in_naive",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "ready_silent_fraction_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 3,
            "expected": [
              {
                "key": "B16_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_KPC_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "activator_chromatin_footprint_gate",
            "role": "gate",
            "field": "activator_chromatin_footprint",
            "kind": "value",
            "expected": "inconsistent_across_formers"
          },
          {
            "name": "exhaustion_comparable_gate",
            "role": "gate",
            "field": "exhaustion_comparable_across_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_nondeterminant_gate",
            "role": "gate",
            "field": "exhaustion_explains_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_separable_program_gate",
            "role": "gate",
            "field": "exhaustion_separable_program",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "exhaustion_counterexample_gate",
            "role": "gate",
            "field": "exhaustion_counterexample_model",
            "kind": "value",
            "expected": "KPC_GP"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "exhaustion_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "cf_inconsistent_present_without_activator",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "program",
            "entity": "present",
            "valueField": "activator_state",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_no_block_but_absent",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "none",
            "valueField": "program",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_late_block_without_poised_locus",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "late_activation",
            "valueField": "locus_state",
            "expected": [
              "closing_committed",
              "active"
            ]
          },
          {
            "name": "cf_inconsistent_early_block_with_brake_released",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "early_release",
            "valueField": "brake_state",
            "expected": [
              "released"
            ]
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: FAIL\n  x list_match: list_match (answer_field='program_map'): FAIL  (score=0.0)\n  + sig_includes_itgae: passed=True, score=1.0\n  x declared_a_known_confounder: passed=False, score=0.0\n  x list_match: list_match (answer_field='generic_signature_relations'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='cd103_program_relations'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='per_model_regulatory_state'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='regulator_tuples'): FAIL  (score=0.0)\n  x locus_differential_gate: passed=False, score=0.0\n  + formation_limited_by_gate: passed=True, score=1.0\n  + locus_competent_in_naive_gate: passed=True, score=1.0\n  x list_match: list_match (answer_field='ready_silent_fraction_relations'): FAIL  (score=0.0)\n  x activator_chromatin_footprint_gate: passed=False, score=0.0\n  x exhaustion_comparable_gate: passed=False, score=0.0\n  + exhaustion_nondeterminant_gate: passed=True, score=1.0\n  + exhaustion_separable_program_gate: passed=True, score=1.0\n  x exhaustion_counterexample_gate: passed=False, score=0.0\n  x list_match: list_match (answer_field='exhaustion_relations'): FAIL  (score=0.0)\n  + hard_fail 'cf_signature_gut_contaminant': not triggered\n  + hard_fail 'cf_inconsistent_present_without_activator': not triggered\n  + hard_fail 'cf_inconsistent_no_block_but_absent': not triggered\n  + hard_fail 'cf_inconsistent_late_block_without_poised_locus': not triggered\n  + hard_fail 'cf_inconsistent_early_block_with_brake_released': not triggered",
      "metrics": {
        "pass_rule": "all",
        "scoring_count": 17,
        "scoring_passed": 5,
        "scoring_total_score": 5,
        "hard_fail_triggered": []
      },
      "fieldScores": {
        "list_match": 0,
        "sig_includes_itgae": 1,
        "declared_a_known_confounder": 0,
        "locus_differential_gate": 0,
        "formation_limited_by_gate": 1,
        "locus_competent_in_naive_gate": 1,
        "activator_chromatin_footprint_gate": 0,
        "exhaustion_comparable_gate": 0,
        "exhaustion_nondeterminant_gate": 1,
        "exhaustion_separable_program_gate": 1,
        "exhaustion_counterexample_gate": 0
      },
      "graderChecks": [
        {
          "name": "program_map",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "sig_includes_itgae",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "declared_a_known_confounder",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "generic_signature_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "cd103_program_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "per_model_regulatory_state",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "regulator_tuples",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "locus_differential_gate",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "formation_limited_by_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "locus_competent_in_naive_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "ready_silent_fraction_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "activator_chromatin_footprint_gate",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "exhaustion_comparable_gate",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "exhaustion_nondeterminant_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "exhaustion_separable_program_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "exhaustion_counterexample_gate",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "exhaustion_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "cf_signature_gut_contaminant",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_present_without_activator",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_no_block_but_absent",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_late_block_without_poised_locus",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_early_block_with_brake_released",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "activator_chromatin_footprint": "consistent_in_formers",
        "cd103_program_relations": [
          {
            "pair": "B16_GP_vs_KPC_GP",
            "relation": "~"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "mWnt_GP_vs_B_ALL_GP",
            "relation": "~"
          }
        ],
        "core_residency_signature": [
          "Itgae",
          "Cd69",
          "Runx3",
          "Batf",
          "Prdm1"
        ],
        "excluded_confounders": [
          "Pdcd1",
          "Tcf7",
          "Tox"
        ],
        "exhaustion_comparable_across_split": "yes",
        "exhaustion_counterexample_model": "mWnt_GP",
        "exhaustion_explains_split": "no",
        "exhaustion_relations": [
          {
            "pair": "B16_GP_vs_KPC_GP",
            "relation": "~"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": "~"
          },
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": "<"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": "~"
          },
          {
            "pair": "KPC_GP_vs_B_ALL_GP",
            "relation": "<"
          },
          {
            "pair": "mWnt_GP_vs_B_ALL_GP",
            "relation": "<"
          }
        ],
        "exhaustion_separable_program": "yes",
        "formation_limited_by": "absent_induction",
        "generic_signature_relations": [
          {
            "pair": "B16_GP_vs_KPC_GP",
            "relation": "~"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "mWnt_GP_vs_B_ALL_GP",
            "relation": "~"
          }
        ],
        "locus_competent_in_naive": "yes",
        "locus_differential": "model_specific_element",
        "per_model_regulatory_state": [
          {
            "activator_state": "present",
            "blocking_step": "none",
            "brake_state": "released",
            "locus_state": "active",
            "model": "B16_GP",
            "program": "present"
          },
          {
            "activator_state": "present",
            "blocking_step": "none",
            "brake_state": "released",
            "locus_state": "active",
            "model": "KPC_GP",
            "program": "present"
          },
          {
            "activator_state": "partial",
            "blocking_step": "late_activation",
            "brake_state": "partial",
            "locus_state": "poised_unfired",
            "model": "mWnt_GP",
            "program": "partial"
          },
          {
            "activator_state": "absent",
            "blocking_step": "both_steps",
            "brake_state": "engaged",
            "locus_state": "repressed",
            "model": "B_ALL_GP",
            "program": "absent"
          }
        ],
        "program_map": [
          {
            "call": "present",
            "model": "B16_GP"
          },
          {
            "call": "present",
            "model": "KPC_GP"
          },
          {
            "call": "partial",
            "model": "mWnt_GP"
          },
          {
            "call": "absent",
            "model": "B_ALL_GP"
          }
        ],
        "ready_silent_fraction_relations": [
          {
            "pair": "B16_GP_vs_KPC_GP",
            "relation": "~"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "mWnt_GP_vs_B_ALL_GP",
            "relation": "~"
          }
        ],
        "regulator_tuples": [
          {
            "evidence_basis": "target_program_behavior",
            "expr_vs_activity": "concordant",
            "gating_step": "late_activation",
            "regulator": "Runx3",
            "role": "activator"
          },
          {
            "evidence_basis": "target_program_behavior",
            "expr_vs_activity": "concordant",
            "gating_step": "late_activation",
            "regulator": "Batf",
            "role": "activator"
          },
          {
            "evidence_basis": "own_abundance",
            "expr_vs_activity": "discordant",
            "gating_step": "early_release",
            "regulator": "Prdm1",
            "role": "repressor"
          }
        ]
      },
      "nTurns": 7,
      "checkpointSignals": {
        "clustering": false,
        "markerAnnotation": false,
        "scoreBasedAnnotation": false,
        "normalization": false,
        "nerveJointAnchor": false,
        "multiomeAssembly": true,
        "depthNormalizedAtac": false,
        "regulonInference": true,
        "cd8SubsetIntegration": true,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": false,
        "orthologProjection": false,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": false,
        "mousePseudobulkDesign": false,
        "cancerEpithelialDenominator": false,
        "geneLevelDe": false,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "trajectoryFile": "scbench-long/trajectories/green_2_tme_context_dependence__grok-4.3__pi__r3.json",
      "runId": "grok-4.3__pi__green_2_tme_context_dependence__r3",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-kimi-k2p6-pi",
      "modelName": "kimi-k2p6",
      "harness": "pi",
      "task": "green_2_tme_context_dependence",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 1,
      "passed": false,
      "score": 7,
      "durationS": 3395.54,
      "cost": 1.0802412,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "program_map",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "sig_includes_itgae",
            "role": "gate",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Itgae"
            ]
          },
          {
            "name": "declared_a_known_confounder",
            "role": "gate",
            "field": "$.excluded_confounders[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "cf_signature_gut_contaminant",
            "role": "hard_fail",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "generic_signature_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd103_program_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "<"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "per_model_regulatory_state",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "poised_unfired"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "late_activation"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "closing_committed"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_tuples",
            "kind": "list",
            "matchKey": "regulator",
            "k": 1,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "role": {
                    "op": "equals",
                    "values": [
                      "repressor"
                    ]
                  },
                  "gating_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  },
                  "evidence_basis": {
                    "op": "equals",
                    "values": [
                      "target_program_behavior"
                    ]
                  },
                  "expr_vs_activity": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "locus_differential_gate",
            "role": "gate",
            "field": "locus_differential",
            "kind": "value",
            "expected": "uniform_across_models"
          },
          {
            "name": "formation_limited_by_gate",
            "role": "gate",
            "field": "formation_limited_by",
            "kind": "value",
            "expected": "absent_induction"
          },
          {
            "name": "locus_competent_in_naive_gate",
            "role": "gate",
            "field": "locus_competent_in_naive",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "ready_silent_fraction_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 3,
            "expected": [
              {
                "key": "B16_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_KPC_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "activator_chromatin_footprint_gate",
            "role": "gate",
            "field": "activator_chromatin_footprint",
            "kind": "value",
            "expected": "inconsistent_across_formers"
          },
          {
            "name": "exhaustion_comparable_gate",
            "role": "gate",
            "field": "exhaustion_comparable_across_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_nondeterminant_gate",
            "role": "gate",
            "field": "exhaustion_explains_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_separable_program_gate",
            "role": "gate",
            "field": "exhaustion_separable_program",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "exhaustion_counterexample_gate",
            "role": "gate",
            "field": "exhaustion_counterexample_model",
            "kind": "value",
            "expected": "KPC_GP"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "exhaustion_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "cf_inconsistent_present_without_activator",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "program",
            "entity": "present",
            "valueField": "activator_state",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_no_block_but_absent",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "none",
            "valueField": "program",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_late_block_without_poised_locus",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "late_activation",
            "valueField": "locus_state",
            "expected": [
              "closing_committed",
              "active"
            ]
          },
          {
            "name": "cf_inconsistent_early_block_with_brake_released",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "early_release",
            "valueField": "brake_state",
            "expected": [
              "released"
            ]
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: FAIL\n  x list_match: list_match (answer_field='program_map'): FAIL  (score=0.0)\n  + sig_includes_itgae: passed=True, score=1.0\n  x declared_a_known_confounder: passed=False, score=0.0\n  x list_match: list_match (answer_field='generic_signature_relations'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='cd103_program_relations'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='per_model_regulatory_state'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='regulator_tuples'): FAIL  (score=0.0)\n  x locus_differential_gate: passed=False, score=0.0\n  x formation_limited_by_gate: passed=False, score=0.0\n  + locus_competent_in_naive_gate: passed=True, score=1.0\n  x list_match: list_match (answer_field='ready_silent_fraction_relations'): FAIL  (score=0.0)\n  + activator_chromatin_footprint_gate: passed=True, score=1.0\n  + exhaustion_comparable_gate: passed=True, score=1.0\n  + exhaustion_nondeterminant_gate: passed=True, score=1.0\n  + exhaustion_separable_program_gate: passed=True, score=1.0\n  + exhaustion_counterexample_gate: passed=True, score=1.0\n  x list_match: list_match (answer_field='exhaustion_relations'): FAIL  (score=0.0)\n  x hard_fail 'cf_signature_gut_contaminant': TRIGGERED\n  + hard_fail 'cf_inconsistent_present_without_activator': not triggered\n  + hard_fail 'cf_inconsistent_no_block_but_absent': not triggered\n  + hard_fail 'cf_inconsistent_late_block_without_poised_locus': not triggered\n  + hard_fail 'cf_inconsistent_early_block_with_brake_released': not triggered",
      "metrics": {
        "pass_rule": "all",
        "scoring_count": 17,
        "scoring_passed": 7,
        "scoring_total_score": 7,
        "hard_fail_triggered": [
          "cf_signature_gut_contaminant"
        ]
      },
      "fieldScores": {
        "list_match": 0,
        "sig_includes_itgae": 1,
        "declared_a_known_confounder": 0,
        "locus_differential_gate": 0,
        "formation_limited_by_gate": 0,
        "locus_competent_in_naive_gate": 1,
        "activator_chromatin_footprint_gate": 1,
        "exhaustion_comparable_gate": 1,
        "exhaustion_nondeterminant_gate": 1,
        "exhaustion_separable_program_gate": 1,
        "exhaustion_counterexample_gate": 1
      },
      "graderChecks": [
        {
          "name": "program_map",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "sig_includes_itgae",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "declared_a_known_confounder",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "generic_signature_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "cd103_program_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "per_model_regulatory_state",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "regulator_tuples",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "locus_differential_gate",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "formation_limited_by_gate",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "locus_competent_in_naive_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "ready_silent_fraction_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "activator_chromatin_footprint_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "exhaustion_comparable_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "exhaustion_nondeterminant_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "exhaustion_separable_program_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "exhaustion_counterexample_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "exhaustion_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "cf_signature_gut_contaminant",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "cf_inconsistent_present_without_activator",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_no_block_but_absent",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_late_block_without_poised_locus",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_early_block_with_brake_released",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "activator_chromatin_footprint": "inconsistent_across_formers",
        "cd103_program_relations": [
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": "~"
          },
          {
            "pair": "B16_GP_vs_KPC_GP",
            "relation": "<"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": "<"
          },
          {
            "pair": "B_ALL_GP_vs_KPC_GP",
            "relation": "<"
          },
          {
            "pair": "B_ALL_GP_vs_mWnt_GP",
            "relation": "<"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": ">"
          }
        ],
        "core_residency_signature": [
          "Itgae",
          "Cd69",
          "Cxcr6",
          "Runx3",
          "Itga1",
          "Xcl1",
          "Bcl2",
          "Il7r"
        ],
        "excluded_confounders": [
          "Tox",
          "Entpd1",
          "Lag3",
          "Havcr2",
          "Pdcd1",
          "Foxp3",
          "Nt5e",
          "Gzma",
          "Gzmb",
          "Neb"
        ],
        "exhaustion_comparable_across_split": "no",
        "exhaustion_counterexample_model": "KPC_GP",
        "exhaustion_explains_split": "no",
        "exhaustion_relations": [
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": "<"
          },
          {
            "pair": "B16_GP_vs_KPC_GP",
            "relation": "~"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "B_ALL_GP_vs_KPC_GP",
            "relation": ">"
          },
          {
            "pair": "B_ALL_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": ">"
          }
        ],
        "exhaustion_separable_program": "yes",
        "formation_limited_by": "both",
        "generic_signature_relations": [
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": ">"
          },
          {
            "pair": "B16_GP_vs_KPC_GP",
            "relation": "<"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": "<"
          },
          {
            "pair": "B_ALL_GP_vs_KPC_GP",
            "relation": "<"
          },
          {
            "pair": "B_ALL_GP_vs_mWnt_GP",
            "relation": "<"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": "~"
          }
        ],
        "locus_competent_in_naive": "yes",
        "locus_differential": "model_specific_element",
        "per_model_regulatory_state": [
          {
            "activator_state": "partial",
            "blocking_step": "late_activation",
            "brake_state": "released",
            "locus_state": "repressed",
            "model": "B16_GP",
            "program": "absent"
          },
          {
            "activator_state": "present",
            "blocking_step": "both_steps",
            "brake_state": "engaged",
            "locus_state": "poised_unfired",
            "model": "B_ALL_GP",
            "program": "absent"
          },
          {
            "activator_state": "present",
            "blocking_step": "none",
            "brake_state": "released",
            "locus_state": "active",
            "model": "KPC_GP",
            "program": "present"
          },
          {
            "activator_state": "partial",
            "blocking_step": "early_release",
            "brake_state": "partial",
            "locus_state": "poised_unfired",
            "model": "mWnt_GP",
            "program": "partial"
          }
        ],
        "program_map": [
          {
            "call": "absent",
            "model": "B16_GP"
          },
          {
            "call": "absent",
            "model": "B_ALL_GP"
          },
          {
            "call": "present",
            "model": "KPC_GP"
          },
          {
            "call": "partial",
            "model": "mWnt_GP"
          }
        ],
        "ready_silent_fraction_relations": [
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": "<"
          },
          {
            "pair": "B16_GP_vs_KPC_GP",
            "relation": "~"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": "<"
          },
          {
            "pair": "B_ALL_GP_vs_KPC_GP",
            "relation": ">"
          },
          {
            "pair": "B_ALL_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": "<"
          }
        ],
        "regulator_tuples": [
          {
            "evidence_basis": "own_abundance",
            "expr_vs_activity": "indeterminate",
            "gating_step": "late_activation",
            "regulator": "Runx3",
            "role": "activator"
          },
          {
            "evidence_basis": "target_program_behavior",
            "expr_vs_activity": "concordant",
            "gating_step": "early_release",
            "regulator": "Tcf7",
            "role": "repressor"
          },
          {
            "evidence_basis": "target_program_behavior",
            "expr_vs_activity": "concordant",
            "gating_step": "early_release",
            "regulator": "Eomes",
            "role": "repressor"
          },
          {
            "evidence_basis": "own_abundance",
            "expr_vs_activity": "indeterminate",
            "gating_step": "late_activation",
            "regulator": "Prdm1",
            "role": "activator"
          }
        ]
      },
      "nTurns": 37,
      "checkpointSignals": {
        "clustering": false,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": false,
        "nerveJointAnchor": false,
        "multiomeAssembly": true,
        "depthNormalizedAtac": true,
        "regulonInference": true,
        "cd8SubsetIntegration": true,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": false,
        "orthologProjection": false,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": false,
        "mousePseudobulkDesign": false,
        "cancerEpithelialDenominator": false,
        "geneLevelDe": false,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "trajectoryFile": "scbench-long/trajectories/green_2_tme_context_dependence__kimi-k2p6__pi__r1.json",
      "runId": "kimi-k2p6__pi__green_2_tme_context_dependence__r1",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-kimi-k2p6-pi",
      "modelName": "kimi-k2p6",
      "harness": "pi",
      "task": "green_2_tme_context_dependence",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 2,
      "passed": false,
      "score": 7,
      "durationS": 3094.98,
      "cost": 0.8700316600000001,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "program_map",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "sig_includes_itgae",
            "role": "gate",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Itgae"
            ]
          },
          {
            "name": "declared_a_known_confounder",
            "role": "gate",
            "field": "$.excluded_confounders[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "cf_signature_gut_contaminant",
            "role": "hard_fail",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "generic_signature_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd103_program_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "<"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "per_model_regulatory_state",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "poised_unfired"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "late_activation"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "closing_committed"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_tuples",
            "kind": "list",
            "matchKey": "regulator",
            "k": 1,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "role": {
                    "op": "equals",
                    "values": [
                      "repressor"
                    ]
                  },
                  "gating_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  },
                  "evidence_basis": {
                    "op": "equals",
                    "values": [
                      "target_program_behavior"
                    ]
                  },
                  "expr_vs_activity": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "locus_differential_gate",
            "role": "gate",
            "field": "locus_differential",
            "kind": "value",
            "expected": "uniform_across_models"
          },
          {
            "name": "formation_limited_by_gate",
            "role": "gate",
            "field": "formation_limited_by",
            "kind": "value",
            "expected": "absent_induction"
          },
          {
            "name": "locus_competent_in_naive_gate",
            "role": "gate",
            "field": "locus_competent_in_naive",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "ready_silent_fraction_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 3,
            "expected": [
              {
                "key": "B16_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_KPC_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "activator_chromatin_footprint_gate",
            "role": "gate",
            "field": "activator_chromatin_footprint",
            "kind": "value",
            "expected": "inconsistent_across_formers"
          },
          {
            "name": "exhaustion_comparable_gate",
            "role": "gate",
            "field": "exhaustion_comparable_across_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_nondeterminant_gate",
            "role": "gate",
            "field": "exhaustion_explains_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_separable_program_gate",
            "role": "gate",
            "field": "exhaustion_separable_program",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "exhaustion_counterexample_gate",
            "role": "gate",
            "field": "exhaustion_counterexample_model",
            "kind": "value",
            "expected": "KPC_GP"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "exhaustion_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "cf_inconsistent_present_without_activator",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "program",
            "entity": "present",
            "valueField": "activator_state",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_no_block_but_absent",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "none",
            "valueField": "program",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_late_block_without_poised_locus",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "late_activation",
            "valueField": "locus_state",
            "expected": [
              "closing_committed",
              "active"
            ]
          },
          {
            "name": "cf_inconsistent_early_block_with_brake_released",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "early_release",
            "valueField": "brake_state",
            "expected": [
              "released"
            ]
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: FAIL\n  x list_match: list_match (answer_field='program_map'): FAIL  (score=0.0)\n  + sig_includes_itgae: passed=True, score=1.0\n  + declared_a_known_confounder: passed=True, score=1.0\n  x list_match: list_match (answer_field='generic_signature_relations'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='cd103_program_relations'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='per_model_regulatory_state'): FAIL  (score=0.0)\n  + list_match: list_match (answer_field='regulator_tuples'): PASS  (score=0.0)\n  x locus_differential_gate: passed=False, score=0.0\n  x formation_limited_by_gate: passed=False, score=0.0\n  + locus_competent_in_naive_gate: passed=True, score=1.0\n  x list_match: list_match (answer_field='ready_silent_fraction_relations'): FAIL  (score=0.0)\n  + activator_chromatin_footprint_gate: passed=True, score=1.0\n  x exhaustion_comparable_gate: passed=False, score=0.0\n  + exhaustion_nondeterminant_gate: passed=True, score=1.0\n  + exhaustion_separable_program_gate: passed=True, score=1.0\n  + exhaustion_counterexample_gate: passed=True, score=1.0\n  x list_match: list_match (answer_field='exhaustion_relations'): FAIL  (score=0.0)\n  x hard_fail 'cf_signature_gut_contaminant': TRIGGERED\n  + hard_fail 'cf_inconsistent_present_without_activator': not triggered\n  + hard_fail 'cf_inconsistent_no_block_but_absent': not triggered\n  + hard_fail 'cf_inconsistent_late_block_without_poised_locus': not triggered\n  + hard_fail 'cf_inconsistent_early_block_with_brake_released': not triggered",
      "metrics": {
        "pass_rule": "all",
        "scoring_count": 17,
        "scoring_passed": 8,
        "scoring_total_score": 7,
        "hard_fail_triggered": [
          "cf_signature_gut_contaminant"
        ]
      },
      "fieldScores": {
        "list_match": 0,
        "sig_includes_itgae": 1,
        "declared_a_known_confounder": 1,
        "locus_differential_gate": 0,
        "formation_limited_by_gate": 0,
        "locus_competent_in_naive_gate": 1,
        "activator_chromatin_footprint_gate": 1,
        "exhaustion_comparable_gate": 0,
        "exhaustion_nondeterminant_gate": 1,
        "exhaustion_separable_program_gate": 1,
        "exhaustion_counterexample_gate": 1
      },
      "graderChecks": [
        {
          "name": "program_map",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "sig_includes_itgae",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "declared_a_known_confounder",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "generic_signature_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "cd103_program_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "per_model_regulatory_state",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "regulator_tuples",
          "kind": "list_match",
          "passed": true,
          "score": 0
        },
        {
          "name": "locus_differential_gate",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "formation_limited_by_gate",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "locus_competent_in_naive_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "ready_silent_fraction_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "activator_chromatin_footprint_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "exhaustion_comparable_gate",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "exhaustion_nondeterminant_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "exhaustion_separable_program_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "exhaustion_counterexample_gate",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "exhaustion_relations",
          "kind": "list_match",
          "passed": false,
          "score": 0
        },
        {
          "name": "cf_signature_gut_contaminant",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "cf_inconsistent_present_without_activator",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_no_block_but_absent",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_late_block_without_poised_locus",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_inconsistent_early_block_with_brake_released",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "activator_chromatin_footprint": "inconsistent_across_formers",
        "cd103_program_relations": [
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": "~"
          },
          {
            "pair": "B16_GP_vs_KPC_GP",
            "relation": "<"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": "<"
          },
          {
            "pair": "B_ALL_GP_vs_KPC_GP",
            "relation": "<"
          },
          {
            "pair": "B_ALL_GP_vs_mWnt_GP",
            "relation": "<"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": ">"
          }
        ],
        "core_residency_signature": [
          "Itgae",
          "Itga1",
          "Chn2",
          "Abi3",
          "Cd160",
          "P2rx7",
          "Rgs1",
          "Dapk2",
          "Actn2",
          "Ldlr",
          "Ckb",
          "Qpct",
          "Gpr55",
          "Hpgds",
          "Hic1",
          "Cyp51"
        ],
        "excluded_confounders": [
          "Gzma",
          "Gzmk",
          "Ccl5",
          "Cd52",
          "Tmsb4x",
          "Actb",
          "Hist1h2ap",
          "Hist1h4d",
          "Uty",
          "Ddx3y",
          "Eif2s3y",
          "Plac8",
          "Atp8a2",
          "Fdps",
          "Gngt2",
          "Slc16a10",
          "Cdh1",
          "AY036118",
          "Dtx1"
        ],
        "exhaustion_comparable_across_split": "yes",
        "exhaustion_counterexample_model": "KPC_GP",
        "exhaustion_explains_split": "no",
        "exhaustion_relations": [
          {
            "pair": "KPC_GP_vs_B_ALL_GP",
            "relation": "~"
          },
          {
            "pair": "KPC_GP_vs_B16_GP",
            "relation": ">"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "B_ALL_GP_vs_B16_GP",
            "relation": "~"
          },
          {
            "pair": "B_ALL_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": ">"
          }
        ],
        "exhaustion_separable_program": "yes",
        "formation_limited_by": "both",
        "generic_signature_relations": [
          {
            "pair": "B16_GP_vs_B_ALL_GP",
            "relation": "~"
          },
          {
            "pair": "B16_GP_vs_KPC_GP",
            "relation": "<"
          },
          {
            "pair": "B16_GP_vs_mWnt_GP",
            "relation": "<"
          },
          {
            "pair": "B_ALL_GP_vs_KPC_GP",
            "relation": "<"
          },
          {
            "pair": "B_ALL_GP_vs_mWnt_GP",
            "relation": "<"
          },
          {
            "pair": "KPC_GP_vs_mWnt_GP",
            "relation": ">"
          }
        ],
        "locus_competent_in_naive": "yes",
        "locus_differential": "model_specific_element",
        "per_model_regulatory_state": [
          {
            "activator_state": "present",
            "blocking_step": "signal_independent",
            "brake_state": "released",
            "locus_state": "repressed",
            "model": "B16_GP",
            "program": "absent"
          },
          {
            "activator_state": "present",
            "blocking_step": "both_steps",
            "brake_state": "partial",
            "locus_state": "poised_unfired",
            "model": "B_ALL_GP",
            "program": "absent"
          },
          {
            "activator_state": "present",
            "blocking_step": "none",
            "brake_state": "released",
            "locus_state": "active",
            "model": "KPC_GP",
            "program": "present"
          },
          {
            "activator_state": "present",
            "blocking_step": "early_release",
            "brake_state": "partial",
            "locus_state": "active",
            "model": "mWnt_GP",
            "program": "partial"
          }
        ],
        "program_map": [
          {
            "call": "absent",
            "model": "B16_GP"
          },
          {
            "call": "absent",
            "model": "B_ALL_GP"
          },
          {
            "call": "present",
            "model": "KPC_GP"
          },
          {
            "call": "partial",
            "model": "mWnt_GP"
          }
        ],
        "ready_silent_fraction_relations": [
          {
            "pair": "B_ALL_GP_vs_mWnt_GP",
            "relation": ">"
          },
          {
            "pair": "B_ALL_GP_vs_B16_GP",
            "relation": ">"
          },
          {
            "pair": "B_ALL_GP_vs_KPC_GP",
            "relation": ">"
          },
          {
            "pair": "mWnt_GP_vs_B16_GP",
            "relation": ">"
          },
          {
            "pair": "mWnt_GP_vs_KPC_GP",
            "relation": ">"
          },
          {
            "pair": "B16_GP_vs_KPC_GP",
            "relation": ">"
          }
        ],
        "regulator_tuples": [
          {
            "evidence_basis": "target_program_behavior",
            "expr_vs_activity": "concordant",
            "gating_step": "early_release",
            "regulator": "Klf2",
            "role": "repressor"
          },
          {
            "evidence_basis": "target_program_behavior",
            "expr_vs_activity": "concordant",
            "gating_step": "early_release",
            "regulator": "S1pr1",
            "role": "repressor"
          },
          {
            "evidence_basis": "target_program_behavior",
            "expr_vs_activity": "concordant",
            "gating_step": "early_release",
            "regulator": "Tcf7",
            "role": "repressor"
          },
          {
            "evidence_basis": "target_program_behavior",
            "expr_vs_activity": "discordant",
            "gating_step": "none",
            "regulator": "Runx3",
            "role": "activator"
          },
          {
            "evidence_basis": "target_program_behavior",
            "expr_vs_activity": "discordant",
            "gating_step": "late_activation",
            "regulator": "Tox",
            "role": "repressor"
          }
        ]
      },
      "nTurns": 38,
      "checkpointSignals": {
        "clustering": false,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": false,
        "nerveJointAnchor": false,
        "multiomeAssembly": true,
        "depthNormalizedAtac": true,
        "regulonInference": true,
        "cd8SubsetIntegration": true,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": false,
        "orthologProjection": false,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": true,
        "mousePseudobulkDesign": false,
        "cancerEpithelialDenominator": false,
        "geneLevelDe": false,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "trajectoryFile": "scbench-long/trajectories/green_2_tme_context_dependence__kimi-k2p6__pi__r2.json",
      "runId": "kimi-k2p6__pi__green_2_tme_context_dependence__r2",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-opus-4-6-claude-code",
      "modelName": "claude-opus-4-6",
      "harness": "claude-code",
      "task": "green_klf2_hidden_repressor_of_trm_like_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 2795,
      "cost": 3.22854675,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "regulator_gate",
            "role": "gate",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "KLF2",
              "Klf2"
            ]
          },
          {
            "name": "controlling_node_character_gate",
            "role": "gate",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "high_differential_expression"
          },
          {
            "name": "expression_vs_activity_gate",
            "role": "gate",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "expression_exceeds_activity"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "target_program_partition",
            "kind": "list",
            "matchKey": "gene",
            "k": 7,
            "expected": [
              {
                "key": "S1pr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "S1pr4",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Klrg1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Cx3cr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Itgae",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Hic1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "P2rx7",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              }
            ]
          },
          {
            "name": "causal_status_gate",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "association_only_sufficiency_requires_perturbation"
          },
          {
            "name": "residency_control_logic_gate",
            "role": "gate",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_release_necessary_not_sufficient_multi_regulon"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "trm_like_program_by_model",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_absent"
                    ],
                    "canonical": "niche_absent"
                  }
                }
              }
            ]
          },
          {
            "name": "cf_regulator_activator_or_wrong_node",
            "role": "hard_fail",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "RUNX3",
              "Runx3",
              "PRDM1",
              "Prdm1",
              "BLIMP1",
              "Blimp1",
              "BHLHE40",
              "Bhlhe40",
              "ZNF683",
              "Zfp683",
              "HOBIT",
              "Hobit",
              "AHR",
              "Ahr",
              "BATF",
              "Batf",
              "TOX",
              "Tox",
              "NR4A1",
              "Nr4a1",
              "NR4A2",
              "Nr4a2",
              "NR4A3",
              "Nr4a3",
              "TCF7",
              "Tcf7",
              "TCF1",
              "Tcf1",
              "BACH2",
              "Bach2",
              "TBX21",
              "Tbx21",
              "TBET",
              "Tbet",
              "FLI1",
              "Fli1",
              "NR3C1",
              "Nr3c1",
              "EOMES",
              "Eomes",
              "ZEB2",
              "Zeb2",
              "ID2",
              "Id2",
              "ID3",
              "Id3",
              "IRF4",
              "Irf4"
            ]
          },
          {
            "name": "cf_node_character_hidden_driver_recall",
            "role": "hard_fail",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "hidden_driver_activity_not_expression"
          },
          {
            "name": "cf_effect_relation_hidden_pattern",
            "role": "hard_fail",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "activity_exceeds_expression"
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "causal_status",
            "kind": "value",
            "op": "in",
            "expected": [
              "sufficiency_established",
              "necessity_established",
              "causal_established_in_dataset",
              "sufficiency_via_perturbation_in_dataset"
            ]
          },
          {
            "name": "cf_residency_logic_activator_prior",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "residency_activator_induction_primary"
          },
          {
            "name": "cf_residency_logic_single_regulon",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_controls_entire_residency_program"
          },
          {
            "name": "cf_decoy_called_target",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_distractor_model_reported",
            "role": "hard_fail",
            "field": "$.trm_like_program_by_model[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_chromatin_only_decoy",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-6-claude-code",
      "modelName": "claude-opus-4-6",
      "harness": "claude-code",
      "task": "green_klf2_hidden_repressor_of_trm_like_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 3724,
      "cost": 4.16974075,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "regulator_gate",
            "role": "gate",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "KLF2",
              "Klf2"
            ]
          },
          {
            "name": "controlling_node_character_gate",
            "role": "gate",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "high_differential_expression"
          },
          {
            "name": "expression_vs_activity_gate",
            "role": "gate",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "expression_exceeds_activity"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "target_program_partition",
            "kind": "list",
            "matchKey": "gene",
            "k": 7,
            "expected": [
              {
                "key": "S1pr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "S1pr4",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Klrg1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Cx3cr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Itgae",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Hic1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "P2rx7",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              }
            ]
          },
          {
            "name": "causal_status_gate",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "association_only_sufficiency_requires_perturbation"
          },
          {
            "name": "residency_control_logic_gate",
            "role": "gate",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_release_necessary_not_sufficient_multi_regulon"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "trm_like_program_by_model",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_absent"
                    ],
                    "canonical": "niche_absent"
                  }
                }
              }
            ]
          },
          {
            "name": "cf_regulator_activator_or_wrong_node",
            "role": "hard_fail",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "RUNX3",
              "Runx3",
              "PRDM1",
              "Prdm1",
              "BLIMP1",
              "Blimp1",
              "BHLHE40",
              "Bhlhe40",
              "ZNF683",
              "Zfp683",
              "HOBIT",
              "Hobit",
              "AHR",
              "Ahr",
              "BATF",
              "Batf",
              "TOX",
              "Tox",
              "NR4A1",
              "Nr4a1",
              "NR4A2",
              "Nr4a2",
              "NR4A3",
              "Nr4a3",
              "TCF7",
              "Tcf7",
              "TCF1",
              "Tcf1",
              "BACH2",
              "Bach2",
              "TBX21",
              "Tbx21",
              "TBET",
              "Tbet",
              "FLI1",
              "Fli1",
              "NR3C1",
              "Nr3c1",
              "EOMES",
              "Eomes",
              "ZEB2",
              "Zeb2",
              "ID2",
              "Id2",
              "ID3",
              "Id3",
              "IRF4",
              "Irf4"
            ]
          },
          {
            "name": "cf_node_character_hidden_driver_recall",
            "role": "hard_fail",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "hidden_driver_activity_not_expression"
          },
          {
            "name": "cf_effect_relation_hidden_pattern",
            "role": "hard_fail",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "activity_exceeds_expression"
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "causal_status",
            "kind": "value",
            "op": "in",
            "expected": [
              "sufficiency_established",
              "necessity_established",
              "causal_established_in_dataset",
              "sufficiency_via_perturbation_in_dataset"
            ]
          },
          {
            "name": "cf_residency_logic_activator_prior",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "residency_activator_induction_primary"
          },
          {
            "name": "cf_residency_logic_single_regulon",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_controls_entire_residency_program"
          },
          {
            "name": "cf_decoy_called_target",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_distractor_model_reported",
            "role": "hard_fail",
            "field": "$.trm_like_program_by_model[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_chromatin_only_decoy",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-6-claude-code",
      "modelName": "claude-opus-4-6",
      "harness": "claude-code",
      "task": "green_klf2_hidden_repressor_of_trm_like_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 3837,
      "cost": 3.5853302500000006,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "regulator_gate",
            "role": "gate",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "KLF2",
              "Klf2"
            ]
          },
          {
            "name": "controlling_node_character_gate",
            "role": "gate",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "high_differential_expression"
          },
          {
            "name": "expression_vs_activity_gate",
            "role": "gate",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "expression_exceeds_activity"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "target_program_partition",
            "kind": "list",
            "matchKey": "gene",
            "k": 7,
            "expected": [
              {
                "key": "S1pr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "S1pr4",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Klrg1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Cx3cr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Itgae",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Hic1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "P2rx7",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              }
            ]
          },
          {
            "name": "causal_status_gate",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "association_only_sufficiency_requires_perturbation"
          },
          {
            "name": "residency_control_logic_gate",
            "role": "gate",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_release_necessary_not_sufficient_multi_regulon"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "trm_like_program_by_model",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_absent"
                    ],
                    "canonical": "niche_absent"
                  }
                }
              }
            ]
          },
          {
            "name": "cf_regulator_activator_or_wrong_node",
            "role": "hard_fail",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "RUNX3",
              "Runx3",
              "PRDM1",
              "Prdm1",
              "BLIMP1",
              "Blimp1",
              "BHLHE40",
              "Bhlhe40",
              "ZNF683",
              "Zfp683",
              "HOBIT",
              "Hobit",
              "AHR",
              "Ahr",
              "BATF",
              "Batf",
              "TOX",
              "Tox",
              "NR4A1",
              "Nr4a1",
              "NR4A2",
              "Nr4a2",
              "NR4A3",
              "Nr4a3",
              "TCF7",
              "Tcf7",
              "TCF1",
              "Tcf1",
              "BACH2",
              "Bach2",
              "TBX21",
              "Tbx21",
              "TBET",
              "Tbet",
              "FLI1",
              "Fli1",
              "NR3C1",
              "Nr3c1",
              "EOMES",
              "Eomes",
              "ZEB2",
              "Zeb2",
              "ID2",
              "Id2",
              "ID3",
              "Id3",
              "IRF4",
              "Irf4"
            ]
          },
          {
            "name": "cf_node_character_hidden_driver_recall",
            "role": "hard_fail",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "hidden_driver_activity_not_expression"
          },
          {
            "name": "cf_effect_relation_hidden_pattern",
            "role": "hard_fail",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "activity_exceeds_expression"
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "causal_status",
            "kind": "value",
            "op": "in",
            "expected": [
              "sufficiency_established",
              "necessity_established",
              "causal_established_in_dataset",
              "sufficiency_via_perturbation_in_dataset"
            ]
          },
          {
            "name": "cf_residency_logic_activator_prior",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "residency_activator_induction_primary"
          },
          {
            "name": "cf_residency_logic_single_regulon",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_controls_entire_residency_program"
          },
          {
            "name": "cf_decoy_called_target",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_distractor_model_reported",
            "role": "hard_fail",
            "field": "$.trm_like_program_by_model[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_chromatin_only_decoy",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-6-pi",
      "modelName": "claude-opus-4-6",
      "harness": "pi",
      "task": "green_klf2_hidden_repressor_of_trm_like_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 3054,
      "cost": 3.14237725,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "regulator_gate",
            "role": "gate",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "KLF2",
              "Klf2"
            ]
          },
          {
            "name": "controlling_node_character_gate",
            "role": "gate",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "high_differential_expression"
          },
          {
            "name": "expression_vs_activity_gate",
            "role": "gate",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "expression_exceeds_activity"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "target_program_partition",
            "kind": "list",
            "matchKey": "gene",
            "k": 7,
            "expected": [
              {
                "key": "S1pr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "S1pr4",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Klrg1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Cx3cr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Itgae",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Hic1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "P2rx7",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              }
            ]
          },
          {
            "name": "causal_status_gate",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "association_only_sufficiency_requires_perturbation"
          },
          {
            "name": "residency_control_logic_gate",
            "role": "gate",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_release_necessary_not_sufficient_multi_regulon"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "trm_like_program_by_model",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_absent"
                    ],
                    "canonical": "niche_absent"
                  }
                }
              }
            ]
          },
          {
            "name": "cf_regulator_activator_or_wrong_node",
            "role": "hard_fail",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "RUNX3",
              "Runx3",
              "PRDM1",
              "Prdm1",
              "BLIMP1",
              "Blimp1",
              "BHLHE40",
              "Bhlhe40",
              "ZNF683",
              "Zfp683",
              "HOBIT",
              "Hobit",
              "AHR",
              "Ahr",
              "BATF",
              "Batf",
              "TOX",
              "Tox",
              "NR4A1",
              "Nr4a1",
              "NR4A2",
              "Nr4a2",
              "NR4A3",
              "Nr4a3",
              "TCF7",
              "Tcf7",
              "TCF1",
              "Tcf1",
              "BACH2",
              "Bach2",
              "TBX21",
              "Tbx21",
              "TBET",
              "Tbet",
              "FLI1",
              "Fli1",
              "NR3C1",
              "Nr3c1",
              "EOMES",
              "Eomes",
              "ZEB2",
              "Zeb2",
              "ID2",
              "Id2",
              "ID3",
              "Id3",
              "IRF4",
              "Irf4"
            ]
          },
          {
            "name": "cf_node_character_hidden_driver_recall",
            "role": "hard_fail",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "hidden_driver_activity_not_expression"
          },
          {
            "name": "cf_effect_relation_hidden_pattern",
            "role": "hard_fail",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "activity_exceeds_expression"
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "causal_status",
            "kind": "value",
            "op": "in",
            "expected": [
              "sufficiency_established",
              "necessity_established",
              "causal_established_in_dataset",
              "sufficiency_via_perturbation_in_dataset"
            ]
          },
          {
            "name": "cf_residency_logic_activator_prior",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "residency_activator_induction_primary"
          },
          {
            "name": "cf_residency_logic_single_regulon",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_controls_entire_residency_program"
          },
          {
            "name": "cf_decoy_called_target",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_distractor_model_reported",
            "role": "hard_fail",
            "field": "$.trm_like_program_by_model[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_chromatin_only_decoy",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-6-pi",
      "modelName": "claude-opus-4-6",
      "harness": "pi",
      "task": "green_klf2_hidden_repressor_of_trm_like_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 4479,
      "cost": 3.6645755000000007,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "regulator_gate",
            "role": "gate",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "KLF2",
              "Klf2"
            ]
          },
          {
            "name": "controlling_node_character_gate",
            "role": "gate",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "high_differential_expression"
          },
          {
            "name": "expression_vs_activity_gate",
            "role": "gate",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "expression_exceeds_activity"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "target_program_partition",
            "kind": "list",
            "matchKey": "gene",
            "k": 7,
            "expected": [
              {
                "key": "S1pr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "S1pr4",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Klrg1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Cx3cr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Itgae",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Hic1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "P2rx7",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              }
            ]
          },
          {
            "name": "causal_status_gate",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "association_only_sufficiency_requires_perturbation"
          },
          {
            "name": "residency_control_logic_gate",
            "role": "gate",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_release_necessary_not_sufficient_multi_regulon"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "trm_like_program_by_model",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_absent"
                    ],
                    "canonical": "niche_absent"
                  }
                }
              }
            ]
          },
          {
            "name": "cf_regulator_activator_or_wrong_node",
            "role": "hard_fail",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "RUNX3",
              "Runx3",
              "PRDM1",
              "Prdm1",
              "BLIMP1",
              "Blimp1",
              "BHLHE40",
              "Bhlhe40",
              "ZNF683",
              "Zfp683",
              "HOBIT",
              "Hobit",
              "AHR",
              "Ahr",
              "BATF",
              "Batf",
              "TOX",
              "Tox",
              "NR4A1",
              "Nr4a1",
              "NR4A2",
              "Nr4a2",
              "NR4A3",
              "Nr4a3",
              "TCF7",
              "Tcf7",
              "TCF1",
              "Tcf1",
              "BACH2",
              "Bach2",
              "TBX21",
              "Tbx21",
              "TBET",
              "Tbet",
              "FLI1",
              "Fli1",
              "NR3C1",
              "Nr3c1",
              "EOMES",
              "Eomes",
              "ZEB2",
              "Zeb2",
              "ID2",
              "Id2",
              "ID3",
              "Id3",
              "IRF4",
              "Irf4"
            ]
          },
          {
            "name": "cf_node_character_hidden_driver_recall",
            "role": "hard_fail",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "hidden_driver_activity_not_expression"
          },
          {
            "name": "cf_effect_relation_hidden_pattern",
            "role": "hard_fail",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "activity_exceeds_expression"
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "causal_status",
            "kind": "value",
            "op": "in",
            "expected": [
              "sufficiency_established",
              "necessity_established",
              "causal_established_in_dataset",
              "sufficiency_via_perturbation_in_dataset"
            ]
          },
          {
            "name": "cf_residency_logic_activator_prior",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "residency_activator_induction_primary"
          },
          {
            "name": "cf_residency_logic_single_regulon",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_controls_entire_residency_program"
          },
          {
            "name": "cf_decoy_called_target",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_distractor_model_reported",
            "role": "hard_fail",
            "field": "$.trm_like_program_by_model[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_chromatin_only_decoy",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-6-pi",
      "modelName": "claude-opus-4-6",
      "harness": "pi",
      "task": "green_klf2_hidden_repressor_of_trm_like_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 6996,
      "cost": 3.4911429999999997,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "regulator_gate",
            "role": "gate",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "KLF2",
              "Klf2"
            ]
          },
          {
            "name": "controlling_node_character_gate",
            "role": "gate",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "high_differential_expression"
          },
          {
            "name": "expression_vs_activity_gate",
            "role": "gate",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "expression_exceeds_activity"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "target_program_partition",
            "kind": "list",
            "matchKey": "gene",
            "k": 7,
            "expected": [
              {
                "key": "S1pr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "S1pr4",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Klrg1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Cx3cr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Itgae",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Hic1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "P2rx7",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              }
            ]
          },
          {
            "name": "causal_status_gate",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "association_only_sufficiency_requires_perturbation"
          },
          {
            "name": "residency_control_logic_gate",
            "role": "gate",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_release_necessary_not_sufficient_multi_regulon"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "trm_like_program_by_model",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_absent"
                    ],
                    "canonical": "niche_absent"
                  }
                }
              }
            ]
          },
          {
            "name": "cf_regulator_activator_or_wrong_node",
            "role": "hard_fail",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "RUNX3",
              "Runx3",
              "PRDM1",
              "Prdm1",
              "BLIMP1",
              "Blimp1",
              "BHLHE40",
              "Bhlhe40",
              "ZNF683",
              "Zfp683",
              "HOBIT",
              "Hobit",
              "AHR",
              "Ahr",
              "BATF",
              "Batf",
              "TOX",
              "Tox",
              "NR4A1",
              "Nr4a1",
              "NR4A2",
              "Nr4a2",
              "NR4A3",
              "Nr4a3",
              "TCF7",
              "Tcf7",
              "TCF1",
              "Tcf1",
              "BACH2",
              "Bach2",
              "TBX21",
              "Tbx21",
              "TBET",
              "Tbet",
              "FLI1",
              "Fli1",
              "NR3C1",
              "Nr3c1",
              "EOMES",
              "Eomes",
              "ZEB2",
              "Zeb2",
              "ID2",
              "Id2",
              "ID3",
              "Id3",
              "IRF4",
              "Irf4"
            ]
          },
          {
            "name": "cf_node_character_hidden_driver_recall",
            "role": "hard_fail",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "hidden_driver_activity_not_expression"
          },
          {
            "name": "cf_effect_relation_hidden_pattern",
            "role": "hard_fail",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "activity_exceeds_expression"
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "causal_status",
            "kind": "value",
            "op": "in",
            "expected": [
              "sufficiency_established",
              "necessity_established",
              "causal_established_in_dataset",
              "sufficiency_via_perturbation_in_dataset"
            ]
          },
          {
            "name": "cf_residency_logic_activator_prior",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "residency_activator_induction_primary"
          },
          {
            "name": "cf_residency_logic_single_regulon",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_controls_entire_residency_program"
          },
          {
            "name": "cf_decoy_called_target",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_distractor_model_reported",
            "role": "hard_fail",
            "field": "$.trm_like_program_by_model[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_chromatin_only_decoy",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-7-claude-code",
      "modelName": "claude-opus-4-7",
      "harness": "claude-code",
      "task": "green_klf2_hidden_repressor_of_trm_like_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 1715,
      "cost": 5.4703895,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "regulator_gate",
            "role": "gate",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "KLF2",
              "Klf2"
            ]
          },
          {
            "name": "controlling_node_character_gate",
            "role": "gate",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "high_differential_expression"
          },
          {
            "name": "expression_vs_activity_gate",
            "role": "gate",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "expression_exceeds_activity"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "target_program_partition",
            "kind": "list",
            "matchKey": "gene",
            "k": 7,
            "expected": [
              {
                "key": "S1pr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "S1pr4",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Klrg1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Cx3cr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Itgae",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Hic1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "P2rx7",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              }
            ]
          },
          {
            "name": "causal_status_gate",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "association_only_sufficiency_requires_perturbation"
          },
          {
            "name": "residency_control_logic_gate",
            "role": "gate",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_release_necessary_not_sufficient_multi_regulon"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "trm_like_program_by_model",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_absent"
                    ],
                    "canonical": "niche_absent"
                  }
                }
              }
            ]
          },
          {
            "name": "cf_regulator_activator_or_wrong_node",
            "role": "hard_fail",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "RUNX3",
              "Runx3",
              "PRDM1",
              "Prdm1",
              "BLIMP1",
              "Blimp1",
              "BHLHE40",
              "Bhlhe40",
              "ZNF683",
              "Zfp683",
              "HOBIT",
              "Hobit",
              "AHR",
              "Ahr",
              "BATF",
              "Batf",
              "TOX",
              "Tox",
              "NR4A1",
              "Nr4a1",
              "NR4A2",
              "Nr4a2",
              "NR4A3",
              "Nr4a3",
              "TCF7",
              "Tcf7",
              "TCF1",
              "Tcf1",
              "BACH2",
              "Bach2",
              "TBX21",
              "Tbx21",
              "TBET",
              "Tbet",
              "FLI1",
              "Fli1",
              "NR3C1",
              "Nr3c1",
              "EOMES",
              "Eomes",
              "ZEB2",
              "Zeb2",
              "ID2",
              "Id2",
              "ID3",
              "Id3",
              "IRF4",
              "Irf4"
            ]
          },
          {
            "name": "cf_node_character_hidden_driver_recall",
            "role": "hard_fail",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "hidden_driver_activity_not_expression"
          },
          {
            "name": "cf_effect_relation_hidden_pattern",
            "role": "hard_fail",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "activity_exceeds_expression"
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "causal_status",
            "kind": "value",
            "op": "in",
            "expected": [
              "sufficiency_established",
              "necessity_established",
              "causal_established_in_dataset",
              "sufficiency_via_perturbation_in_dataset"
            ]
          },
          {
            "name": "cf_residency_logic_activator_prior",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "residency_activator_induction_primary"
          },
          {
            "name": "cf_residency_logic_single_regulon",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_controls_entire_residency_program"
          },
          {
            "name": "cf_decoy_called_target",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_distractor_model_reported",
            "role": "hard_fail",
            "field": "$.trm_like_program_by_model[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_chromatin_only_decoy",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-7-claude-code",
      "modelName": "claude-opus-4-7",
      "harness": "claude-code",
      "task": "green_klf2_hidden_repressor_of_trm_like_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 1316,
      "cost": 4.307014249999999,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "regulator_gate",
            "role": "gate",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "KLF2",
              "Klf2"
            ]
          },
          {
            "name": "controlling_node_character_gate",
            "role": "gate",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "high_differential_expression"
          },
          {
            "name": "expression_vs_activity_gate",
            "role": "gate",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "expression_exceeds_activity"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "target_program_partition",
            "kind": "list",
            "matchKey": "gene",
            "k": 7,
            "expected": [
              {
                "key": "S1pr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "S1pr4",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Klrg1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Cx3cr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Itgae",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Hic1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "P2rx7",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              }
            ]
          },
          {
            "name": "causal_status_gate",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "association_only_sufficiency_requires_perturbation"
          },
          {
            "name": "residency_control_logic_gate",
            "role": "gate",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_release_necessary_not_sufficient_multi_regulon"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "trm_like_program_by_model",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_absent"
                    ],
                    "canonical": "niche_absent"
                  }
                }
              }
            ]
          },
          {
            "name": "cf_regulator_activator_or_wrong_node",
            "role": "hard_fail",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "RUNX3",
              "Runx3",
              "PRDM1",
              "Prdm1",
              "BLIMP1",
              "Blimp1",
              "BHLHE40",
              "Bhlhe40",
              "ZNF683",
              "Zfp683",
              "HOBIT",
              "Hobit",
              "AHR",
              "Ahr",
              "BATF",
              "Batf",
              "TOX",
              "Tox",
              "NR4A1",
              "Nr4a1",
              "NR4A2",
              "Nr4a2",
              "NR4A3",
              "Nr4a3",
              "TCF7",
              "Tcf7",
              "TCF1",
              "Tcf1",
              "BACH2",
              "Bach2",
              "TBX21",
              "Tbx21",
              "TBET",
              "Tbet",
              "FLI1",
              "Fli1",
              "NR3C1",
              "Nr3c1",
              "EOMES",
              "Eomes",
              "ZEB2",
              "Zeb2",
              "ID2",
              "Id2",
              "ID3",
              "Id3",
              "IRF4",
              "Irf4"
            ]
          },
          {
            "name": "cf_node_character_hidden_driver_recall",
            "role": "hard_fail",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "hidden_driver_activity_not_expression"
          },
          {
            "name": "cf_effect_relation_hidden_pattern",
            "role": "hard_fail",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "activity_exceeds_expression"
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "causal_status",
            "kind": "value",
            "op": "in",
            "expected": [
              "sufficiency_established",
              "necessity_established",
              "causal_established_in_dataset",
              "sufficiency_via_perturbation_in_dataset"
            ]
          },
          {
            "name": "cf_residency_logic_activator_prior",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "residency_activator_induction_primary"
          },
          {
            "name": "cf_residency_logic_single_regulon",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_controls_entire_residency_program"
          },
          {
            "name": "cf_decoy_called_target",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_distractor_model_reported",
            "role": "hard_fail",
            "field": "$.trm_like_program_by_model[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_chromatin_only_decoy",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-7-claude-code",
      "modelName": "claude-opus-4-7",
      "harness": "claude-code",
      "task": "green_klf2_hidden_repressor_of_trm_like_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1872,
      "cost": 4.3661277499999995,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "regulator_gate",
            "role": "gate",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "KLF2",
              "Klf2"
            ]
          },
          {
            "name": "controlling_node_character_gate",
            "role": "gate",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "high_differential_expression"
          },
          {
            "name": "expression_vs_activity_gate",
            "role": "gate",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "expression_exceeds_activity"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "target_program_partition",
            "kind": "list",
            "matchKey": "gene",
            "k": 7,
            "expected": [
              {
                "key": "S1pr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "S1pr4",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Klrg1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Cx3cr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Itgae",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Hic1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "P2rx7",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              }
            ]
          },
          {
            "name": "causal_status_gate",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "association_only_sufficiency_requires_perturbation"
          },
          {
            "name": "residency_control_logic_gate",
            "role": "gate",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_release_necessary_not_sufficient_multi_regulon"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "trm_like_program_by_model",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_absent"
                    ],
                    "canonical": "niche_absent"
                  }
                }
              }
            ]
          },
          {
            "name": "cf_regulator_activator_or_wrong_node",
            "role": "hard_fail",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "RUNX3",
              "Runx3",
              "PRDM1",
              "Prdm1",
              "BLIMP1",
              "Blimp1",
              "BHLHE40",
              "Bhlhe40",
              "ZNF683",
              "Zfp683",
              "HOBIT",
              "Hobit",
              "AHR",
              "Ahr",
              "BATF",
              "Batf",
              "TOX",
              "Tox",
              "NR4A1",
              "Nr4a1",
              "NR4A2",
              "Nr4a2",
              "NR4A3",
              "Nr4a3",
              "TCF7",
              "Tcf7",
              "TCF1",
              "Tcf1",
              "BACH2",
              "Bach2",
              "TBX21",
              "Tbx21",
              "TBET",
              "Tbet",
              "FLI1",
              "Fli1",
              "NR3C1",
              "Nr3c1",
              "EOMES",
              "Eomes",
              "ZEB2",
              "Zeb2",
              "ID2",
              "Id2",
              "ID3",
              "Id3",
              "IRF4",
              "Irf4"
            ]
          },
          {
            "name": "cf_node_character_hidden_driver_recall",
            "role": "hard_fail",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "hidden_driver_activity_not_expression"
          },
          {
            "name": "cf_effect_relation_hidden_pattern",
            "role": "hard_fail",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "activity_exceeds_expression"
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "causal_status",
            "kind": "value",
            "op": "in",
            "expected": [
              "sufficiency_established",
              "necessity_established",
              "causal_established_in_dataset",
              "sufficiency_via_perturbation_in_dataset"
            ]
          },
          {
            "name": "cf_residency_logic_activator_prior",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "residency_activator_induction_primary"
          },
          {
            "name": "cf_residency_logic_single_regulon",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_controls_entire_residency_program"
          },
          {
            "name": "cf_decoy_called_target",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_distractor_model_reported",
            "role": "hard_fail",
            "field": "$.trm_like_program_by_model[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_chromatin_only_decoy",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-7-pi",
      "modelName": "claude-opus-4-7",
      "harness": "pi",
      "task": "green_klf2_hidden_repressor_of_trm_like_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 2761,
      "cost": 3.6348014999999996,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "regulator_gate",
            "role": "gate",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "KLF2",
              "Klf2"
            ]
          },
          {
            "name": "controlling_node_character_gate",
            "role": "gate",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "high_differential_expression"
          },
          {
            "name": "expression_vs_activity_gate",
            "role": "gate",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "expression_exceeds_activity"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "target_program_partition",
            "kind": "list",
            "matchKey": "gene",
            "k": 7,
            "expected": [
              {
                "key": "S1pr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "S1pr4",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Klrg1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Cx3cr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Itgae",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Hic1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "P2rx7",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              }
            ]
          },
          {
            "name": "causal_status_gate",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "association_only_sufficiency_requires_perturbation"
          },
          {
            "name": "residency_control_logic_gate",
            "role": "gate",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_release_necessary_not_sufficient_multi_regulon"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "trm_like_program_by_model",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_absent"
                    ],
                    "canonical": "niche_absent"
                  }
                }
              }
            ]
          },
          {
            "name": "cf_regulator_activator_or_wrong_node",
            "role": "hard_fail",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "RUNX3",
              "Runx3",
              "PRDM1",
              "Prdm1",
              "BLIMP1",
              "Blimp1",
              "BHLHE40",
              "Bhlhe40",
              "ZNF683",
              "Zfp683",
              "HOBIT",
              "Hobit",
              "AHR",
              "Ahr",
              "BATF",
              "Batf",
              "TOX",
              "Tox",
              "NR4A1",
              "Nr4a1",
              "NR4A2",
              "Nr4a2",
              "NR4A3",
              "Nr4a3",
              "TCF7",
              "Tcf7",
              "TCF1",
              "Tcf1",
              "BACH2",
              "Bach2",
              "TBX21",
              "Tbx21",
              "TBET",
              "Tbet",
              "FLI1",
              "Fli1",
              "NR3C1",
              "Nr3c1",
              "EOMES",
              "Eomes",
              "ZEB2",
              "Zeb2",
              "ID2",
              "Id2",
              "ID3",
              "Id3",
              "IRF4",
              "Irf4"
            ]
          },
          {
            "name": "cf_node_character_hidden_driver_recall",
            "role": "hard_fail",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "hidden_driver_activity_not_expression"
          },
          {
            "name": "cf_effect_relation_hidden_pattern",
            "role": "hard_fail",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "activity_exceeds_expression"
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "causal_status",
            "kind": "value",
            "op": "in",
            "expected": [
              "sufficiency_established",
              "necessity_established",
              "causal_established_in_dataset",
              "sufficiency_via_perturbation_in_dataset"
            ]
          },
          {
            "name": "cf_residency_logic_activator_prior",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "residency_activator_induction_primary"
          },
          {
            "name": "cf_residency_logic_single_regulon",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_controls_entire_residency_program"
          },
          {
            "name": "cf_decoy_called_target",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_distractor_model_reported",
            "role": "hard_fail",
            "field": "$.trm_like_program_by_model[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_chromatin_only_decoy",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-7-pi",
      "modelName": "claude-opus-4-7",
      "harness": "pi",
      "task": "green_klf2_hidden_repressor_of_trm_like_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 2377,
      "cost": 2.9198619999999997,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "regulator_gate",
            "role": "gate",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "KLF2",
              "Klf2"
            ]
          },
          {
            "name": "controlling_node_character_gate",
            "role": "gate",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "high_differential_expression"
          },
          {
            "name": "expression_vs_activity_gate",
            "role": "gate",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "expression_exceeds_activity"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "target_program_partition",
            "kind": "list",
            "matchKey": "gene",
            "k": 7,
            "expected": [
              {
                "key": "S1pr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "S1pr4",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Klrg1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Cx3cr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Itgae",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Hic1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "P2rx7",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              }
            ]
          },
          {
            "name": "causal_status_gate",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "association_only_sufficiency_requires_perturbation"
          },
          {
            "name": "residency_control_logic_gate",
            "role": "gate",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_release_necessary_not_sufficient_multi_regulon"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "trm_like_program_by_model",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_absent"
                    ],
                    "canonical": "niche_absent"
                  }
                }
              }
            ]
          },
          {
            "name": "cf_regulator_activator_or_wrong_node",
            "role": "hard_fail",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "RUNX3",
              "Runx3",
              "PRDM1",
              "Prdm1",
              "BLIMP1",
              "Blimp1",
              "BHLHE40",
              "Bhlhe40",
              "ZNF683",
              "Zfp683",
              "HOBIT",
              "Hobit",
              "AHR",
              "Ahr",
              "BATF",
              "Batf",
              "TOX",
              "Tox",
              "NR4A1",
              "Nr4a1",
              "NR4A2",
              "Nr4a2",
              "NR4A3",
              "Nr4a3",
              "TCF7",
              "Tcf7",
              "TCF1",
              "Tcf1",
              "BACH2",
              "Bach2",
              "TBX21",
              "Tbx21",
              "TBET",
              "Tbet",
              "FLI1",
              "Fli1",
              "NR3C1",
              "Nr3c1",
              "EOMES",
              "Eomes",
              "ZEB2",
              "Zeb2",
              "ID2",
              "Id2",
              "ID3",
              "Id3",
              "IRF4",
              "Irf4"
            ]
          },
          {
            "name": "cf_node_character_hidden_driver_recall",
            "role": "hard_fail",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "hidden_driver_activity_not_expression"
          },
          {
            "name": "cf_effect_relation_hidden_pattern",
            "role": "hard_fail",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "activity_exceeds_expression"
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "causal_status",
            "kind": "value",
            "op": "in",
            "expected": [
              "sufficiency_established",
              "necessity_established",
              "causal_established_in_dataset",
              "sufficiency_via_perturbation_in_dataset"
            ]
          },
          {
            "name": "cf_residency_logic_activator_prior",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "residency_activator_induction_primary"
          },
          {
            "name": "cf_residency_logic_single_regulon",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_controls_entire_residency_program"
          },
          {
            "name": "cf_decoy_called_target",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_distractor_model_reported",
            "role": "hard_fail",
            "field": "$.trm_like_program_by_model[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_chromatin_only_decoy",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-7-pi",
      "modelName": "claude-opus-4-7",
      "harness": "pi",
      "task": "green_klf2_hidden_repressor_of_trm_like_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 4344,
      "cost": 5.16882775,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "regulator_gate",
            "role": "gate",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "KLF2",
              "Klf2"
            ]
          },
          {
            "name": "controlling_node_character_gate",
            "role": "gate",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "high_differential_expression"
          },
          {
            "name": "expression_vs_activity_gate",
            "role": "gate",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "expression_exceeds_activity"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "target_program_partition",
            "kind": "list",
            "matchKey": "gene",
            "k": 7,
            "expected": [
              {
                "key": "S1pr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "S1pr4",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Klrg1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Cx3cr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Itgae",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Hic1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "P2rx7",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              }
            ]
          },
          {
            "name": "causal_status_gate",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "association_only_sufficiency_requires_perturbation"
          },
          {
            "name": "residency_control_logic_gate",
            "role": "gate",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_release_necessary_not_sufficient_multi_regulon"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "trm_like_program_by_model",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_absent"
                    ],
                    "canonical": "niche_absent"
                  }
                }
              }
            ]
          },
          {
            "name": "cf_regulator_activator_or_wrong_node",
            "role": "hard_fail",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "RUNX3",
              "Runx3",
              "PRDM1",
              "Prdm1",
              "BLIMP1",
              "Blimp1",
              "BHLHE40",
              "Bhlhe40",
              "ZNF683",
              "Zfp683",
              "HOBIT",
              "Hobit",
              "AHR",
              "Ahr",
              "BATF",
              "Batf",
              "TOX",
              "Tox",
              "NR4A1",
              "Nr4a1",
              "NR4A2",
              "Nr4a2",
              "NR4A3",
              "Nr4a3",
              "TCF7",
              "Tcf7",
              "TCF1",
              "Tcf1",
              "BACH2",
              "Bach2",
              "TBX21",
              "Tbx21",
              "TBET",
              "Tbet",
              "FLI1",
              "Fli1",
              "NR3C1",
              "Nr3c1",
              "EOMES",
              "Eomes",
              "ZEB2",
              "Zeb2",
              "ID2",
              "Id2",
              "ID3",
              "Id3",
              "IRF4",
              "Irf4"
            ]
          },
          {
            "name": "cf_node_character_hidden_driver_recall",
            "role": "hard_fail",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "hidden_driver_activity_not_expression"
          },
          {
            "name": "cf_effect_relation_hidden_pattern",
            "role": "hard_fail",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "activity_exceeds_expression"
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "causal_status",
            "kind": "value",
            "op": "in",
            "expected": [
              "sufficiency_established",
              "necessity_established",
              "causal_established_in_dataset",
              "sufficiency_via_perturbation_in_dataset"
            ]
          },
          {
            "name": "cf_residency_logic_activator_prior",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "residency_activator_induction_primary"
          },
          {
            "name": "cf_residency_logic_single_regulon",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_controls_entire_residency_program"
          },
          {
            "name": "cf_decoy_called_target",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_distractor_model_reported",
            "role": "hard_fail",
            "field": "$.trm_like_program_by_model[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_chromatin_only_decoy",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-8-claude-code",
      "modelName": "claude-opus-4-8",
      "harness": "claude-code",
      "task": "green_klf2_hidden_repressor_of_trm_like_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 3025,
      "cost": 10.629302249999999,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "regulator_gate",
            "role": "gate",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "KLF2",
              "Klf2"
            ]
          },
          {
            "name": "controlling_node_character_gate",
            "role": "gate",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "high_differential_expression"
          },
          {
            "name": "expression_vs_activity_gate",
            "role": "gate",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "expression_exceeds_activity"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "target_program_partition",
            "kind": "list",
            "matchKey": "gene",
            "k": 7,
            "expected": [
              {
                "key": "S1pr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "S1pr4",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Klrg1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Cx3cr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Itgae",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Hic1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "P2rx7",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              }
            ]
          },
          {
            "name": "causal_status_gate",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "association_only_sufficiency_requires_perturbation"
          },
          {
            "name": "residency_control_logic_gate",
            "role": "gate",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_release_necessary_not_sufficient_multi_regulon"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "trm_like_program_by_model",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_absent"
                    ],
                    "canonical": "niche_absent"
                  }
                }
              }
            ]
          },
          {
            "name": "cf_regulator_activator_or_wrong_node",
            "role": "hard_fail",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "RUNX3",
              "Runx3",
              "PRDM1",
              "Prdm1",
              "BLIMP1",
              "Blimp1",
              "BHLHE40",
              "Bhlhe40",
              "ZNF683",
              "Zfp683",
              "HOBIT",
              "Hobit",
              "AHR",
              "Ahr",
              "BATF",
              "Batf",
              "TOX",
              "Tox",
              "NR4A1",
              "Nr4a1",
              "NR4A2",
              "Nr4a2",
              "NR4A3",
              "Nr4a3",
              "TCF7",
              "Tcf7",
              "TCF1",
              "Tcf1",
              "BACH2",
              "Bach2",
              "TBX21",
              "Tbx21",
              "TBET",
              "Tbet",
              "FLI1",
              "Fli1",
              "NR3C1",
              "Nr3c1",
              "EOMES",
              "Eomes",
              "ZEB2",
              "Zeb2",
              "ID2",
              "Id2",
              "ID3",
              "Id3",
              "IRF4",
              "Irf4"
            ]
          },
          {
            "name": "cf_node_character_hidden_driver_recall",
            "role": "hard_fail",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "hidden_driver_activity_not_expression"
          },
          {
            "name": "cf_effect_relation_hidden_pattern",
            "role": "hard_fail",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "activity_exceeds_expression"
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "causal_status",
            "kind": "value",
            "op": "in",
            "expected": [
              "sufficiency_established",
              "necessity_established",
              "causal_established_in_dataset",
              "sufficiency_via_perturbation_in_dataset"
            ]
          },
          {
            "name": "cf_residency_logic_activator_prior",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "residency_activator_induction_primary"
          },
          {
            "name": "cf_residency_logic_single_regulon",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_controls_entire_residency_program"
          },
          {
            "name": "cf_decoy_called_target",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_distractor_model_reported",
            "role": "hard_fail",
            "field": "$.trm_like_program_by_model[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_chromatin_only_decoy",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-8-claude-code",
      "modelName": "claude-opus-4-8",
      "harness": "claude-code",
      "task": "green_klf2_hidden_repressor_of_trm_like_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 7625,
      "cost": 21.112907000000014,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "regulator_gate",
            "role": "gate",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "KLF2",
              "Klf2"
            ]
          },
          {
            "name": "controlling_node_character_gate",
            "role": "gate",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "high_differential_expression"
          },
          {
            "name": "expression_vs_activity_gate",
            "role": "gate",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "expression_exceeds_activity"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "target_program_partition",
            "kind": "list",
            "matchKey": "gene",
            "k": 7,
            "expected": [
              {
                "key": "S1pr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "S1pr4",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Klrg1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Cx3cr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Itgae",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Hic1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "P2rx7",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              }
            ]
          },
          {
            "name": "causal_status_gate",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "association_only_sufficiency_requires_perturbation"
          },
          {
            "name": "residency_control_logic_gate",
            "role": "gate",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_release_necessary_not_sufficient_multi_regulon"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "trm_like_program_by_model",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_absent"
                    ],
                    "canonical": "niche_absent"
                  }
                }
              }
            ]
          },
          {
            "name": "cf_regulator_activator_or_wrong_node",
            "role": "hard_fail",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "RUNX3",
              "Runx3",
              "PRDM1",
              "Prdm1",
              "BLIMP1",
              "Blimp1",
              "BHLHE40",
              "Bhlhe40",
              "ZNF683",
              "Zfp683",
              "HOBIT",
              "Hobit",
              "AHR",
              "Ahr",
              "BATF",
              "Batf",
              "TOX",
              "Tox",
              "NR4A1",
              "Nr4a1",
              "NR4A2",
              "Nr4a2",
              "NR4A3",
              "Nr4a3",
              "TCF7",
              "Tcf7",
              "TCF1",
              "Tcf1",
              "BACH2",
              "Bach2",
              "TBX21",
              "Tbx21",
              "TBET",
              "Tbet",
              "FLI1",
              "Fli1",
              "NR3C1",
              "Nr3c1",
              "EOMES",
              "Eomes",
              "ZEB2",
              "Zeb2",
              "ID2",
              "Id2",
              "ID3",
              "Id3",
              "IRF4",
              "Irf4"
            ]
          },
          {
            "name": "cf_node_character_hidden_driver_recall",
            "role": "hard_fail",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "hidden_driver_activity_not_expression"
          },
          {
            "name": "cf_effect_relation_hidden_pattern",
            "role": "hard_fail",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "activity_exceeds_expression"
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "causal_status",
            "kind": "value",
            "op": "in",
            "expected": [
              "sufficiency_established",
              "necessity_established",
              "causal_established_in_dataset",
              "sufficiency_via_perturbation_in_dataset"
            ]
          },
          {
            "name": "cf_residency_logic_activator_prior",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "residency_activator_induction_primary"
          },
          {
            "name": "cf_residency_logic_single_regulon",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_controls_entire_residency_program"
          },
          {
            "name": "cf_decoy_called_target",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_distractor_model_reported",
            "role": "hard_fail",
            "field": "$.trm_like_program_by_model[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_chromatin_only_decoy",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-8-claude-code",
      "modelName": "claude-opus-4-8",
      "harness": "claude-code",
      "task": "green_klf2_hidden_repressor_of_trm_like_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 2117,
      "cost": 6.3784089999999996,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "regulator_gate",
            "role": "gate",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "KLF2",
              "Klf2"
            ]
          },
          {
            "name": "controlling_node_character_gate",
            "role": "gate",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "high_differential_expression"
          },
          {
            "name": "expression_vs_activity_gate",
            "role": "gate",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "expression_exceeds_activity"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "target_program_partition",
            "kind": "list",
            "matchKey": "gene",
            "k": 7,
            "expected": [
              {
                "key": "S1pr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "S1pr4",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Klrg1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Cx3cr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Itgae",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Hic1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "P2rx7",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              }
            ]
          },
          {
            "name": "causal_status_gate",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "association_only_sufficiency_requires_perturbation"
          },
          {
            "name": "residency_control_logic_gate",
            "role": "gate",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_release_necessary_not_sufficient_multi_regulon"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "trm_like_program_by_model",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_absent"
                    ],
                    "canonical": "niche_absent"
                  }
                }
              }
            ]
          },
          {
            "name": "cf_regulator_activator_or_wrong_node",
            "role": "hard_fail",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "RUNX3",
              "Runx3",
              "PRDM1",
              "Prdm1",
              "BLIMP1",
              "Blimp1",
              "BHLHE40",
              "Bhlhe40",
              "ZNF683",
              "Zfp683",
              "HOBIT",
              "Hobit",
              "AHR",
              "Ahr",
              "BATF",
              "Batf",
              "TOX",
              "Tox",
              "NR4A1",
              "Nr4a1",
              "NR4A2",
              "Nr4a2",
              "NR4A3",
              "Nr4a3",
              "TCF7",
              "Tcf7",
              "TCF1",
              "Tcf1",
              "BACH2",
              "Bach2",
              "TBX21",
              "Tbx21",
              "TBET",
              "Tbet",
              "FLI1",
              "Fli1",
              "NR3C1",
              "Nr3c1",
              "EOMES",
              "Eomes",
              "ZEB2",
              "Zeb2",
              "ID2",
              "Id2",
              "ID3",
              "Id3",
              "IRF4",
              "Irf4"
            ]
          },
          {
            "name": "cf_node_character_hidden_driver_recall",
            "role": "hard_fail",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "hidden_driver_activity_not_expression"
          },
          {
            "name": "cf_effect_relation_hidden_pattern",
            "role": "hard_fail",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "activity_exceeds_expression"
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "causal_status",
            "kind": "value",
            "op": "in",
            "expected": [
              "sufficiency_established",
              "necessity_established",
              "causal_established_in_dataset",
              "sufficiency_via_perturbation_in_dataset"
            ]
          },
          {
            "name": "cf_residency_logic_activator_prior",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "residency_activator_induction_primary"
          },
          {
            "name": "cf_residency_logic_single_regulon",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_controls_entire_residency_program"
          },
          {
            "name": "cf_decoy_called_target",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_distractor_model_reported",
            "role": "hard_fail",
            "field": "$.trm_like_program_by_model[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_chromatin_only_decoy",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-8-pi",
      "modelName": "claude-opus-4-8",
      "harness": "pi",
      "task": "green_klf2_hidden_repressor_of_trm_like_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 2772,
      "cost": 5.3207715,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "regulator_gate",
            "role": "gate",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "KLF2",
              "Klf2"
            ]
          },
          {
            "name": "controlling_node_character_gate",
            "role": "gate",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "high_differential_expression"
          },
          {
            "name": "expression_vs_activity_gate",
            "role": "gate",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "expression_exceeds_activity"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "target_program_partition",
            "kind": "list",
            "matchKey": "gene",
            "k": 7,
            "expected": [
              {
                "key": "S1pr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "S1pr4",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Klrg1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Cx3cr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Itgae",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Hic1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "P2rx7",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              }
            ]
          },
          {
            "name": "causal_status_gate",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "association_only_sufficiency_requires_perturbation"
          },
          {
            "name": "residency_control_logic_gate",
            "role": "gate",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_release_necessary_not_sufficient_multi_regulon"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "trm_like_program_by_model",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_absent"
                    ],
                    "canonical": "niche_absent"
                  }
                }
              }
            ]
          },
          {
            "name": "cf_regulator_activator_or_wrong_node",
            "role": "hard_fail",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "RUNX3",
              "Runx3",
              "PRDM1",
              "Prdm1",
              "BLIMP1",
              "Blimp1",
              "BHLHE40",
              "Bhlhe40",
              "ZNF683",
              "Zfp683",
              "HOBIT",
              "Hobit",
              "AHR",
              "Ahr",
              "BATF",
              "Batf",
              "TOX",
              "Tox",
              "NR4A1",
              "Nr4a1",
              "NR4A2",
              "Nr4a2",
              "NR4A3",
              "Nr4a3",
              "TCF7",
              "Tcf7",
              "TCF1",
              "Tcf1",
              "BACH2",
              "Bach2",
              "TBX21",
              "Tbx21",
              "TBET",
              "Tbet",
              "FLI1",
              "Fli1",
              "NR3C1",
              "Nr3c1",
              "EOMES",
              "Eomes",
              "ZEB2",
              "Zeb2",
              "ID2",
              "Id2",
              "ID3",
              "Id3",
              "IRF4",
              "Irf4"
            ]
          },
          {
            "name": "cf_node_character_hidden_driver_recall",
            "role": "hard_fail",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "hidden_driver_activity_not_expression"
          },
          {
            "name": "cf_effect_relation_hidden_pattern",
            "role": "hard_fail",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "activity_exceeds_expression"
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "causal_status",
            "kind": "value",
            "op": "in",
            "expected": [
              "sufficiency_established",
              "necessity_established",
              "causal_established_in_dataset",
              "sufficiency_via_perturbation_in_dataset"
            ]
          },
          {
            "name": "cf_residency_logic_activator_prior",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "residency_activator_induction_primary"
          },
          {
            "name": "cf_residency_logic_single_regulon",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_controls_entire_residency_program"
          },
          {
            "name": "cf_decoy_called_target",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_distractor_model_reported",
            "role": "hard_fail",
            "field": "$.trm_like_program_by_model[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_chromatin_only_decoy",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-8-pi",
      "modelName": "claude-opus-4-8",
      "harness": "pi",
      "task": "green_klf2_hidden_repressor_of_trm_like_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 2295,
      "cost": 4.75321575,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "regulator_gate",
            "role": "gate",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "KLF2",
              "Klf2"
            ]
          },
          {
            "name": "controlling_node_character_gate",
            "role": "gate",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "high_differential_expression"
          },
          {
            "name": "expression_vs_activity_gate",
            "role": "gate",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "expression_exceeds_activity"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "target_program_partition",
            "kind": "list",
            "matchKey": "gene",
            "k": 7,
            "expected": [
              {
                "key": "S1pr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "S1pr4",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Klrg1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Cx3cr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Itgae",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Hic1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "P2rx7",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              }
            ]
          },
          {
            "name": "causal_status_gate",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "association_only_sufficiency_requires_perturbation"
          },
          {
            "name": "residency_control_logic_gate",
            "role": "gate",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_release_necessary_not_sufficient_multi_regulon"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "trm_like_program_by_model",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_absent"
                    ],
                    "canonical": "niche_absent"
                  }
                }
              }
            ]
          },
          {
            "name": "cf_regulator_activator_or_wrong_node",
            "role": "hard_fail",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "RUNX3",
              "Runx3",
              "PRDM1",
              "Prdm1",
              "BLIMP1",
              "Blimp1",
              "BHLHE40",
              "Bhlhe40",
              "ZNF683",
              "Zfp683",
              "HOBIT",
              "Hobit",
              "AHR",
              "Ahr",
              "BATF",
              "Batf",
              "TOX",
              "Tox",
              "NR4A1",
              "Nr4a1",
              "NR4A2",
              "Nr4a2",
              "NR4A3",
              "Nr4a3",
              "TCF7",
              "Tcf7",
              "TCF1",
              "Tcf1",
              "BACH2",
              "Bach2",
              "TBX21",
              "Tbx21",
              "TBET",
              "Tbet",
              "FLI1",
              "Fli1",
              "NR3C1",
              "Nr3c1",
              "EOMES",
              "Eomes",
              "ZEB2",
              "Zeb2",
              "ID2",
              "Id2",
              "ID3",
              "Id3",
              "IRF4",
              "Irf4"
            ]
          },
          {
            "name": "cf_node_character_hidden_driver_recall",
            "role": "hard_fail",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "hidden_driver_activity_not_expression"
          },
          {
            "name": "cf_effect_relation_hidden_pattern",
            "role": "hard_fail",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "activity_exceeds_expression"
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "causal_status",
            "kind": "value",
            "op": "in",
            "expected": [
              "sufficiency_established",
              "necessity_established",
              "causal_established_in_dataset",
              "sufficiency_via_perturbation_in_dataset"
            ]
          },
          {
            "name": "cf_residency_logic_activator_prior",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "residency_activator_induction_primary"
          },
          {
            "name": "cf_residency_logic_single_regulon",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_controls_entire_residency_program"
          },
          {
            "name": "cf_decoy_called_target",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_distractor_model_reported",
            "role": "hard_fail",
            "field": "$.trm_like_program_by_model[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_chromatin_only_decoy",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-8-pi",
      "modelName": "claude-opus-4-8",
      "harness": "pi",
      "task": "green_klf2_hidden_repressor_of_trm_like_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 2827,
      "cost": 4.4824322500000005,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "regulator_gate",
            "role": "gate",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "KLF2",
              "Klf2"
            ]
          },
          {
            "name": "controlling_node_character_gate",
            "role": "gate",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "high_differential_expression"
          },
          {
            "name": "expression_vs_activity_gate",
            "role": "gate",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "expression_exceeds_activity"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "target_program_partition",
            "kind": "list",
            "matchKey": "gene",
            "k": 7,
            "expected": [
              {
                "key": "S1pr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "S1pr4",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Klrg1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Cx3cr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Itgae",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Hic1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "P2rx7",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              }
            ]
          },
          {
            "name": "causal_status_gate",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "association_only_sufficiency_requires_perturbation"
          },
          {
            "name": "residency_control_logic_gate",
            "role": "gate",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_release_necessary_not_sufficient_multi_regulon"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "trm_like_program_by_model",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_absent"
                    ],
                    "canonical": "niche_absent"
                  }
                }
              }
            ]
          },
          {
            "name": "cf_regulator_activator_or_wrong_node",
            "role": "hard_fail",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "RUNX3",
              "Runx3",
              "PRDM1",
              "Prdm1",
              "BLIMP1",
              "Blimp1",
              "BHLHE40",
              "Bhlhe40",
              "ZNF683",
              "Zfp683",
              "HOBIT",
              "Hobit",
              "AHR",
              "Ahr",
              "BATF",
              "Batf",
              "TOX",
              "Tox",
              "NR4A1",
              "Nr4a1",
              "NR4A2",
              "Nr4a2",
              "NR4A3",
              "Nr4a3",
              "TCF7",
              "Tcf7",
              "TCF1",
              "Tcf1",
              "BACH2",
              "Bach2",
              "TBX21",
              "Tbx21",
              "TBET",
              "Tbet",
              "FLI1",
              "Fli1",
              "NR3C1",
              "Nr3c1",
              "EOMES",
              "Eomes",
              "ZEB2",
              "Zeb2",
              "ID2",
              "Id2",
              "ID3",
              "Id3",
              "IRF4",
              "Irf4"
            ]
          },
          {
            "name": "cf_node_character_hidden_driver_recall",
            "role": "hard_fail",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "hidden_driver_activity_not_expression"
          },
          {
            "name": "cf_effect_relation_hidden_pattern",
            "role": "hard_fail",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "activity_exceeds_expression"
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "causal_status",
            "kind": "value",
            "op": "in",
            "expected": [
              "sufficiency_established",
              "necessity_established",
              "causal_established_in_dataset",
              "sufficiency_via_perturbation_in_dataset"
            ]
          },
          {
            "name": "cf_residency_logic_activator_prior",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "residency_activator_induction_primary"
          },
          {
            "name": "cf_residency_logic_single_regulon",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_controls_entire_residency_program"
          },
          {
            "name": "cf_decoy_called_target",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_distractor_model_reported",
            "role": "hard_fail",
            "field": "$.trm_like_program_by_model[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_chromatin_only_decoy",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-4-6-pi",
      "modelName": "claude-sonnet-4-6",
      "harness": "pi",
      "task": "green_klf2_hidden_repressor_of_trm_like_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 4708,
      "cost": 2.43114975,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "regulator_gate",
            "role": "gate",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "KLF2",
              "Klf2"
            ]
          },
          {
            "name": "controlling_node_character_gate",
            "role": "gate",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "high_differential_expression"
          },
          {
            "name": "expression_vs_activity_gate",
            "role": "gate",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "expression_exceeds_activity"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "target_program_partition",
            "kind": "list",
            "matchKey": "gene",
            "k": 7,
            "expected": [
              {
                "key": "S1pr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "S1pr4",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Klrg1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Cx3cr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Itgae",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Hic1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "P2rx7",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              }
            ]
          },
          {
            "name": "causal_status_gate",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "association_only_sufficiency_requires_perturbation"
          },
          {
            "name": "residency_control_logic_gate",
            "role": "gate",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_release_necessary_not_sufficient_multi_regulon"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "trm_like_program_by_model",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_absent"
                    ],
                    "canonical": "niche_absent"
                  }
                }
              }
            ]
          },
          {
            "name": "cf_regulator_activator_or_wrong_node",
            "role": "hard_fail",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "RUNX3",
              "Runx3",
              "PRDM1",
              "Prdm1",
              "BLIMP1",
              "Blimp1",
              "BHLHE40",
              "Bhlhe40",
              "ZNF683",
              "Zfp683",
              "HOBIT",
              "Hobit",
              "AHR",
              "Ahr",
              "BATF",
              "Batf",
              "TOX",
              "Tox",
              "NR4A1",
              "Nr4a1",
              "NR4A2",
              "Nr4a2",
              "NR4A3",
              "Nr4a3",
              "TCF7",
              "Tcf7",
              "TCF1",
              "Tcf1",
              "BACH2",
              "Bach2",
              "TBX21",
              "Tbx21",
              "TBET",
              "Tbet",
              "FLI1",
              "Fli1",
              "NR3C1",
              "Nr3c1",
              "EOMES",
              "Eomes",
              "ZEB2",
              "Zeb2",
              "ID2",
              "Id2",
              "ID3",
              "Id3",
              "IRF4",
              "Irf4"
            ]
          },
          {
            "name": "cf_node_character_hidden_driver_recall",
            "role": "hard_fail",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "hidden_driver_activity_not_expression"
          },
          {
            "name": "cf_effect_relation_hidden_pattern",
            "role": "hard_fail",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "activity_exceeds_expression"
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "causal_status",
            "kind": "value",
            "op": "in",
            "expected": [
              "sufficiency_established",
              "necessity_established",
              "causal_established_in_dataset",
              "sufficiency_via_perturbation_in_dataset"
            ]
          },
          {
            "name": "cf_residency_logic_activator_prior",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "residency_activator_induction_primary"
          },
          {
            "name": "cf_residency_logic_single_regulon",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_controls_entire_residency_program"
          },
          {
            "name": "cf_decoy_called_target",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_distractor_model_reported",
            "role": "hard_fail",
            "field": "$.trm_like_program_by_model[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_chromatin_only_decoy",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-4-6-pi",
      "modelName": "claude-sonnet-4-6",
      "harness": "pi",
      "task": "green_klf2_hidden_repressor_of_trm_like_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 3372,
      "cost": 2.5036538999999998,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "regulator_gate",
            "role": "gate",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "KLF2",
              "Klf2"
            ]
          },
          {
            "name": "controlling_node_character_gate",
            "role": "gate",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "high_differential_expression"
          },
          {
            "name": "expression_vs_activity_gate",
            "role": "gate",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "expression_exceeds_activity"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "target_program_partition",
            "kind": "list",
            "matchKey": "gene",
            "k": 7,
            "expected": [
              {
                "key": "S1pr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "S1pr4",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Klrg1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Cx3cr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Itgae",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Hic1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "P2rx7",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              }
            ]
          },
          {
            "name": "causal_status_gate",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "association_only_sufficiency_requires_perturbation"
          },
          {
            "name": "residency_control_logic_gate",
            "role": "gate",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_release_necessary_not_sufficient_multi_regulon"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "trm_like_program_by_model",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_absent"
                    ],
                    "canonical": "niche_absent"
                  }
                }
              }
            ]
          },
          {
            "name": "cf_regulator_activator_or_wrong_node",
            "role": "hard_fail",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "RUNX3",
              "Runx3",
              "PRDM1",
              "Prdm1",
              "BLIMP1",
              "Blimp1",
              "BHLHE40",
              "Bhlhe40",
              "ZNF683",
              "Zfp683",
              "HOBIT",
              "Hobit",
              "AHR",
              "Ahr",
              "BATF",
              "Batf",
              "TOX",
              "Tox",
              "NR4A1",
              "Nr4a1",
              "NR4A2",
              "Nr4a2",
              "NR4A3",
              "Nr4a3",
              "TCF7",
              "Tcf7",
              "TCF1",
              "Tcf1",
              "BACH2",
              "Bach2",
              "TBX21",
              "Tbx21",
              "TBET",
              "Tbet",
              "FLI1",
              "Fli1",
              "NR3C1",
              "Nr3c1",
              "EOMES",
              "Eomes",
              "ZEB2",
              "Zeb2",
              "ID2",
              "Id2",
              "ID3",
              "Id3",
              "IRF4",
              "Irf4"
            ]
          },
          {
            "name": "cf_node_character_hidden_driver_recall",
            "role": "hard_fail",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "hidden_driver_activity_not_expression"
          },
          {
            "name": "cf_effect_relation_hidden_pattern",
            "role": "hard_fail",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "activity_exceeds_expression"
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "causal_status",
            "kind": "value",
            "op": "in",
            "expected": [
              "sufficiency_established",
              "necessity_established",
              "causal_established_in_dataset",
              "sufficiency_via_perturbation_in_dataset"
            ]
          },
          {
            "name": "cf_residency_logic_activator_prior",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "residency_activator_induction_primary"
          },
          {
            "name": "cf_residency_logic_single_regulon",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_controls_entire_residency_program"
          },
          {
            "name": "cf_decoy_called_target",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_distractor_model_reported",
            "role": "hard_fail",
            "field": "$.trm_like_program_by_model[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_chromatin_only_decoy",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-4-6-pi",
      "modelName": "claude-sonnet-4-6",
      "harness": "pi",
      "task": "green_klf2_hidden_repressor_of_trm_like_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 3067,
      "cost": 1.9676093999999997,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "regulator_gate",
            "role": "gate",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "KLF2",
              "Klf2"
            ]
          },
          {
            "name": "controlling_node_character_gate",
            "role": "gate",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "high_differential_expression"
          },
          {
            "name": "expression_vs_activity_gate",
            "role": "gate",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "expression_exceeds_activity"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "target_program_partition",
            "kind": "list",
            "matchKey": "gene",
            "k": 7,
            "expected": [
              {
                "key": "S1pr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "S1pr4",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Klrg1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Cx3cr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Itgae",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Hic1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "P2rx7",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              }
            ]
          },
          {
            "name": "causal_status_gate",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "association_only_sufficiency_requires_perturbation"
          },
          {
            "name": "residency_control_logic_gate",
            "role": "gate",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_release_necessary_not_sufficient_multi_regulon"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "trm_like_program_by_model",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_absent"
                    ],
                    "canonical": "niche_absent"
                  }
                }
              }
            ]
          },
          {
            "name": "cf_regulator_activator_or_wrong_node",
            "role": "hard_fail",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "RUNX3",
              "Runx3",
              "PRDM1",
              "Prdm1",
              "BLIMP1",
              "Blimp1",
              "BHLHE40",
              "Bhlhe40",
              "ZNF683",
              "Zfp683",
              "HOBIT",
              "Hobit",
              "AHR",
              "Ahr",
              "BATF",
              "Batf",
              "TOX",
              "Tox",
              "NR4A1",
              "Nr4a1",
              "NR4A2",
              "Nr4a2",
              "NR4A3",
              "Nr4a3",
              "TCF7",
              "Tcf7",
              "TCF1",
              "Tcf1",
              "BACH2",
              "Bach2",
              "TBX21",
              "Tbx21",
              "TBET",
              "Tbet",
              "FLI1",
              "Fli1",
              "NR3C1",
              "Nr3c1",
              "EOMES",
              "Eomes",
              "ZEB2",
              "Zeb2",
              "ID2",
              "Id2",
              "ID3",
              "Id3",
              "IRF4",
              "Irf4"
            ]
          },
          {
            "name": "cf_node_character_hidden_driver_recall",
            "role": "hard_fail",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "hidden_driver_activity_not_expression"
          },
          {
            "name": "cf_effect_relation_hidden_pattern",
            "role": "hard_fail",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "activity_exceeds_expression"
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "causal_status",
            "kind": "value",
            "op": "in",
            "expected": [
              "sufficiency_established",
              "necessity_established",
              "causal_established_in_dataset",
              "sufficiency_via_perturbation_in_dataset"
            ]
          },
          {
            "name": "cf_residency_logic_activator_prior",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "residency_activator_induction_primary"
          },
          {
            "name": "cf_residency_logic_single_regulon",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_controls_entire_residency_program"
          },
          {
            "name": "cf_decoy_called_target",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_distractor_model_reported",
            "role": "hard_fail",
            "field": "$.trm_like_program_by_model[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_chromatin_only_decoy",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          }
        ]
      }
    },
    {
      "modelId": "sclong-gemini-3-1-pro-preview-pi",
      "modelName": "gemini-3.1-pro-preview",
      "harness": "pi",
      "task": "green_klf2_hidden_repressor_of_trm_like_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 540,
      "cost": 0.595646,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "regulator_gate",
            "role": "gate",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "KLF2",
              "Klf2"
            ]
          },
          {
            "name": "controlling_node_character_gate",
            "role": "gate",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "high_differential_expression"
          },
          {
            "name": "expression_vs_activity_gate",
            "role": "gate",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "expression_exceeds_activity"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "target_program_partition",
            "kind": "list",
            "matchKey": "gene",
            "k": 7,
            "expected": [
              {
                "key": "S1pr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "S1pr4",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Klrg1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Cx3cr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Itgae",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Hic1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "P2rx7",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              }
            ]
          },
          {
            "name": "causal_status_gate",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "association_only_sufficiency_requires_perturbation"
          },
          {
            "name": "residency_control_logic_gate",
            "role": "gate",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_release_necessary_not_sufficient_multi_regulon"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "trm_like_program_by_model",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_absent"
                    ],
                    "canonical": "niche_absent"
                  }
                }
              }
            ]
          },
          {
            "name": "cf_regulator_activator_or_wrong_node",
            "role": "hard_fail",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "RUNX3",
              "Runx3",
              "PRDM1",
              "Prdm1",
              "BLIMP1",
              "Blimp1",
              "BHLHE40",
              "Bhlhe40",
              "ZNF683",
              "Zfp683",
              "HOBIT",
              "Hobit",
              "AHR",
              "Ahr",
              "BATF",
              "Batf",
              "TOX",
              "Tox",
              "NR4A1",
              "Nr4a1",
              "NR4A2",
              "Nr4a2",
              "NR4A3",
              "Nr4a3",
              "TCF7",
              "Tcf7",
              "TCF1",
              "Tcf1",
              "BACH2",
              "Bach2",
              "TBX21",
              "Tbx21",
              "TBET",
              "Tbet",
              "FLI1",
              "Fli1",
              "NR3C1",
              "Nr3c1",
              "EOMES",
              "Eomes",
              "ZEB2",
              "Zeb2",
              "ID2",
              "Id2",
              "ID3",
              "Id3",
              "IRF4",
              "Irf4"
            ]
          },
          {
            "name": "cf_node_character_hidden_driver_recall",
            "role": "hard_fail",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "hidden_driver_activity_not_expression"
          },
          {
            "name": "cf_effect_relation_hidden_pattern",
            "role": "hard_fail",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "activity_exceeds_expression"
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "causal_status",
            "kind": "value",
            "op": "in",
            "expected": [
              "sufficiency_established",
              "necessity_established",
              "causal_established_in_dataset",
              "sufficiency_via_perturbation_in_dataset"
            ]
          },
          {
            "name": "cf_residency_logic_activator_prior",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "residency_activator_induction_primary"
          },
          {
            "name": "cf_residency_logic_single_regulon",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_controls_entire_residency_program"
          },
          {
            "name": "cf_decoy_called_target",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_distractor_model_reported",
            "role": "hard_fail",
            "field": "$.trm_like_program_by_model[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_chromatin_only_decoy",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          }
        ]
      }
    },
    {
      "modelId": "sclong-gemini-3-1-pro-preview-pi",
      "modelName": "gemini-3.1-pro-preview",
      "harness": "pi",
      "task": "green_klf2_hidden_repressor_of_trm_like_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 1149,
      "cost": 0.45634600000000003,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "regulator_gate",
            "role": "gate",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "KLF2",
              "Klf2"
            ]
          },
          {
            "name": "controlling_node_character_gate",
            "role": "gate",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "high_differential_expression"
          },
          {
            "name": "expression_vs_activity_gate",
            "role": "gate",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "expression_exceeds_activity"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "target_program_partition",
            "kind": "list",
            "matchKey": "gene",
            "k": 7,
            "expected": [
              {
                "key": "S1pr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "S1pr4",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Klrg1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Cx3cr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Itgae",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Hic1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "P2rx7",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              }
            ]
          },
          {
            "name": "causal_status_gate",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "association_only_sufficiency_requires_perturbation"
          },
          {
            "name": "residency_control_logic_gate",
            "role": "gate",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_release_necessary_not_sufficient_multi_regulon"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "trm_like_program_by_model",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_absent"
                    ],
                    "canonical": "niche_absent"
                  }
                }
              }
            ]
          },
          {
            "name": "cf_regulator_activator_or_wrong_node",
            "role": "hard_fail",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "RUNX3",
              "Runx3",
              "PRDM1",
              "Prdm1",
              "BLIMP1",
              "Blimp1",
              "BHLHE40",
              "Bhlhe40",
              "ZNF683",
              "Zfp683",
              "HOBIT",
              "Hobit",
              "AHR",
              "Ahr",
              "BATF",
              "Batf",
              "TOX",
              "Tox",
              "NR4A1",
              "Nr4a1",
              "NR4A2",
              "Nr4a2",
              "NR4A3",
              "Nr4a3",
              "TCF7",
              "Tcf7",
              "TCF1",
              "Tcf1",
              "BACH2",
              "Bach2",
              "TBX21",
              "Tbx21",
              "TBET",
              "Tbet",
              "FLI1",
              "Fli1",
              "NR3C1",
              "Nr3c1",
              "EOMES",
              "Eomes",
              "ZEB2",
              "Zeb2",
              "ID2",
              "Id2",
              "ID3",
              "Id3",
              "IRF4",
              "Irf4"
            ]
          },
          {
            "name": "cf_node_character_hidden_driver_recall",
            "role": "hard_fail",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "hidden_driver_activity_not_expression"
          },
          {
            "name": "cf_effect_relation_hidden_pattern",
            "role": "hard_fail",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "activity_exceeds_expression"
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "causal_status",
            "kind": "value",
            "op": "in",
            "expected": [
              "sufficiency_established",
              "necessity_established",
              "causal_established_in_dataset",
              "sufficiency_via_perturbation_in_dataset"
            ]
          },
          {
            "name": "cf_residency_logic_activator_prior",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "residency_activator_induction_primary"
          },
          {
            "name": "cf_residency_logic_single_regulon",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_controls_entire_residency_program"
          },
          {
            "name": "cf_decoy_called_target",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_distractor_model_reported",
            "role": "hard_fail",
            "field": "$.trm_like_program_by_model[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_chromatin_only_decoy",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          }
        ]
      }
    },
    {
      "modelId": "sclong-gemini-3-1-pro-preview-pi",
      "modelName": "gemini-3.1-pro-preview",
      "harness": "pi",
      "task": "green_klf2_hidden_repressor_of_trm_like_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 483,
      "cost": 0.687321,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "regulator_gate",
            "role": "gate",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "KLF2",
              "Klf2"
            ]
          },
          {
            "name": "controlling_node_character_gate",
            "role": "gate",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "high_differential_expression"
          },
          {
            "name": "expression_vs_activity_gate",
            "role": "gate",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "expression_exceeds_activity"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "target_program_partition",
            "kind": "list",
            "matchKey": "gene",
            "k": 7,
            "expected": [
              {
                "key": "S1pr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "S1pr4",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Klrg1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Cx3cr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Itgae",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Hic1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "P2rx7",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              }
            ]
          },
          {
            "name": "causal_status_gate",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "association_only_sufficiency_requires_perturbation"
          },
          {
            "name": "residency_control_logic_gate",
            "role": "gate",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_release_necessary_not_sufficient_multi_regulon"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "trm_like_program_by_model",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_absent"
                    ],
                    "canonical": "niche_absent"
                  }
                }
              }
            ]
          },
          {
            "name": "cf_regulator_activator_or_wrong_node",
            "role": "hard_fail",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "RUNX3",
              "Runx3",
              "PRDM1",
              "Prdm1",
              "BLIMP1",
              "Blimp1",
              "BHLHE40",
              "Bhlhe40",
              "ZNF683",
              "Zfp683",
              "HOBIT",
              "Hobit",
              "AHR",
              "Ahr",
              "BATF",
              "Batf",
              "TOX",
              "Tox",
              "NR4A1",
              "Nr4a1",
              "NR4A2",
              "Nr4a2",
              "NR4A3",
              "Nr4a3",
              "TCF7",
              "Tcf7",
              "TCF1",
              "Tcf1",
              "BACH2",
              "Bach2",
              "TBX21",
              "Tbx21",
              "TBET",
              "Tbet",
              "FLI1",
              "Fli1",
              "NR3C1",
              "Nr3c1",
              "EOMES",
              "Eomes",
              "ZEB2",
              "Zeb2",
              "ID2",
              "Id2",
              "ID3",
              "Id3",
              "IRF4",
              "Irf4"
            ]
          },
          {
            "name": "cf_node_character_hidden_driver_recall",
            "role": "hard_fail",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "hidden_driver_activity_not_expression"
          },
          {
            "name": "cf_effect_relation_hidden_pattern",
            "role": "hard_fail",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "activity_exceeds_expression"
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "causal_status",
            "kind": "value",
            "op": "in",
            "expected": [
              "sufficiency_established",
              "necessity_established",
              "causal_established_in_dataset",
              "sufficiency_via_perturbation_in_dataset"
            ]
          },
          {
            "name": "cf_residency_logic_activator_prior",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "residency_activator_induction_primary"
          },
          {
            "name": "cf_residency_logic_single_regulon",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_controls_entire_residency_program"
          },
          {
            "name": "cf_decoy_called_target",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_distractor_model_reported",
            "role": "hard_fail",
            "field": "$.trm_like_program_by_model[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_chromatin_only_decoy",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          }
        ]
      }
    },
    {
      "modelId": "sclong-gemini-3-5-flash-pi",
      "modelName": "gemini-3.5-flash",
      "harness": "pi",
      "task": "green_klf2_hidden_repressor_of_trm_like_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 2009,
      "cost": 1.31614395,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "regulator_gate",
            "role": "gate",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "KLF2",
              "Klf2"
            ]
          },
          {
            "name": "controlling_node_character_gate",
            "role": "gate",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "high_differential_expression"
          },
          {
            "name": "expression_vs_activity_gate",
            "role": "gate",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "expression_exceeds_activity"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "target_program_partition",
            "kind": "list",
            "matchKey": "gene",
            "k": 7,
            "expected": [
              {
                "key": "S1pr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "S1pr4",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Klrg1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Cx3cr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Itgae",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Hic1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "P2rx7",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              }
            ]
          },
          {
            "name": "causal_status_gate",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "association_only_sufficiency_requires_perturbation"
          },
          {
            "name": "residency_control_logic_gate",
            "role": "gate",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_release_necessary_not_sufficient_multi_regulon"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "trm_like_program_by_model",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_absent"
                    ],
                    "canonical": "niche_absent"
                  }
                }
              }
            ]
          },
          {
            "name": "cf_regulator_activator_or_wrong_node",
            "role": "hard_fail",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "RUNX3",
              "Runx3",
              "PRDM1",
              "Prdm1",
              "BLIMP1",
              "Blimp1",
              "BHLHE40",
              "Bhlhe40",
              "ZNF683",
              "Zfp683",
              "HOBIT",
              "Hobit",
              "AHR",
              "Ahr",
              "BATF",
              "Batf",
              "TOX",
              "Tox",
              "NR4A1",
              "Nr4a1",
              "NR4A2",
              "Nr4a2",
              "NR4A3",
              "Nr4a3",
              "TCF7",
              "Tcf7",
              "TCF1",
              "Tcf1",
              "BACH2",
              "Bach2",
              "TBX21",
              "Tbx21",
              "TBET",
              "Tbet",
              "FLI1",
              "Fli1",
              "NR3C1",
              "Nr3c1",
              "EOMES",
              "Eomes",
              "ZEB2",
              "Zeb2",
              "ID2",
              "Id2",
              "ID3",
              "Id3",
              "IRF4",
              "Irf4"
            ]
          },
          {
            "name": "cf_node_character_hidden_driver_recall",
            "role": "hard_fail",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "hidden_driver_activity_not_expression"
          },
          {
            "name": "cf_effect_relation_hidden_pattern",
            "role": "hard_fail",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "activity_exceeds_expression"
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "causal_status",
            "kind": "value",
            "op": "in",
            "expected": [
              "sufficiency_established",
              "necessity_established",
              "causal_established_in_dataset",
              "sufficiency_via_perturbation_in_dataset"
            ]
          },
          {
            "name": "cf_residency_logic_activator_prior",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "residency_activator_induction_primary"
          },
          {
            "name": "cf_residency_logic_single_regulon",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_controls_entire_residency_program"
          },
          {
            "name": "cf_decoy_called_target",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_distractor_model_reported",
            "role": "hard_fail",
            "field": "$.trm_like_program_by_model[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_chromatin_only_decoy",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          }
        ]
      }
    },
    {
      "modelId": "sclong-gemini-3-5-flash-pi",
      "modelName": "gemini-3.5-flash",
      "harness": "pi",
      "task": "green_klf2_hidden_repressor_of_trm_like_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 681,
      "cost": 0.6767002499999999,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "regulator_gate",
            "role": "gate",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "KLF2",
              "Klf2"
            ]
          },
          {
            "name": "controlling_node_character_gate",
            "role": "gate",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "high_differential_expression"
          },
          {
            "name": "expression_vs_activity_gate",
            "role": "gate",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "expression_exceeds_activity"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "target_program_partition",
            "kind": "list",
            "matchKey": "gene",
            "k": 7,
            "expected": [
              {
                "key": "S1pr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "S1pr4",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Klrg1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Cx3cr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Itgae",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Hic1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "P2rx7",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              }
            ]
          },
          {
            "name": "causal_status_gate",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "association_only_sufficiency_requires_perturbation"
          },
          {
            "name": "residency_control_logic_gate",
            "role": "gate",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_release_necessary_not_sufficient_multi_regulon"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "trm_like_program_by_model",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_absent"
                    ],
                    "canonical": "niche_absent"
                  }
                }
              }
            ]
          },
          {
            "name": "cf_regulator_activator_or_wrong_node",
            "role": "hard_fail",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "RUNX3",
              "Runx3",
              "PRDM1",
              "Prdm1",
              "BLIMP1",
              "Blimp1",
              "BHLHE40",
              "Bhlhe40",
              "ZNF683",
              "Zfp683",
              "HOBIT",
              "Hobit",
              "AHR",
              "Ahr",
              "BATF",
              "Batf",
              "TOX",
              "Tox",
              "NR4A1",
              "Nr4a1",
              "NR4A2",
              "Nr4a2",
              "NR4A3",
              "Nr4a3",
              "TCF7",
              "Tcf7",
              "TCF1",
              "Tcf1",
              "BACH2",
              "Bach2",
              "TBX21",
              "Tbx21",
              "TBET",
              "Tbet",
              "FLI1",
              "Fli1",
              "NR3C1",
              "Nr3c1",
              "EOMES",
              "Eomes",
              "ZEB2",
              "Zeb2",
              "ID2",
              "Id2",
              "ID3",
              "Id3",
              "IRF4",
              "Irf4"
            ]
          },
          {
            "name": "cf_node_character_hidden_driver_recall",
            "role": "hard_fail",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "hidden_driver_activity_not_expression"
          },
          {
            "name": "cf_effect_relation_hidden_pattern",
            "role": "hard_fail",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "activity_exceeds_expression"
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "causal_status",
            "kind": "value",
            "op": "in",
            "expected": [
              "sufficiency_established",
              "necessity_established",
              "causal_established_in_dataset",
              "sufficiency_via_perturbation_in_dataset"
            ]
          },
          {
            "name": "cf_residency_logic_activator_prior",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "residency_activator_induction_primary"
          },
          {
            "name": "cf_residency_logic_single_regulon",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_controls_entire_residency_program"
          },
          {
            "name": "cf_decoy_called_target",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_distractor_model_reported",
            "role": "hard_fail",
            "field": "$.trm_like_program_by_model[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_chromatin_only_decoy",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          }
        ]
      }
    },
    {
      "modelId": "sclong-gemini-3-5-flash-pi",
      "modelName": "gemini-3.5-flash",
      "harness": "pi",
      "task": "green_klf2_hidden_repressor_of_trm_like_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 816,
      "cost": 0.7418138999999999,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "regulator_gate",
            "role": "gate",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "KLF2",
              "Klf2"
            ]
          },
          {
            "name": "controlling_node_character_gate",
            "role": "gate",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "high_differential_expression"
          },
          {
            "name": "expression_vs_activity_gate",
            "role": "gate",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "expression_exceeds_activity"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "target_program_partition",
            "kind": "list",
            "matchKey": "gene",
            "k": 7,
            "expected": [
              {
                "key": "S1pr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "S1pr4",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Klrg1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Cx3cr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Itgae",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Hic1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "P2rx7",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              }
            ]
          },
          {
            "name": "causal_status_gate",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "association_only_sufficiency_requires_perturbation"
          },
          {
            "name": "residency_control_logic_gate",
            "role": "gate",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_release_necessary_not_sufficient_multi_regulon"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "trm_like_program_by_model",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_absent"
                    ],
                    "canonical": "niche_absent"
                  }
                }
              }
            ]
          },
          {
            "name": "cf_regulator_activator_or_wrong_node",
            "role": "hard_fail",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "RUNX3",
              "Runx3",
              "PRDM1",
              "Prdm1",
              "BLIMP1",
              "Blimp1",
              "BHLHE40",
              "Bhlhe40",
              "ZNF683",
              "Zfp683",
              "HOBIT",
              "Hobit",
              "AHR",
              "Ahr",
              "BATF",
              "Batf",
              "TOX",
              "Tox",
              "NR4A1",
              "Nr4a1",
              "NR4A2",
              "Nr4a2",
              "NR4A3",
              "Nr4a3",
              "TCF7",
              "Tcf7",
              "TCF1",
              "Tcf1",
              "BACH2",
              "Bach2",
              "TBX21",
              "Tbx21",
              "TBET",
              "Tbet",
              "FLI1",
              "Fli1",
              "NR3C1",
              "Nr3c1",
              "EOMES",
              "Eomes",
              "ZEB2",
              "Zeb2",
              "ID2",
              "Id2",
              "ID3",
              "Id3",
              "IRF4",
              "Irf4"
            ]
          },
          {
            "name": "cf_node_character_hidden_driver_recall",
            "role": "hard_fail",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "hidden_driver_activity_not_expression"
          },
          {
            "name": "cf_effect_relation_hidden_pattern",
            "role": "hard_fail",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "activity_exceeds_expression"
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "causal_status",
            "kind": "value",
            "op": "in",
            "expected": [
              "sufficiency_established",
              "necessity_established",
              "causal_established_in_dataset",
              "sufficiency_via_perturbation_in_dataset"
            ]
          },
          {
            "name": "cf_residency_logic_activator_prior",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "residency_activator_induction_primary"
          },
          {
            "name": "cf_residency_logic_single_regulon",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_controls_entire_residency_program"
          },
          {
            "name": "cf_decoy_called_target",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_distractor_model_reported",
            "role": "hard_fail",
            "field": "$.trm_like_program_by_model[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_chromatin_only_decoy",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          }
        ]
      }
    },
    {
      "modelId": "sclong-glm-5p2-pi",
      "modelName": "glm-5p2",
      "harness": "pi",
      "task": "green_klf2_hidden_repressor_of_trm_like_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 3248,
      "cost": 2.41476458,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "regulator_gate",
            "role": "gate",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "KLF2",
              "Klf2"
            ]
          },
          {
            "name": "controlling_node_character_gate",
            "role": "gate",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "high_differential_expression"
          },
          {
            "name": "expression_vs_activity_gate",
            "role": "gate",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "expression_exceeds_activity"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "target_program_partition",
            "kind": "list",
            "matchKey": "gene",
            "k": 7,
            "expected": [
              {
                "key": "S1pr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "S1pr4",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Klrg1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Cx3cr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Itgae",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Hic1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "P2rx7",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              }
            ]
          },
          {
            "name": "causal_status_gate",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "association_only_sufficiency_requires_perturbation"
          },
          {
            "name": "residency_control_logic_gate",
            "role": "gate",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_release_necessary_not_sufficient_multi_regulon"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "trm_like_program_by_model",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_absent"
                    ],
                    "canonical": "niche_absent"
                  }
                }
              }
            ]
          },
          {
            "name": "cf_regulator_activator_or_wrong_node",
            "role": "hard_fail",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "RUNX3",
              "Runx3",
              "PRDM1",
              "Prdm1",
              "BLIMP1",
              "Blimp1",
              "BHLHE40",
              "Bhlhe40",
              "ZNF683",
              "Zfp683",
              "HOBIT",
              "Hobit",
              "AHR",
              "Ahr",
              "BATF",
              "Batf",
              "TOX",
              "Tox",
              "NR4A1",
              "Nr4a1",
              "NR4A2",
              "Nr4a2",
              "NR4A3",
              "Nr4a3",
              "TCF7",
              "Tcf7",
              "TCF1",
              "Tcf1",
              "BACH2",
              "Bach2",
              "TBX21",
              "Tbx21",
              "TBET",
              "Tbet",
              "FLI1",
              "Fli1",
              "NR3C1",
              "Nr3c1",
              "EOMES",
              "Eomes",
              "ZEB2",
              "Zeb2",
              "ID2",
              "Id2",
              "ID3",
              "Id3",
              "IRF4",
              "Irf4"
            ]
          },
          {
            "name": "cf_node_character_hidden_driver_recall",
            "role": "hard_fail",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "hidden_driver_activity_not_expression"
          },
          {
            "name": "cf_effect_relation_hidden_pattern",
            "role": "hard_fail",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "activity_exceeds_expression"
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "causal_status",
            "kind": "value",
            "op": "in",
            "expected": [
              "sufficiency_established",
              "necessity_established",
              "causal_established_in_dataset",
              "sufficiency_via_perturbation_in_dataset"
            ]
          },
          {
            "name": "cf_residency_logic_activator_prior",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "residency_activator_induction_primary"
          },
          {
            "name": "cf_residency_logic_single_regulon",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_controls_entire_residency_program"
          },
          {
            "name": "cf_decoy_called_target",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_distractor_model_reported",
            "role": "hard_fail",
            "field": "$.trm_like_program_by_model[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_chromatin_only_decoy",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          }
        ]
      }
    },
    {
      "modelId": "sclong-glm-5p2-pi",
      "modelName": "glm-5p2",
      "harness": "pi",
      "task": "green_klf2_hidden_repressor_of_trm_like_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 4314,
      "cost": 5.138669199999998,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "regulator_gate",
            "role": "gate",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "KLF2",
              "Klf2"
            ]
          },
          {
            "name": "controlling_node_character_gate",
            "role": "gate",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "high_differential_expression"
          },
          {
            "name": "expression_vs_activity_gate",
            "role": "gate",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "expression_exceeds_activity"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "target_program_partition",
            "kind": "list",
            "matchKey": "gene",
            "k": 7,
            "expected": [
              {
                "key": "S1pr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "S1pr4",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Klrg1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Cx3cr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Itgae",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Hic1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "P2rx7",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              }
            ]
          },
          {
            "name": "causal_status_gate",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "association_only_sufficiency_requires_perturbation"
          },
          {
            "name": "residency_control_logic_gate",
            "role": "gate",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_release_necessary_not_sufficient_multi_regulon"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "trm_like_program_by_model",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_absent"
                    ],
                    "canonical": "niche_absent"
                  }
                }
              }
            ]
          },
          {
            "name": "cf_regulator_activator_or_wrong_node",
            "role": "hard_fail",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "RUNX3",
              "Runx3",
              "PRDM1",
              "Prdm1",
              "BLIMP1",
              "Blimp1",
              "BHLHE40",
              "Bhlhe40",
              "ZNF683",
              "Zfp683",
              "HOBIT",
              "Hobit",
              "AHR",
              "Ahr",
              "BATF",
              "Batf",
              "TOX",
              "Tox",
              "NR4A1",
              "Nr4a1",
              "NR4A2",
              "Nr4a2",
              "NR4A3",
              "Nr4a3",
              "TCF7",
              "Tcf7",
              "TCF1",
              "Tcf1",
              "BACH2",
              "Bach2",
              "TBX21",
              "Tbx21",
              "TBET",
              "Tbet",
              "FLI1",
              "Fli1",
              "NR3C1",
              "Nr3c1",
              "EOMES",
              "Eomes",
              "ZEB2",
              "Zeb2",
              "ID2",
              "Id2",
              "ID3",
              "Id3",
              "IRF4",
              "Irf4"
            ]
          },
          {
            "name": "cf_node_character_hidden_driver_recall",
            "role": "hard_fail",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "hidden_driver_activity_not_expression"
          },
          {
            "name": "cf_effect_relation_hidden_pattern",
            "role": "hard_fail",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "activity_exceeds_expression"
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "causal_status",
            "kind": "value",
            "op": "in",
            "expected": [
              "sufficiency_established",
              "necessity_established",
              "causal_established_in_dataset",
              "sufficiency_via_perturbation_in_dataset"
            ]
          },
          {
            "name": "cf_residency_logic_activator_prior",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "residency_activator_induction_primary"
          },
          {
            "name": "cf_residency_logic_single_regulon",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_controls_entire_residency_program"
          },
          {
            "name": "cf_decoy_called_target",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_distractor_model_reported",
            "role": "hard_fail",
            "field": "$.trm_like_program_by_model[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_chromatin_only_decoy",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          }
        ]
      }
    },
    {
      "modelId": "sclong-glm-5p2-pi",
      "modelName": "glm-5p2",
      "harness": "pi",
      "task": "green_klf2_hidden_repressor_of_trm_like_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 2978,
      "cost": 3.1263894600000004,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "regulator_gate",
            "role": "gate",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "KLF2",
              "Klf2"
            ]
          },
          {
            "name": "controlling_node_character_gate",
            "role": "gate",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "high_differential_expression"
          },
          {
            "name": "expression_vs_activity_gate",
            "role": "gate",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "expression_exceeds_activity"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "target_program_partition",
            "kind": "list",
            "matchKey": "gene",
            "k": 7,
            "expected": [
              {
                "key": "S1pr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "S1pr4",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Klrg1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Cx3cr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Itgae",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Hic1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "P2rx7",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              }
            ]
          },
          {
            "name": "causal_status_gate",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "association_only_sufficiency_requires_perturbation"
          },
          {
            "name": "residency_control_logic_gate",
            "role": "gate",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_release_necessary_not_sufficient_multi_regulon"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "trm_like_program_by_model",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_absent"
                    ],
                    "canonical": "niche_absent"
                  }
                }
              }
            ]
          },
          {
            "name": "cf_regulator_activator_or_wrong_node",
            "role": "hard_fail",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "RUNX3",
              "Runx3",
              "PRDM1",
              "Prdm1",
              "BLIMP1",
              "Blimp1",
              "BHLHE40",
              "Bhlhe40",
              "ZNF683",
              "Zfp683",
              "HOBIT",
              "Hobit",
              "AHR",
              "Ahr",
              "BATF",
              "Batf",
              "TOX",
              "Tox",
              "NR4A1",
              "Nr4a1",
              "NR4A2",
              "Nr4a2",
              "NR4A3",
              "Nr4a3",
              "TCF7",
              "Tcf7",
              "TCF1",
              "Tcf1",
              "BACH2",
              "Bach2",
              "TBX21",
              "Tbx21",
              "TBET",
              "Tbet",
              "FLI1",
              "Fli1",
              "NR3C1",
              "Nr3c1",
              "EOMES",
              "Eomes",
              "ZEB2",
              "Zeb2",
              "ID2",
              "Id2",
              "ID3",
              "Id3",
              "IRF4",
              "Irf4"
            ]
          },
          {
            "name": "cf_node_character_hidden_driver_recall",
            "role": "hard_fail",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "hidden_driver_activity_not_expression"
          },
          {
            "name": "cf_effect_relation_hidden_pattern",
            "role": "hard_fail",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "activity_exceeds_expression"
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "causal_status",
            "kind": "value",
            "op": "in",
            "expected": [
              "sufficiency_established",
              "necessity_established",
              "causal_established_in_dataset",
              "sufficiency_via_perturbation_in_dataset"
            ]
          },
          {
            "name": "cf_residency_logic_activator_prior",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "residency_activator_induction_primary"
          },
          {
            "name": "cf_residency_logic_single_regulon",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_controls_entire_residency_program"
          },
          {
            "name": "cf_decoy_called_target",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_distractor_model_reported",
            "role": "hard_fail",
            "field": "$.trm_like_program_by_model[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_chromatin_only_decoy",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-4-codex",
      "modelName": "gpt-5.4",
      "harness": "openai-codex",
      "task": "green_klf2_hidden_repressor_of_trm_like_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 1113,
      "cost": 2.6238875000000004,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "regulator_gate",
            "role": "gate",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "KLF2",
              "Klf2"
            ]
          },
          {
            "name": "controlling_node_character_gate",
            "role": "gate",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "high_differential_expression"
          },
          {
            "name": "expression_vs_activity_gate",
            "role": "gate",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "expression_exceeds_activity"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "target_program_partition",
            "kind": "list",
            "matchKey": "gene",
            "k": 7,
            "expected": [
              {
                "key": "S1pr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "S1pr4",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Klrg1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Cx3cr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Itgae",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Hic1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "P2rx7",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              }
            ]
          },
          {
            "name": "causal_status_gate",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "association_only_sufficiency_requires_perturbation"
          },
          {
            "name": "residency_control_logic_gate",
            "role": "gate",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_release_necessary_not_sufficient_multi_regulon"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "trm_like_program_by_model",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_absent"
                    ],
                    "canonical": "niche_absent"
                  }
                }
              }
            ]
          },
          {
            "name": "cf_regulator_activator_or_wrong_node",
            "role": "hard_fail",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "RUNX3",
              "Runx3",
              "PRDM1",
              "Prdm1",
              "BLIMP1",
              "Blimp1",
              "BHLHE40",
              "Bhlhe40",
              "ZNF683",
              "Zfp683",
              "HOBIT",
              "Hobit",
              "AHR",
              "Ahr",
              "BATF",
              "Batf",
              "TOX",
              "Tox",
              "NR4A1",
              "Nr4a1",
              "NR4A2",
              "Nr4a2",
              "NR4A3",
              "Nr4a3",
              "TCF7",
              "Tcf7",
              "TCF1",
              "Tcf1",
              "BACH2",
              "Bach2",
              "TBX21",
              "Tbx21",
              "TBET",
              "Tbet",
              "FLI1",
              "Fli1",
              "NR3C1",
              "Nr3c1",
              "EOMES",
              "Eomes",
              "ZEB2",
              "Zeb2",
              "ID2",
              "Id2",
              "ID3",
              "Id3",
              "IRF4",
              "Irf4"
            ]
          },
          {
            "name": "cf_node_character_hidden_driver_recall",
            "role": "hard_fail",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "hidden_driver_activity_not_expression"
          },
          {
            "name": "cf_effect_relation_hidden_pattern",
            "role": "hard_fail",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "activity_exceeds_expression"
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "causal_status",
            "kind": "value",
            "op": "in",
            "expected": [
              "sufficiency_established",
              "necessity_established",
              "causal_established_in_dataset",
              "sufficiency_via_perturbation_in_dataset"
            ]
          },
          {
            "name": "cf_residency_logic_activator_prior",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "residency_activator_induction_primary"
          },
          {
            "name": "cf_residency_logic_single_regulon",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_controls_entire_residency_program"
          },
          {
            "name": "cf_decoy_called_target",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_distractor_model_reported",
            "role": "hard_fail",
            "field": "$.trm_like_program_by_model[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_chromatin_only_decoy",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-4-codex",
      "modelName": "gpt-5.4",
      "harness": "openai-codex",
      "task": "green_klf2_hidden_repressor_of_trm_like_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 975,
      "cost": 2.7058014999999997,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "regulator_gate",
            "role": "gate",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "KLF2",
              "Klf2"
            ]
          },
          {
            "name": "controlling_node_character_gate",
            "role": "gate",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "high_differential_expression"
          },
          {
            "name": "expression_vs_activity_gate",
            "role": "gate",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "expression_exceeds_activity"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "target_program_partition",
            "kind": "list",
            "matchKey": "gene",
            "k": 7,
            "expected": [
              {
                "key": "S1pr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "S1pr4",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Klrg1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Cx3cr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Itgae",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Hic1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "P2rx7",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              }
            ]
          },
          {
            "name": "causal_status_gate",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "association_only_sufficiency_requires_perturbation"
          },
          {
            "name": "residency_control_logic_gate",
            "role": "gate",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_release_necessary_not_sufficient_multi_regulon"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "trm_like_program_by_model",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_absent"
                    ],
                    "canonical": "niche_absent"
                  }
                }
              }
            ]
          },
          {
            "name": "cf_regulator_activator_or_wrong_node",
            "role": "hard_fail",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "RUNX3",
              "Runx3",
              "PRDM1",
              "Prdm1",
              "BLIMP1",
              "Blimp1",
              "BHLHE40",
              "Bhlhe40",
              "ZNF683",
              "Zfp683",
              "HOBIT",
              "Hobit",
              "AHR",
              "Ahr",
              "BATF",
              "Batf",
              "TOX",
              "Tox",
              "NR4A1",
              "Nr4a1",
              "NR4A2",
              "Nr4a2",
              "NR4A3",
              "Nr4a3",
              "TCF7",
              "Tcf7",
              "TCF1",
              "Tcf1",
              "BACH2",
              "Bach2",
              "TBX21",
              "Tbx21",
              "TBET",
              "Tbet",
              "FLI1",
              "Fli1",
              "NR3C1",
              "Nr3c1",
              "EOMES",
              "Eomes",
              "ZEB2",
              "Zeb2",
              "ID2",
              "Id2",
              "ID3",
              "Id3",
              "IRF4",
              "Irf4"
            ]
          },
          {
            "name": "cf_node_character_hidden_driver_recall",
            "role": "hard_fail",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "hidden_driver_activity_not_expression"
          },
          {
            "name": "cf_effect_relation_hidden_pattern",
            "role": "hard_fail",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "activity_exceeds_expression"
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "causal_status",
            "kind": "value",
            "op": "in",
            "expected": [
              "sufficiency_established",
              "necessity_established",
              "causal_established_in_dataset",
              "sufficiency_via_perturbation_in_dataset"
            ]
          },
          {
            "name": "cf_residency_logic_activator_prior",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "residency_activator_induction_primary"
          },
          {
            "name": "cf_residency_logic_single_regulon",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_controls_entire_residency_program"
          },
          {
            "name": "cf_decoy_called_target",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_distractor_model_reported",
            "role": "hard_fail",
            "field": "$.trm_like_program_by_model[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_chromatin_only_decoy",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-4-codex",
      "modelName": "gpt-5.4",
      "harness": "openai-codex",
      "task": "green_klf2_hidden_repressor_of_trm_like_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 2069,
      "cost": 2.8370405,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "regulator_gate",
            "role": "gate",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "KLF2",
              "Klf2"
            ]
          },
          {
            "name": "controlling_node_character_gate",
            "role": "gate",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "high_differential_expression"
          },
          {
            "name": "expression_vs_activity_gate",
            "role": "gate",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "expression_exceeds_activity"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "target_program_partition",
            "kind": "list",
            "matchKey": "gene",
            "k": 7,
            "expected": [
              {
                "key": "S1pr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "S1pr4",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Klrg1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Cx3cr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Itgae",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Hic1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "P2rx7",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              }
            ]
          },
          {
            "name": "causal_status_gate",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "association_only_sufficiency_requires_perturbation"
          },
          {
            "name": "residency_control_logic_gate",
            "role": "gate",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_release_necessary_not_sufficient_multi_regulon"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "trm_like_program_by_model",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_absent"
                    ],
                    "canonical": "niche_absent"
                  }
                }
              }
            ]
          },
          {
            "name": "cf_regulator_activator_or_wrong_node",
            "role": "hard_fail",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "RUNX3",
              "Runx3",
              "PRDM1",
              "Prdm1",
              "BLIMP1",
              "Blimp1",
              "BHLHE40",
              "Bhlhe40",
              "ZNF683",
              "Zfp683",
              "HOBIT",
              "Hobit",
              "AHR",
              "Ahr",
              "BATF",
              "Batf",
              "TOX",
              "Tox",
              "NR4A1",
              "Nr4a1",
              "NR4A2",
              "Nr4a2",
              "NR4A3",
              "Nr4a3",
              "TCF7",
              "Tcf7",
              "TCF1",
              "Tcf1",
              "BACH2",
              "Bach2",
              "TBX21",
              "Tbx21",
              "TBET",
              "Tbet",
              "FLI1",
              "Fli1",
              "NR3C1",
              "Nr3c1",
              "EOMES",
              "Eomes",
              "ZEB2",
              "Zeb2",
              "ID2",
              "Id2",
              "ID3",
              "Id3",
              "IRF4",
              "Irf4"
            ]
          },
          {
            "name": "cf_node_character_hidden_driver_recall",
            "role": "hard_fail",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "hidden_driver_activity_not_expression"
          },
          {
            "name": "cf_effect_relation_hidden_pattern",
            "role": "hard_fail",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "activity_exceeds_expression"
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "causal_status",
            "kind": "value",
            "op": "in",
            "expected": [
              "sufficiency_established",
              "necessity_established",
              "causal_established_in_dataset",
              "sufficiency_via_perturbation_in_dataset"
            ]
          },
          {
            "name": "cf_residency_logic_activator_prior",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "residency_activator_induction_primary"
          },
          {
            "name": "cf_residency_logic_single_regulon",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_controls_entire_residency_program"
          },
          {
            "name": "cf_decoy_called_target",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_distractor_model_reported",
            "role": "hard_fail",
            "field": "$.trm_like_program_by_model[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_chromatin_only_decoy",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-4-pi",
      "modelName": "gpt-5.4",
      "harness": "pi",
      "task": "green_klf2_hidden_repressor_of_trm_like_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 5579,
      "cost": 1.941985,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "regulator_gate",
            "role": "gate",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "KLF2",
              "Klf2"
            ]
          },
          {
            "name": "controlling_node_character_gate",
            "role": "gate",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "high_differential_expression"
          },
          {
            "name": "expression_vs_activity_gate",
            "role": "gate",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "expression_exceeds_activity"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "target_program_partition",
            "kind": "list",
            "matchKey": "gene",
            "k": 7,
            "expected": [
              {
                "key": "S1pr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "S1pr4",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Klrg1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Cx3cr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Itgae",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Hic1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "P2rx7",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              }
            ]
          },
          {
            "name": "causal_status_gate",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "association_only_sufficiency_requires_perturbation"
          },
          {
            "name": "residency_control_logic_gate",
            "role": "gate",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_release_necessary_not_sufficient_multi_regulon"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "trm_like_program_by_model",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_absent"
                    ],
                    "canonical": "niche_absent"
                  }
                }
              }
            ]
          },
          {
            "name": "cf_regulator_activator_or_wrong_node",
            "role": "hard_fail",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "RUNX3",
              "Runx3",
              "PRDM1",
              "Prdm1",
              "BLIMP1",
              "Blimp1",
              "BHLHE40",
              "Bhlhe40",
              "ZNF683",
              "Zfp683",
              "HOBIT",
              "Hobit",
              "AHR",
              "Ahr",
              "BATF",
              "Batf",
              "TOX",
              "Tox",
              "NR4A1",
              "Nr4a1",
              "NR4A2",
              "Nr4a2",
              "NR4A3",
              "Nr4a3",
              "TCF7",
              "Tcf7",
              "TCF1",
              "Tcf1",
              "BACH2",
              "Bach2",
              "TBX21",
              "Tbx21",
              "TBET",
              "Tbet",
              "FLI1",
              "Fli1",
              "NR3C1",
              "Nr3c1",
              "EOMES",
              "Eomes",
              "ZEB2",
              "Zeb2",
              "ID2",
              "Id2",
              "ID3",
              "Id3",
              "IRF4",
              "Irf4"
            ]
          },
          {
            "name": "cf_node_character_hidden_driver_recall",
            "role": "hard_fail",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "hidden_driver_activity_not_expression"
          },
          {
            "name": "cf_effect_relation_hidden_pattern",
            "role": "hard_fail",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "activity_exceeds_expression"
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "causal_status",
            "kind": "value",
            "op": "in",
            "expected": [
              "sufficiency_established",
              "necessity_established",
              "causal_established_in_dataset",
              "sufficiency_via_perturbation_in_dataset"
            ]
          },
          {
            "name": "cf_residency_logic_activator_prior",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "residency_activator_induction_primary"
          },
          {
            "name": "cf_residency_logic_single_regulon",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_controls_entire_residency_program"
          },
          {
            "name": "cf_decoy_called_target",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_distractor_model_reported",
            "role": "hard_fail",
            "field": "$.trm_like_program_by_model[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_chromatin_only_decoy",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-4-pi",
      "modelName": "gpt-5.4",
      "harness": "pi",
      "task": "green_klf2_hidden_repressor_of_trm_like_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 4909,
      "cost": 1.5527800000000005,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "regulator_gate",
            "role": "gate",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "KLF2",
              "Klf2"
            ]
          },
          {
            "name": "controlling_node_character_gate",
            "role": "gate",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "high_differential_expression"
          },
          {
            "name": "expression_vs_activity_gate",
            "role": "gate",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "expression_exceeds_activity"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "target_program_partition",
            "kind": "list",
            "matchKey": "gene",
            "k": 7,
            "expected": [
              {
                "key": "S1pr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "S1pr4",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Klrg1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Cx3cr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Itgae",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Hic1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "P2rx7",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              }
            ]
          },
          {
            "name": "causal_status_gate",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "association_only_sufficiency_requires_perturbation"
          },
          {
            "name": "residency_control_logic_gate",
            "role": "gate",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_release_necessary_not_sufficient_multi_regulon"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "trm_like_program_by_model",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_absent"
                    ],
                    "canonical": "niche_absent"
                  }
                }
              }
            ]
          },
          {
            "name": "cf_regulator_activator_or_wrong_node",
            "role": "hard_fail",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "RUNX3",
              "Runx3",
              "PRDM1",
              "Prdm1",
              "BLIMP1",
              "Blimp1",
              "BHLHE40",
              "Bhlhe40",
              "ZNF683",
              "Zfp683",
              "HOBIT",
              "Hobit",
              "AHR",
              "Ahr",
              "BATF",
              "Batf",
              "TOX",
              "Tox",
              "NR4A1",
              "Nr4a1",
              "NR4A2",
              "Nr4a2",
              "NR4A3",
              "Nr4a3",
              "TCF7",
              "Tcf7",
              "TCF1",
              "Tcf1",
              "BACH2",
              "Bach2",
              "TBX21",
              "Tbx21",
              "TBET",
              "Tbet",
              "FLI1",
              "Fli1",
              "NR3C1",
              "Nr3c1",
              "EOMES",
              "Eomes",
              "ZEB2",
              "Zeb2",
              "ID2",
              "Id2",
              "ID3",
              "Id3",
              "IRF4",
              "Irf4"
            ]
          },
          {
            "name": "cf_node_character_hidden_driver_recall",
            "role": "hard_fail",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "hidden_driver_activity_not_expression"
          },
          {
            "name": "cf_effect_relation_hidden_pattern",
            "role": "hard_fail",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "activity_exceeds_expression"
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "causal_status",
            "kind": "value",
            "op": "in",
            "expected": [
              "sufficiency_established",
              "necessity_established",
              "causal_established_in_dataset",
              "sufficiency_via_perturbation_in_dataset"
            ]
          },
          {
            "name": "cf_residency_logic_activator_prior",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "residency_activator_induction_primary"
          },
          {
            "name": "cf_residency_logic_single_regulon",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_controls_entire_residency_program"
          },
          {
            "name": "cf_decoy_called_target",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_distractor_model_reported",
            "role": "hard_fail",
            "field": "$.trm_like_program_by_model[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_chromatin_only_decoy",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-4-pi",
      "modelName": "gpt-5.4",
      "harness": "pi",
      "task": "green_klf2_hidden_repressor_of_trm_like_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 6805,
      "cost": 2.146731,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "regulator_gate",
            "role": "gate",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "KLF2",
              "Klf2"
            ]
          },
          {
            "name": "controlling_node_character_gate",
            "role": "gate",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "high_differential_expression"
          },
          {
            "name": "expression_vs_activity_gate",
            "role": "gate",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "expression_exceeds_activity"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "target_program_partition",
            "kind": "list",
            "matchKey": "gene",
            "k": 7,
            "expected": [
              {
                "key": "S1pr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "S1pr4",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Klrg1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Cx3cr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Itgae",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Hic1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "P2rx7",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              }
            ]
          },
          {
            "name": "causal_status_gate",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "association_only_sufficiency_requires_perturbation"
          },
          {
            "name": "residency_control_logic_gate",
            "role": "gate",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_release_necessary_not_sufficient_multi_regulon"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "trm_like_program_by_model",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_absent"
                    ],
                    "canonical": "niche_absent"
                  }
                }
              }
            ]
          },
          {
            "name": "cf_regulator_activator_or_wrong_node",
            "role": "hard_fail",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "RUNX3",
              "Runx3",
              "PRDM1",
              "Prdm1",
              "BLIMP1",
              "Blimp1",
              "BHLHE40",
              "Bhlhe40",
              "ZNF683",
              "Zfp683",
              "HOBIT",
              "Hobit",
              "AHR",
              "Ahr",
              "BATF",
              "Batf",
              "TOX",
              "Tox",
              "NR4A1",
              "Nr4a1",
              "NR4A2",
              "Nr4a2",
              "NR4A3",
              "Nr4a3",
              "TCF7",
              "Tcf7",
              "TCF1",
              "Tcf1",
              "BACH2",
              "Bach2",
              "TBX21",
              "Tbx21",
              "TBET",
              "Tbet",
              "FLI1",
              "Fli1",
              "NR3C1",
              "Nr3c1",
              "EOMES",
              "Eomes",
              "ZEB2",
              "Zeb2",
              "ID2",
              "Id2",
              "ID3",
              "Id3",
              "IRF4",
              "Irf4"
            ]
          },
          {
            "name": "cf_node_character_hidden_driver_recall",
            "role": "hard_fail",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "hidden_driver_activity_not_expression"
          },
          {
            "name": "cf_effect_relation_hidden_pattern",
            "role": "hard_fail",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "activity_exceeds_expression"
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "causal_status",
            "kind": "value",
            "op": "in",
            "expected": [
              "sufficiency_established",
              "necessity_established",
              "causal_established_in_dataset",
              "sufficiency_via_perturbation_in_dataset"
            ]
          },
          {
            "name": "cf_residency_logic_activator_prior",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "residency_activator_induction_primary"
          },
          {
            "name": "cf_residency_logic_single_regulon",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_controls_entire_residency_program"
          },
          {
            "name": "cf_decoy_called_target",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_distractor_model_reported",
            "role": "hard_fail",
            "field": "$.trm_like_program_by_model[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_chromatin_only_decoy",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-5-codex",
      "modelName": "gpt-5.5",
      "harness": "openai-codex",
      "task": "green_klf2_hidden_repressor_of_trm_like_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 1018,
      "cost": 2.126462,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "regulator_gate",
            "role": "gate",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "KLF2",
              "Klf2"
            ]
          },
          {
            "name": "controlling_node_character_gate",
            "role": "gate",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "high_differential_expression"
          },
          {
            "name": "expression_vs_activity_gate",
            "role": "gate",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "expression_exceeds_activity"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "target_program_partition",
            "kind": "list",
            "matchKey": "gene",
            "k": 7,
            "expected": [
              {
                "key": "S1pr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "S1pr4",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Klrg1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Cx3cr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Itgae",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Hic1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "P2rx7",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              }
            ]
          },
          {
            "name": "causal_status_gate",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "association_only_sufficiency_requires_perturbation"
          },
          {
            "name": "residency_control_logic_gate",
            "role": "gate",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_release_necessary_not_sufficient_multi_regulon"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "trm_like_program_by_model",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_absent"
                    ],
                    "canonical": "niche_absent"
                  }
                }
              }
            ]
          },
          {
            "name": "cf_regulator_activator_or_wrong_node",
            "role": "hard_fail",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "RUNX3",
              "Runx3",
              "PRDM1",
              "Prdm1",
              "BLIMP1",
              "Blimp1",
              "BHLHE40",
              "Bhlhe40",
              "ZNF683",
              "Zfp683",
              "HOBIT",
              "Hobit",
              "AHR",
              "Ahr",
              "BATF",
              "Batf",
              "TOX",
              "Tox",
              "NR4A1",
              "Nr4a1",
              "NR4A2",
              "Nr4a2",
              "NR4A3",
              "Nr4a3",
              "TCF7",
              "Tcf7",
              "TCF1",
              "Tcf1",
              "BACH2",
              "Bach2",
              "TBX21",
              "Tbx21",
              "TBET",
              "Tbet",
              "FLI1",
              "Fli1",
              "NR3C1",
              "Nr3c1",
              "EOMES",
              "Eomes",
              "ZEB2",
              "Zeb2",
              "ID2",
              "Id2",
              "ID3",
              "Id3",
              "IRF4",
              "Irf4"
            ]
          },
          {
            "name": "cf_node_character_hidden_driver_recall",
            "role": "hard_fail",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "hidden_driver_activity_not_expression"
          },
          {
            "name": "cf_effect_relation_hidden_pattern",
            "role": "hard_fail",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "activity_exceeds_expression"
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "causal_status",
            "kind": "value",
            "op": "in",
            "expected": [
              "sufficiency_established",
              "necessity_established",
              "causal_established_in_dataset",
              "sufficiency_via_perturbation_in_dataset"
            ]
          },
          {
            "name": "cf_residency_logic_activator_prior",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "residency_activator_induction_primary"
          },
          {
            "name": "cf_residency_logic_single_regulon",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_controls_entire_residency_program"
          },
          {
            "name": "cf_decoy_called_target",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_distractor_model_reported",
            "role": "hard_fail",
            "field": "$.trm_like_program_by_model[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_chromatin_only_decoy",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-5-codex",
      "modelName": "gpt-5.5",
      "harness": "openai-codex",
      "task": "green_klf2_hidden_repressor_of_trm_like_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 1708,
      "cost": 2.822905,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "regulator_gate",
            "role": "gate",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "KLF2",
              "Klf2"
            ]
          },
          {
            "name": "controlling_node_character_gate",
            "role": "gate",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "high_differential_expression"
          },
          {
            "name": "expression_vs_activity_gate",
            "role": "gate",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "expression_exceeds_activity"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "target_program_partition",
            "kind": "list",
            "matchKey": "gene",
            "k": 7,
            "expected": [
              {
                "key": "S1pr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "S1pr4",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Klrg1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Cx3cr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Itgae",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Hic1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "P2rx7",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              }
            ]
          },
          {
            "name": "causal_status_gate",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "association_only_sufficiency_requires_perturbation"
          },
          {
            "name": "residency_control_logic_gate",
            "role": "gate",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_release_necessary_not_sufficient_multi_regulon"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "trm_like_program_by_model",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_absent"
                    ],
                    "canonical": "niche_absent"
                  }
                }
              }
            ]
          },
          {
            "name": "cf_regulator_activator_or_wrong_node",
            "role": "hard_fail",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "RUNX3",
              "Runx3",
              "PRDM1",
              "Prdm1",
              "BLIMP1",
              "Blimp1",
              "BHLHE40",
              "Bhlhe40",
              "ZNF683",
              "Zfp683",
              "HOBIT",
              "Hobit",
              "AHR",
              "Ahr",
              "BATF",
              "Batf",
              "TOX",
              "Tox",
              "NR4A1",
              "Nr4a1",
              "NR4A2",
              "Nr4a2",
              "NR4A3",
              "Nr4a3",
              "TCF7",
              "Tcf7",
              "TCF1",
              "Tcf1",
              "BACH2",
              "Bach2",
              "TBX21",
              "Tbx21",
              "TBET",
              "Tbet",
              "FLI1",
              "Fli1",
              "NR3C1",
              "Nr3c1",
              "EOMES",
              "Eomes",
              "ZEB2",
              "Zeb2",
              "ID2",
              "Id2",
              "ID3",
              "Id3",
              "IRF4",
              "Irf4"
            ]
          },
          {
            "name": "cf_node_character_hidden_driver_recall",
            "role": "hard_fail",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "hidden_driver_activity_not_expression"
          },
          {
            "name": "cf_effect_relation_hidden_pattern",
            "role": "hard_fail",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "activity_exceeds_expression"
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "causal_status",
            "kind": "value",
            "op": "in",
            "expected": [
              "sufficiency_established",
              "necessity_established",
              "causal_established_in_dataset",
              "sufficiency_via_perturbation_in_dataset"
            ]
          },
          {
            "name": "cf_residency_logic_activator_prior",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "residency_activator_induction_primary"
          },
          {
            "name": "cf_residency_logic_single_regulon",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_controls_entire_residency_program"
          },
          {
            "name": "cf_decoy_called_target",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_distractor_model_reported",
            "role": "hard_fail",
            "field": "$.trm_like_program_by_model[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_chromatin_only_decoy",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-5-codex",
      "modelName": "gpt-5.5",
      "harness": "openai-codex",
      "task": "green_klf2_hidden_repressor_of_trm_like_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1266,
      "cost": 3.126933,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "regulator_gate",
            "role": "gate",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "KLF2",
              "Klf2"
            ]
          },
          {
            "name": "controlling_node_character_gate",
            "role": "gate",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "high_differential_expression"
          },
          {
            "name": "expression_vs_activity_gate",
            "role": "gate",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "expression_exceeds_activity"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "target_program_partition",
            "kind": "list",
            "matchKey": "gene",
            "k": 7,
            "expected": [
              {
                "key": "S1pr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "S1pr4",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Klrg1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Cx3cr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Itgae",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Hic1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "P2rx7",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              }
            ]
          },
          {
            "name": "causal_status_gate",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "association_only_sufficiency_requires_perturbation"
          },
          {
            "name": "residency_control_logic_gate",
            "role": "gate",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_release_necessary_not_sufficient_multi_regulon"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "trm_like_program_by_model",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_absent"
                    ],
                    "canonical": "niche_absent"
                  }
                }
              }
            ]
          },
          {
            "name": "cf_regulator_activator_or_wrong_node",
            "role": "hard_fail",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "RUNX3",
              "Runx3",
              "PRDM1",
              "Prdm1",
              "BLIMP1",
              "Blimp1",
              "BHLHE40",
              "Bhlhe40",
              "ZNF683",
              "Zfp683",
              "HOBIT",
              "Hobit",
              "AHR",
              "Ahr",
              "BATF",
              "Batf",
              "TOX",
              "Tox",
              "NR4A1",
              "Nr4a1",
              "NR4A2",
              "Nr4a2",
              "NR4A3",
              "Nr4a3",
              "TCF7",
              "Tcf7",
              "TCF1",
              "Tcf1",
              "BACH2",
              "Bach2",
              "TBX21",
              "Tbx21",
              "TBET",
              "Tbet",
              "FLI1",
              "Fli1",
              "NR3C1",
              "Nr3c1",
              "EOMES",
              "Eomes",
              "ZEB2",
              "Zeb2",
              "ID2",
              "Id2",
              "ID3",
              "Id3",
              "IRF4",
              "Irf4"
            ]
          },
          {
            "name": "cf_node_character_hidden_driver_recall",
            "role": "hard_fail",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "hidden_driver_activity_not_expression"
          },
          {
            "name": "cf_effect_relation_hidden_pattern",
            "role": "hard_fail",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "activity_exceeds_expression"
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "causal_status",
            "kind": "value",
            "op": "in",
            "expected": [
              "sufficiency_established",
              "necessity_established",
              "causal_established_in_dataset",
              "sufficiency_via_perturbation_in_dataset"
            ]
          },
          {
            "name": "cf_residency_logic_activator_prior",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "residency_activator_induction_primary"
          },
          {
            "name": "cf_residency_logic_single_regulon",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_controls_entire_residency_program"
          },
          {
            "name": "cf_decoy_called_target",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_distractor_model_reported",
            "role": "hard_fail",
            "field": "$.trm_like_program_by_model[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_chromatin_only_decoy",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-5-pi",
      "modelName": "gpt-5.5",
      "harness": "pi",
      "task": "green_klf2_hidden_repressor_of_trm_like_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 1794,
      "cost": 1.7367040000000002,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "regulator_gate",
            "role": "gate",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "KLF2",
              "Klf2"
            ]
          },
          {
            "name": "controlling_node_character_gate",
            "role": "gate",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "high_differential_expression"
          },
          {
            "name": "expression_vs_activity_gate",
            "role": "gate",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "expression_exceeds_activity"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "target_program_partition",
            "kind": "list",
            "matchKey": "gene",
            "k": 7,
            "expected": [
              {
                "key": "S1pr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "S1pr4",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Klrg1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Cx3cr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Itgae",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Hic1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "P2rx7",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              }
            ]
          },
          {
            "name": "causal_status_gate",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "association_only_sufficiency_requires_perturbation"
          },
          {
            "name": "residency_control_logic_gate",
            "role": "gate",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_release_necessary_not_sufficient_multi_regulon"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "trm_like_program_by_model",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_absent"
                    ],
                    "canonical": "niche_absent"
                  }
                }
              }
            ]
          },
          {
            "name": "cf_regulator_activator_or_wrong_node",
            "role": "hard_fail",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "RUNX3",
              "Runx3",
              "PRDM1",
              "Prdm1",
              "BLIMP1",
              "Blimp1",
              "BHLHE40",
              "Bhlhe40",
              "ZNF683",
              "Zfp683",
              "HOBIT",
              "Hobit",
              "AHR",
              "Ahr",
              "BATF",
              "Batf",
              "TOX",
              "Tox",
              "NR4A1",
              "Nr4a1",
              "NR4A2",
              "Nr4a2",
              "NR4A3",
              "Nr4a3",
              "TCF7",
              "Tcf7",
              "TCF1",
              "Tcf1",
              "BACH2",
              "Bach2",
              "TBX21",
              "Tbx21",
              "TBET",
              "Tbet",
              "FLI1",
              "Fli1",
              "NR3C1",
              "Nr3c1",
              "EOMES",
              "Eomes",
              "ZEB2",
              "Zeb2",
              "ID2",
              "Id2",
              "ID3",
              "Id3",
              "IRF4",
              "Irf4"
            ]
          },
          {
            "name": "cf_node_character_hidden_driver_recall",
            "role": "hard_fail",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "hidden_driver_activity_not_expression"
          },
          {
            "name": "cf_effect_relation_hidden_pattern",
            "role": "hard_fail",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "activity_exceeds_expression"
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "causal_status",
            "kind": "value",
            "op": "in",
            "expected": [
              "sufficiency_established",
              "necessity_established",
              "causal_established_in_dataset",
              "sufficiency_via_perturbation_in_dataset"
            ]
          },
          {
            "name": "cf_residency_logic_activator_prior",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "residency_activator_induction_primary"
          },
          {
            "name": "cf_residency_logic_single_regulon",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_controls_entire_residency_program"
          },
          {
            "name": "cf_decoy_called_target",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_distractor_model_reported",
            "role": "hard_fail",
            "field": "$.trm_like_program_by_model[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_chromatin_only_decoy",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-5-pi",
      "modelName": "gpt-5.5",
      "harness": "pi",
      "task": "green_klf2_hidden_repressor_of_trm_like_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 3299,
      "cost": 5.716387,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "regulator_gate",
            "role": "gate",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "KLF2",
              "Klf2"
            ]
          },
          {
            "name": "controlling_node_character_gate",
            "role": "gate",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "high_differential_expression"
          },
          {
            "name": "expression_vs_activity_gate",
            "role": "gate",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "expression_exceeds_activity"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "target_program_partition",
            "kind": "list",
            "matchKey": "gene",
            "k": 7,
            "expected": [
              {
                "key": "S1pr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "S1pr4",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Klrg1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Cx3cr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Itgae",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Hic1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "P2rx7",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              }
            ]
          },
          {
            "name": "causal_status_gate",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "association_only_sufficiency_requires_perturbation"
          },
          {
            "name": "residency_control_logic_gate",
            "role": "gate",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_release_necessary_not_sufficient_multi_regulon"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "trm_like_program_by_model",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_absent"
                    ],
                    "canonical": "niche_absent"
                  }
                }
              }
            ]
          },
          {
            "name": "cf_regulator_activator_or_wrong_node",
            "role": "hard_fail",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "RUNX3",
              "Runx3",
              "PRDM1",
              "Prdm1",
              "BLIMP1",
              "Blimp1",
              "BHLHE40",
              "Bhlhe40",
              "ZNF683",
              "Zfp683",
              "HOBIT",
              "Hobit",
              "AHR",
              "Ahr",
              "BATF",
              "Batf",
              "TOX",
              "Tox",
              "NR4A1",
              "Nr4a1",
              "NR4A2",
              "Nr4a2",
              "NR4A3",
              "Nr4a3",
              "TCF7",
              "Tcf7",
              "TCF1",
              "Tcf1",
              "BACH2",
              "Bach2",
              "TBX21",
              "Tbx21",
              "TBET",
              "Tbet",
              "FLI1",
              "Fli1",
              "NR3C1",
              "Nr3c1",
              "EOMES",
              "Eomes",
              "ZEB2",
              "Zeb2",
              "ID2",
              "Id2",
              "ID3",
              "Id3",
              "IRF4",
              "Irf4"
            ]
          },
          {
            "name": "cf_node_character_hidden_driver_recall",
            "role": "hard_fail",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "hidden_driver_activity_not_expression"
          },
          {
            "name": "cf_effect_relation_hidden_pattern",
            "role": "hard_fail",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "activity_exceeds_expression"
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "causal_status",
            "kind": "value",
            "op": "in",
            "expected": [
              "sufficiency_established",
              "necessity_established",
              "causal_established_in_dataset",
              "sufficiency_via_perturbation_in_dataset"
            ]
          },
          {
            "name": "cf_residency_logic_activator_prior",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "residency_activator_induction_primary"
          },
          {
            "name": "cf_residency_logic_single_regulon",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_controls_entire_residency_program"
          },
          {
            "name": "cf_decoy_called_target",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_distractor_model_reported",
            "role": "hard_fail",
            "field": "$.trm_like_program_by_model[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_chromatin_only_decoy",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-5-pi",
      "modelName": "gpt-5.5",
      "harness": "pi",
      "task": "green_klf2_hidden_repressor_of_trm_like_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 2854,
      "cost": 4.331231000000002,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "regulator_gate",
            "role": "gate",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "KLF2",
              "Klf2"
            ]
          },
          {
            "name": "controlling_node_character_gate",
            "role": "gate",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "high_differential_expression"
          },
          {
            "name": "expression_vs_activity_gate",
            "role": "gate",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "expression_exceeds_activity"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "target_program_partition",
            "kind": "list",
            "matchKey": "gene",
            "k": 7,
            "expected": [
              {
                "key": "S1pr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "S1pr4",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Klrg1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Cx3cr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Itgae",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Hic1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "P2rx7",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              }
            ]
          },
          {
            "name": "causal_status_gate",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "association_only_sufficiency_requires_perturbation"
          },
          {
            "name": "residency_control_logic_gate",
            "role": "gate",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_release_necessary_not_sufficient_multi_regulon"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "trm_like_program_by_model",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_absent"
                    ],
                    "canonical": "niche_absent"
                  }
                }
              }
            ]
          },
          {
            "name": "cf_regulator_activator_or_wrong_node",
            "role": "hard_fail",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "RUNX3",
              "Runx3",
              "PRDM1",
              "Prdm1",
              "BLIMP1",
              "Blimp1",
              "BHLHE40",
              "Bhlhe40",
              "ZNF683",
              "Zfp683",
              "HOBIT",
              "Hobit",
              "AHR",
              "Ahr",
              "BATF",
              "Batf",
              "TOX",
              "Tox",
              "NR4A1",
              "Nr4a1",
              "NR4A2",
              "Nr4a2",
              "NR4A3",
              "Nr4a3",
              "TCF7",
              "Tcf7",
              "TCF1",
              "Tcf1",
              "BACH2",
              "Bach2",
              "TBX21",
              "Tbx21",
              "TBET",
              "Tbet",
              "FLI1",
              "Fli1",
              "NR3C1",
              "Nr3c1",
              "EOMES",
              "Eomes",
              "ZEB2",
              "Zeb2",
              "ID2",
              "Id2",
              "ID3",
              "Id3",
              "IRF4",
              "Irf4"
            ]
          },
          {
            "name": "cf_node_character_hidden_driver_recall",
            "role": "hard_fail",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "hidden_driver_activity_not_expression"
          },
          {
            "name": "cf_effect_relation_hidden_pattern",
            "role": "hard_fail",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "activity_exceeds_expression"
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "causal_status",
            "kind": "value",
            "op": "in",
            "expected": [
              "sufficiency_established",
              "necessity_established",
              "causal_established_in_dataset",
              "sufficiency_via_perturbation_in_dataset"
            ]
          },
          {
            "name": "cf_residency_logic_activator_prior",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "residency_activator_induction_primary"
          },
          {
            "name": "cf_residency_logic_single_regulon",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_controls_entire_residency_program"
          },
          {
            "name": "cf_decoy_called_target",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_distractor_model_reported",
            "role": "hard_fail",
            "field": "$.trm_like_program_by_model[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_chromatin_only_decoy",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-20-beta-reasoning-pi",
      "modelName": "grok-4.20-beta-0309-reasoning",
      "harness": "pi",
      "task": "green_klf2_hidden_repressor_of_trm_like_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 232,
      "cost": 0.015731750000000003,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "regulator_gate",
            "role": "gate",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "KLF2",
              "Klf2"
            ]
          },
          {
            "name": "controlling_node_character_gate",
            "role": "gate",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "high_differential_expression"
          },
          {
            "name": "expression_vs_activity_gate",
            "role": "gate",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "expression_exceeds_activity"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "target_program_partition",
            "kind": "list",
            "matchKey": "gene",
            "k": 7,
            "expected": [
              {
                "key": "S1pr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "S1pr4",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Klrg1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Cx3cr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Itgae",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Hic1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "P2rx7",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              }
            ]
          },
          {
            "name": "causal_status_gate",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "association_only_sufficiency_requires_perturbation"
          },
          {
            "name": "residency_control_logic_gate",
            "role": "gate",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_release_necessary_not_sufficient_multi_regulon"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "trm_like_program_by_model",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_absent"
                    ],
                    "canonical": "niche_absent"
                  }
                }
              }
            ]
          },
          {
            "name": "cf_regulator_activator_or_wrong_node",
            "role": "hard_fail",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "RUNX3",
              "Runx3",
              "PRDM1",
              "Prdm1",
              "BLIMP1",
              "Blimp1",
              "BHLHE40",
              "Bhlhe40",
              "ZNF683",
              "Zfp683",
              "HOBIT",
              "Hobit",
              "AHR",
              "Ahr",
              "BATF",
              "Batf",
              "TOX",
              "Tox",
              "NR4A1",
              "Nr4a1",
              "NR4A2",
              "Nr4a2",
              "NR4A3",
              "Nr4a3",
              "TCF7",
              "Tcf7",
              "TCF1",
              "Tcf1",
              "BACH2",
              "Bach2",
              "TBX21",
              "Tbx21",
              "TBET",
              "Tbet",
              "FLI1",
              "Fli1",
              "NR3C1",
              "Nr3c1",
              "EOMES",
              "Eomes",
              "ZEB2",
              "Zeb2",
              "ID2",
              "Id2",
              "ID3",
              "Id3",
              "IRF4",
              "Irf4"
            ]
          },
          {
            "name": "cf_node_character_hidden_driver_recall",
            "role": "hard_fail",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "hidden_driver_activity_not_expression"
          },
          {
            "name": "cf_effect_relation_hidden_pattern",
            "role": "hard_fail",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "activity_exceeds_expression"
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "causal_status",
            "kind": "value",
            "op": "in",
            "expected": [
              "sufficiency_established",
              "necessity_established",
              "causal_established_in_dataset",
              "sufficiency_via_perturbation_in_dataset"
            ]
          },
          {
            "name": "cf_residency_logic_activator_prior",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "residency_activator_induction_primary"
          },
          {
            "name": "cf_residency_logic_single_regulon",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_controls_entire_residency_program"
          },
          {
            "name": "cf_decoy_called_target",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_distractor_model_reported",
            "role": "hard_fail",
            "field": "$.trm_like_program_by_model[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_chromatin_only_decoy",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-20-beta-reasoning-pi",
      "modelName": "grok-4.20-beta-0309-reasoning",
      "harness": "pi",
      "task": "green_klf2_hidden_repressor_of_trm_like_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 1471,
      "cost": 0.05709949999999999,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "regulator_gate",
            "role": "gate",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "KLF2",
              "Klf2"
            ]
          },
          {
            "name": "controlling_node_character_gate",
            "role": "gate",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "high_differential_expression"
          },
          {
            "name": "expression_vs_activity_gate",
            "role": "gate",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "expression_exceeds_activity"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "target_program_partition",
            "kind": "list",
            "matchKey": "gene",
            "k": 7,
            "expected": [
              {
                "key": "S1pr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "S1pr4",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Klrg1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Cx3cr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Itgae",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Hic1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "P2rx7",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              }
            ]
          },
          {
            "name": "causal_status_gate",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "association_only_sufficiency_requires_perturbation"
          },
          {
            "name": "residency_control_logic_gate",
            "role": "gate",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_release_necessary_not_sufficient_multi_regulon"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "trm_like_program_by_model",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_absent"
                    ],
                    "canonical": "niche_absent"
                  }
                }
              }
            ]
          },
          {
            "name": "cf_regulator_activator_or_wrong_node",
            "role": "hard_fail",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "RUNX3",
              "Runx3",
              "PRDM1",
              "Prdm1",
              "BLIMP1",
              "Blimp1",
              "BHLHE40",
              "Bhlhe40",
              "ZNF683",
              "Zfp683",
              "HOBIT",
              "Hobit",
              "AHR",
              "Ahr",
              "BATF",
              "Batf",
              "TOX",
              "Tox",
              "NR4A1",
              "Nr4a1",
              "NR4A2",
              "Nr4a2",
              "NR4A3",
              "Nr4a3",
              "TCF7",
              "Tcf7",
              "TCF1",
              "Tcf1",
              "BACH2",
              "Bach2",
              "TBX21",
              "Tbx21",
              "TBET",
              "Tbet",
              "FLI1",
              "Fli1",
              "NR3C1",
              "Nr3c1",
              "EOMES",
              "Eomes",
              "ZEB2",
              "Zeb2",
              "ID2",
              "Id2",
              "ID3",
              "Id3",
              "IRF4",
              "Irf4"
            ]
          },
          {
            "name": "cf_node_character_hidden_driver_recall",
            "role": "hard_fail",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "hidden_driver_activity_not_expression"
          },
          {
            "name": "cf_effect_relation_hidden_pattern",
            "role": "hard_fail",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "activity_exceeds_expression"
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "causal_status",
            "kind": "value",
            "op": "in",
            "expected": [
              "sufficiency_established",
              "necessity_established",
              "causal_established_in_dataset",
              "sufficiency_via_perturbation_in_dataset"
            ]
          },
          {
            "name": "cf_residency_logic_activator_prior",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "residency_activator_induction_primary"
          },
          {
            "name": "cf_residency_logic_single_regulon",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_controls_entire_residency_program"
          },
          {
            "name": "cf_decoy_called_target",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_distractor_model_reported",
            "role": "hard_fail",
            "field": "$.trm_like_program_by_model[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_chromatin_only_decoy",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-20-beta-reasoning-pi",
      "modelName": "grok-4.20-beta-0309-reasoning",
      "harness": "pi",
      "task": "green_klf2_hidden_repressor_of_trm_like_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 959,
      "cost": 0.08195035000000002,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "regulator_gate",
            "role": "gate",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "KLF2",
              "Klf2"
            ]
          },
          {
            "name": "controlling_node_character_gate",
            "role": "gate",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "high_differential_expression"
          },
          {
            "name": "expression_vs_activity_gate",
            "role": "gate",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "expression_exceeds_activity"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "target_program_partition",
            "kind": "list",
            "matchKey": "gene",
            "k": 7,
            "expected": [
              {
                "key": "S1pr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "S1pr4",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Klrg1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Cx3cr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Itgae",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Hic1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "P2rx7",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              }
            ]
          },
          {
            "name": "causal_status_gate",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "association_only_sufficiency_requires_perturbation"
          },
          {
            "name": "residency_control_logic_gate",
            "role": "gate",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_release_necessary_not_sufficient_multi_regulon"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "trm_like_program_by_model",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_absent"
                    ],
                    "canonical": "niche_absent"
                  }
                }
              }
            ]
          },
          {
            "name": "cf_regulator_activator_or_wrong_node",
            "role": "hard_fail",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "RUNX3",
              "Runx3",
              "PRDM1",
              "Prdm1",
              "BLIMP1",
              "Blimp1",
              "BHLHE40",
              "Bhlhe40",
              "ZNF683",
              "Zfp683",
              "HOBIT",
              "Hobit",
              "AHR",
              "Ahr",
              "BATF",
              "Batf",
              "TOX",
              "Tox",
              "NR4A1",
              "Nr4a1",
              "NR4A2",
              "Nr4a2",
              "NR4A3",
              "Nr4a3",
              "TCF7",
              "Tcf7",
              "TCF1",
              "Tcf1",
              "BACH2",
              "Bach2",
              "TBX21",
              "Tbx21",
              "TBET",
              "Tbet",
              "FLI1",
              "Fli1",
              "NR3C1",
              "Nr3c1",
              "EOMES",
              "Eomes",
              "ZEB2",
              "Zeb2",
              "ID2",
              "Id2",
              "ID3",
              "Id3",
              "IRF4",
              "Irf4"
            ]
          },
          {
            "name": "cf_node_character_hidden_driver_recall",
            "role": "hard_fail",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "hidden_driver_activity_not_expression"
          },
          {
            "name": "cf_effect_relation_hidden_pattern",
            "role": "hard_fail",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "activity_exceeds_expression"
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "causal_status",
            "kind": "value",
            "op": "in",
            "expected": [
              "sufficiency_established",
              "necessity_established",
              "causal_established_in_dataset",
              "sufficiency_via_perturbation_in_dataset"
            ]
          },
          {
            "name": "cf_residency_logic_activator_prior",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "residency_activator_induction_primary"
          },
          {
            "name": "cf_residency_logic_single_regulon",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_controls_entire_residency_program"
          },
          {
            "name": "cf_decoy_called_target",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_distractor_model_reported",
            "role": "hard_fail",
            "field": "$.trm_like_program_by_model[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_chromatin_only_decoy",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-3-pi",
      "modelName": "grok-4.3",
      "harness": "pi",
      "task": "green_klf2_hidden_repressor_of_trm_like_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 356,
      "cost": 0.027374250000000003,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "regulator_gate",
            "role": "gate",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "KLF2",
              "Klf2"
            ]
          },
          {
            "name": "controlling_node_character_gate",
            "role": "gate",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "high_differential_expression"
          },
          {
            "name": "expression_vs_activity_gate",
            "role": "gate",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "expression_exceeds_activity"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "target_program_partition",
            "kind": "list",
            "matchKey": "gene",
            "k": 7,
            "expected": [
              {
                "key": "S1pr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "S1pr4",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Klrg1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Cx3cr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Itgae",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Hic1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "P2rx7",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              }
            ]
          },
          {
            "name": "causal_status_gate",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "association_only_sufficiency_requires_perturbation"
          },
          {
            "name": "residency_control_logic_gate",
            "role": "gate",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_release_necessary_not_sufficient_multi_regulon"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "trm_like_program_by_model",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_absent"
                    ],
                    "canonical": "niche_absent"
                  }
                }
              }
            ]
          },
          {
            "name": "cf_regulator_activator_or_wrong_node",
            "role": "hard_fail",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "RUNX3",
              "Runx3",
              "PRDM1",
              "Prdm1",
              "BLIMP1",
              "Blimp1",
              "BHLHE40",
              "Bhlhe40",
              "ZNF683",
              "Zfp683",
              "HOBIT",
              "Hobit",
              "AHR",
              "Ahr",
              "BATF",
              "Batf",
              "TOX",
              "Tox",
              "NR4A1",
              "Nr4a1",
              "NR4A2",
              "Nr4a2",
              "NR4A3",
              "Nr4a3",
              "TCF7",
              "Tcf7",
              "TCF1",
              "Tcf1",
              "BACH2",
              "Bach2",
              "TBX21",
              "Tbx21",
              "TBET",
              "Tbet",
              "FLI1",
              "Fli1",
              "NR3C1",
              "Nr3c1",
              "EOMES",
              "Eomes",
              "ZEB2",
              "Zeb2",
              "ID2",
              "Id2",
              "ID3",
              "Id3",
              "IRF4",
              "Irf4"
            ]
          },
          {
            "name": "cf_node_character_hidden_driver_recall",
            "role": "hard_fail",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "hidden_driver_activity_not_expression"
          },
          {
            "name": "cf_effect_relation_hidden_pattern",
            "role": "hard_fail",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "activity_exceeds_expression"
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "causal_status",
            "kind": "value",
            "op": "in",
            "expected": [
              "sufficiency_established",
              "necessity_established",
              "causal_established_in_dataset",
              "sufficiency_via_perturbation_in_dataset"
            ]
          },
          {
            "name": "cf_residency_logic_activator_prior",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "residency_activator_induction_primary"
          },
          {
            "name": "cf_residency_logic_single_regulon",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_controls_entire_residency_program"
          },
          {
            "name": "cf_decoy_called_target",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_distractor_model_reported",
            "role": "hard_fail",
            "field": "$.trm_like_program_by_model[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_chromatin_only_decoy",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-3-pi",
      "modelName": "grok-4.3",
      "harness": "pi",
      "task": "green_klf2_hidden_repressor_of_trm_like_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 222,
      "cost": 0.01428335,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "regulator_gate",
            "role": "gate",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "KLF2",
              "Klf2"
            ]
          },
          {
            "name": "controlling_node_character_gate",
            "role": "gate",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "high_differential_expression"
          },
          {
            "name": "expression_vs_activity_gate",
            "role": "gate",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "expression_exceeds_activity"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "target_program_partition",
            "kind": "list",
            "matchKey": "gene",
            "k": 7,
            "expected": [
              {
                "key": "S1pr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "S1pr4",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Klrg1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Cx3cr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Itgae",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Hic1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "P2rx7",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              }
            ]
          },
          {
            "name": "causal_status_gate",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "association_only_sufficiency_requires_perturbation"
          },
          {
            "name": "residency_control_logic_gate",
            "role": "gate",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_release_necessary_not_sufficient_multi_regulon"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "trm_like_program_by_model",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_absent"
                    ],
                    "canonical": "niche_absent"
                  }
                }
              }
            ]
          },
          {
            "name": "cf_regulator_activator_or_wrong_node",
            "role": "hard_fail",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "RUNX3",
              "Runx3",
              "PRDM1",
              "Prdm1",
              "BLIMP1",
              "Blimp1",
              "BHLHE40",
              "Bhlhe40",
              "ZNF683",
              "Zfp683",
              "HOBIT",
              "Hobit",
              "AHR",
              "Ahr",
              "BATF",
              "Batf",
              "TOX",
              "Tox",
              "NR4A1",
              "Nr4a1",
              "NR4A2",
              "Nr4a2",
              "NR4A3",
              "Nr4a3",
              "TCF7",
              "Tcf7",
              "TCF1",
              "Tcf1",
              "BACH2",
              "Bach2",
              "TBX21",
              "Tbx21",
              "TBET",
              "Tbet",
              "FLI1",
              "Fli1",
              "NR3C1",
              "Nr3c1",
              "EOMES",
              "Eomes",
              "ZEB2",
              "Zeb2",
              "ID2",
              "Id2",
              "ID3",
              "Id3",
              "IRF4",
              "Irf4"
            ]
          },
          {
            "name": "cf_node_character_hidden_driver_recall",
            "role": "hard_fail",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "hidden_driver_activity_not_expression"
          },
          {
            "name": "cf_effect_relation_hidden_pattern",
            "role": "hard_fail",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "activity_exceeds_expression"
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "causal_status",
            "kind": "value",
            "op": "in",
            "expected": [
              "sufficiency_established",
              "necessity_established",
              "causal_established_in_dataset",
              "sufficiency_via_perturbation_in_dataset"
            ]
          },
          {
            "name": "cf_residency_logic_activator_prior",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "residency_activator_induction_primary"
          },
          {
            "name": "cf_residency_logic_single_regulon",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_controls_entire_residency_program"
          },
          {
            "name": "cf_decoy_called_target",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_distractor_model_reported",
            "role": "hard_fail",
            "field": "$.trm_like_program_by_model[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_chromatin_only_decoy",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-3-pi",
      "modelName": "grok-4.3",
      "harness": "pi",
      "task": "green_klf2_hidden_repressor_of_trm_like_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 233,
      "cost": 0.015625150000000004,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "regulator_gate",
            "role": "gate",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "KLF2",
              "Klf2"
            ]
          },
          {
            "name": "controlling_node_character_gate",
            "role": "gate",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "high_differential_expression"
          },
          {
            "name": "expression_vs_activity_gate",
            "role": "gate",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "expression_exceeds_activity"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "target_program_partition",
            "kind": "list",
            "matchKey": "gene",
            "k": 7,
            "expected": [
              {
                "key": "S1pr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "S1pr4",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Klrg1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Cx3cr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Itgae",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Hic1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "P2rx7",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              }
            ]
          },
          {
            "name": "causal_status_gate",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "association_only_sufficiency_requires_perturbation"
          },
          {
            "name": "residency_control_logic_gate",
            "role": "gate",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_release_necessary_not_sufficient_multi_regulon"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "trm_like_program_by_model",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_absent"
                    ],
                    "canonical": "niche_absent"
                  }
                }
              }
            ]
          },
          {
            "name": "cf_regulator_activator_or_wrong_node",
            "role": "hard_fail",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "RUNX3",
              "Runx3",
              "PRDM1",
              "Prdm1",
              "BLIMP1",
              "Blimp1",
              "BHLHE40",
              "Bhlhe40",
              "ZNF683",
              "Zfp683",
              "HOBIT",
              "Hobit",
              "AHR",
              "Ahr",
              "BATF",
              "Batf",
              "TOX",
              "Tox",
              "NR4A1",
              "Nr4a1",
              "NR4A2",
              "Nr4a2",
              "NR4A3",
              "Nr4a3",
              "TCF7",
              "Tcf7",
              "TCF1",
              "Tcf1",
              "BACH2",
              "Bach2",
              "TBX21",
              "Tbx21",
              "TBET",
              "Tbet",
              "FLI1",
              "Fli1",
              "NR3C1",
              "Nr3c1",
              "EOMES",
              "Eomes",
              "ZEB2",
              "Zeb2",
              "ID2",
              "Id2",
              "ID3",
              "Id3",
              "IRF4",
              "Irf4"
            ]
          },
          {
            "name": "cf_node_character_hidden_driver_recall",
            "role": "hard_fail",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "hidden_driver_activity_not_expression"
          },
          {
            "name": "cf_effect_relation_hidden_pattern",
            "role": "hard_fail",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "activity_exceeds_expression"
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "causal_status",
            "kind": "value",
            "op": "in",
            "expected": [
              "sufficiency_established",
              "necessity_established",
              "causal_established_in_dataset",
              "sufficiency_via_perturbation_in_dataset"
            ]
          },
          {
            "name": "cf_residency_logic_activator_prior",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "residency_activator_induction_primary"
          },
          {
            "name": "cf_residency_logic_single_regulon",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_controls_entire_residency_program"
          },
          {
            "name": "cf_decoy_called_target",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_distractor_model_reported",
            "role": "hard_fail",
            "field": "$.trm_like_program_by_model[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_chromatin_only_decoy",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          }
        ]
      }
    },
    {
      "modelId": "sclong-kimi-k2p6-pi",
      "modelName": "kimi-k2p6",
      "harness": "pi",
      "task": "green_klf2_hidden_repressor_of_trm_like_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 3006,
      "cost": 0.7447050900000001,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "regulator_gate",
            "role": "gate",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "KLF2",
              "Klf2"
            ]
          },
          {
            "name": "controlling_node_character_gate",
            "role": "gate",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "high_differential_expression"
          },
          {
            "name": "expression_vs_activity_gate",
            "role": "gate",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "expression_exceeds_activity"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "target_program_partition",
            "kind": "list",
            "matchKey": "gene",
            "k": 7,
            "expected": [
              {
                "key": "S1pr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "S1pr4",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Klrg1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Cx3cr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Itgae",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Hic1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "P2rx7",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              }
            ]
          },
          {
            "name": "causal_status_gate",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "association_only_sufficiency_requires_perturbation"
          },
          {
            "name": "residency_control_logic_gate",
            "role": "gate",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_release_necessary_not_sufficient_multi_regulon"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "trm_like_program_by_model",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_absent"
                    ],
                    "canonical": "niche_absent"
                  }
                }
              }
            ]
          },
          {
            "name": "cf_regulator_activator_or_wrong_node",
            "role": "hard_fail",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "RUNX3",
              "Runx3",
              "PRDM1",
              "Prdm1",
              "BLIMP1",
              "Blimp1",
              "BHLHE40",
              "Bhlhe40",
              "ZNF683",
              "Zfp683",
              "HOBIT",
              "Hobit",
              "AHR",
              "Ahr",
              "BATF",
              "Batf",
              "TOX",
              "Tox",
              "NR4A1",
              "Nr4a1",
              "NR4A2",
              "Nr4a2",
              "NR4A3",
              "Nr4a3",
              "TCF7",
              "Tcf7",
              "TCF1",
              "Tcf1",
              "BACH2",
              "Bach2",
              "TBX21",
              "Tbx21",
              "TBET",
              "Tbet",
              "FLI1",
              "Fli1",
              "NR3C1",
              "Nr3c1",
              "EOMES",
              "Eomes",
              "ZEB2",
              "Zeb2",
              "ID2",
              "Id2",
              "ID3",
              "Id3",
              "IRF4",
              "Irf4"
            ]
          },
          {
            "name": "cf_node_character_hidden_driver_recall",
            "role": "hard_fail",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "hidden_driver_activity_not_expression"
          },
          {
            "name": "cf_effect_relation_hidden_pattern",
            "role": "hard_fail",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "activity_exceeds_expression"
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "causal_status",
            "kind": "value",
            "op": "in",
            "expected": [
              "sufficiency_established",
              "necessity_established",
              "causal_established_in_dataset",
              "sufficiency_via_perturbation_in_dataset"
            ]
          },
          {
            "name": "cf_residency_logic_activator_prior",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "residency_activator_induction_primary"
          },
          {
            "name": "cf_residency_logic_single_regulon",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_controls_entire_residency_program"
          },
          {
            "name": "cf_decoy_called_target",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_distractor_model_reported",
            "role": "hard_fail",
            "field": "$.trm_like_program_by_model[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_chromatin_only_decoy",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          }
        ]
      }
    },
    {
      "modelId": "sclong-kimi-k2p6-pi",
      "modelName": "kimi-k2p6",
      "harness": "pi",
      "task": "green_klf2_hidden_repressor_of_trm_like_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 1530,
      "cost": 0.5836037599999999,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "regulator_gate",
            "role": "gate",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "KLF2",
              "Klf2"
            ]
          },
          {
            "name": "controlling_node_character_gate",
            "role": "gate",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "high_differential_expression"
          },
          {
            "name": "expression_vs_activity_gate",
            "role": "gate",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "expression_exceeds_activity"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "target_program_partition",
            "kind": "list",
            "matchKey": "gene",
            "k": 7,
            "expected": [
              {
                "key": "S1pr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "S1pr4",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Klrg1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Cx3cr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Itgae",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Hic1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "P2rx7",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              }
            ]
          },
          {
            "name": "causal_status_gate",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "association_only_sufficiency_requires_perturbation"
          },
          {
            "name": "residency_control_logic_gate",
            "role": "gate",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_release_necessary_not_sufficient_multi_regulon"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "trm_like_program_by_model",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_absent"
                    ],
                    "canonical": "niche_absent"
                  }
                }
              }
            ]
          },
          {
            "name": "cf_regulator_activator_or_wrong_node",
            "role": "hard_fail",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "RUNX3",
              "Runx3",
              "PRDM1",
              "Prdm1",
              "BLIMP1",
              "Blimp1",
              "BHLHE40",
              "Bhlhe40",
              "ZNF683",
              "Zfp683",
              "HOBIT",
              "Hobit",
              "AHR",
              "Ahr",
              "BATF",
              "Batf",
              "TOX",
              "Tox",
              "NR4A1",
              "Nr4a1",
              "NR4A2",
              "Nr4a2",
              "NR4A3",
              "Nr4a3",
              "TCF7",
              "Tcf7",
              "TCF1",
              "Tcf1",
              "BACH2",
              "Bach2",
              "TBX21",
              "Tbx21",
              "TBET",
              "Tbet",
              "FLI1",
              "Fli1",
              "NR3C1",
              "Nr3c1",
              "EOMES",
              "Eomes",
              "ZEB2",
              "Zeb2",
              "ID2",
              "Id2",
              "ID3",
              "Id3",
              "IRF4",
              "Irf4"
            ]
          },
          {
            "name": "cf_node_character_hidden_driver_recall",
            "role": "hard_fail",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "hidden_driver_activity_not_expression"
          },
          {
            "name": "cf_effect_relation_hidden_pattern",
            "role": "hard_fail",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "activity_exceeds_expression"
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "causal_status",
            "kind": "value",
            "op": "in",
            "expected": [
              "sufficiency_established",
              "necessity_established",
              "causal_established_in_dataset",
              "sufficiency_via_perturbation_in_dataset"
            ]
          },
          {
            "name": "cf_residency_logic_activator_prior",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "residency_activator_induction_primary"
          },
          {
            "name": "cf_residency_logic_single_regulon",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_controls_entire_residency_program"
          },
          {
            "name": "cf_decoy_called_target",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_distractor_model_reported",
            "role": "hard_fail",
            "field": "$.trm_like_program_by_model[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_chromatin_only_decoy",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          }
        ]
      }
    },
    {
      "modelId": "sclong-kimi-k2p6-pi",
      "modelName": "kimi-k2p6",
      "harness": "pi",
      "task": "green_klf2_hidden_repressor_of_trm_like_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 4332,
      "cost": 1.70222753,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "regulator_gate",
            "role": "gate",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "KLF2",
              "Klf2"
            ]
          },
          {
            "name": "controlling_node_character_gate",
            "role": "gate",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "high_differential_expression"
          },
          {
            "name": "expression_vs_activity_gate",
            "role": "gate",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "expression_exceeds_activity"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "target_program_partition",
            "kind": "list",
            "matchKey": "gene",
            "k": 7,
            "expected": [
              {
                "key": "S1pr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "S1pr4",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Klrg1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Cx3cr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Itgae",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Hic1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "P2rx7",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              }
            ]
          },
          {
            "name": "causal_status_gate",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "association_only_sufficiency_requires_perturbation"
          },
          {
            "name": "residency_control_logic_gate",
            "role": "gate",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_release_necessary_not_sufficient_multi_regulon"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "trm_like_program_by_model",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_absent"
                    ],
                    "canonical": "niche_absent"
                  }
                }
              }
            ]
          },
          {
            "name": "cf_regulator_activator_or_wrong_node",
            "role": "hard_fail",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "RUNX3",
              "Runx3",
              "PRDM1",
              "Prdm1",
              "BLIMP1",
              "Blimp1",
              "BHLHE40",
              "Bhlhe40",
              "ZNF683",
              "Zfp683",
              "HOBIT",
              "Hobit",
              "AHR",
              "Ahr",
              "BATF",
              "Batf",
              "TOX",
              "Tox",
              "NR4A1",
              "Nr4a1",
              "NR4A2",
              "Nr4a2",
              "NR4A3",
              "Nr4a3",
              "TCF7",
              "Tcf7",
              "TCF1",
              "Tcf1",
              "BACH2",
              "Bach2",
              "TBX21",
              "Tbx21",
              "TBET",
              "Tbet",
              "FLI1",
              "Fli1",
              "NR3C1",
              "Nr3c1",
              "EOMES",
              "Eomes",
              "ZEB2",
              "Zeb2",
              "ID2",
              "Id2",
              "ID3",
              "Id3",
              "IRF4",
              "Irf4"
            ]
          },
          {
            "name": "cf_node_character_hidden_driver_recall",
            "role": "hard_fail",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "hidden_driver_activity_not_expression"
          },
          {
            "name": "cf_effect_relation_hidden_pattern",
            "role": "hard_fail",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "activity_exceeds_expression"
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "causal_status",
            "kind": "value",
            "op": "in",
            "expected": [
              "sufficiency_established",
              "necessity_established",
              "causal_established_in_dataset",
              "sufficiency_via_perturbation_in_dataset"
            ]
          },
          {
            "name": "cf_residency_logic_activator_prior",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "residency_activator_induction_primary"
          },
          {
            "name": "cf_residency_logic_single_regulon",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_controls_entire_residency_program"
          },
          {
            "name": "cf_decoy_called_target",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_distractor_model_reported",
            "role": "hard_fail",
            "field": "$.trm_like_program_by_model[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_chromatin_only_decoy",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-6-claude-code",
      "modelName": "claude-opus-4-6",
      "harness": "claude-code",
      "task": "green_shared_vs_distinct_regulators_trm_vs_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 1417,
      "cost": 1.7383642499999998,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_program_map",
            "kind": "list",
            "matchKey": "regulator",
            "k": 3,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "both"
                    ],
                    "canonical": "both"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "represses"
                    ],
                    "canonical": "represses"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Batf",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "trm_like_til_specific"
                    ],
                    "canonical": "trm_like_til_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "chromatin_over_rna"
                    ],
                    "canonical": "chromatin_over_rna"
                  }
                }
              },
              {
                "key": "Ahr",
                "fields": {
                  "program_specificity": {
                    "op": "in",
                    "values": [
                      "trm_specific",
                      "both"
                    ],
                    "canonical": "trm_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "rna_over_chromatin"
                    ],
                    "canonical": "rna_over_chromatin"
                  }
                }
              }
            ]
          },
          {
            "name": "regulator_set_no_extras_gate",
            "role": "gate",
            "field": "$.regulator_program_map[*]",
            "kind": "other",
            "expected": null
          },
          {
            "name": "program_identity_conclusion_gate",
            "role": "gate",
            "field": "program_identity_conclusion",
            "kind": "value",
            "expected": "distinct_state_shared_brake_divergent_activators"
          },
          {
            "name": "residency_install_logic_gate",
            "role": "gate",
            "field": "residency_install_logic",
            "kind": "value",
            "expected": "brake_release_permits_context_matched_activator"
          },
          {
            "name": "activator_context_dependence_gate",
            "role": "gate",
            "field": "activator_context_dependence",
            "kind": "value",
            "expected": "context_matched_crossover"
          },
          {
            "name": "evidence_ceiling_gate",
            "role": "gate",
            "field": "evidence_ceiling",
            "kind": "value",
            "expected": "specificity_dissociations_require_perturbation"
          },
          {
            "name": "cf_install_logic_flat_or_single_step",
            "role": "hard_fail",
            "field": "residency_install_logic",
            "kind": "value",
            "op": "in",
            "expected": [
              "flat_shared_residency_toolkit",
              "activator_induction_sufficient",
              "brake_release_alone_sufficient"
            ]
          },
          {
            "name": "cf_program_identity_lumped_or_wrong",
            "role": "hard_fail",
            "field": "program_identity_conclusion",
            "kind": "value",
            "op": "in",
            "expected": [
              "trm_like_til_is_tumor_trm",
              "trm_like_til_is_exhausted_subset",
              "trm_like_til_is_effector_state",
              "trm_like_til_is_transitional_intermediate",
              "trm_like_til_is_shared_program_distinct_niche"
            ]
          },
          {
            "name": "cf_ahr_assigned_to_til",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Ahr",
            "valueField": "program_specificity",
            "expected": [
              "trm_like_til_specific"
            ]
          },
          {
            "name": "cf_batf_assigned_to_trm_or_shared",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Batf",
            "valueField": "program_specificity",
            "expected": [
              "trm_specific",
              "both"
            ]
          },
          {
            "name": "cf_klf2_called_activator",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Klf2",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          },
          {
            "name": "cf_evidence_ceiling_overclaim",
            "role": "hard_fail",
            "field": "evidence_ceiling",
            "kind": "value",
            "op": "in",
            "expected": [
              "specificity_established_from_regulon_activity",
              "specificity_established_in_dataset",
              "causation_established_by_cross_sectional_concordance",
              "necessity_and_sufficiency_established"
            ]
          },
          {
            "name": "cf_activator_context_shared_or_single",
            "role": "hard_fail",
            "field": "activator_context_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "shared_activators_both_contexts",
              "single_global_activator",
              "redundant_activators"
            ]
          },
          {
            "name": "cf_single_modality_rule",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_hic1_motif_brake_mislabel",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_runx3_anticanonical_as_controller",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Runx3",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-6-claude-code",
      "modelName": "claude-opus-4-6",
      "harness": "claude-code",
      "task": "green_shared_vs_distinct_regulators_trm_vs_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 1570,
      "cost": 2.3805105,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_program_map",
            "kind": "list",
            "matchKey": "regulator",
            "k": 3,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "both"
                    ],
                    "canonical": "both"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "represses"
                    ],
                    "canonical": "represses"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Batf",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "trm_like_til_specific"
                    ],
                    "canonical": "trm_like_til_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "chromatin_over_rna"
                    ],
                    "canonical": "chromatin_over_rna"
                  }
                }
              },
              {
                "key": "Ahr",
                "fields": {
                  "program_specificity": {
                    "op": "in",
                    "values": [
                      "trm_specific",
                      "both"
                    ],
                    "canonical": "trm_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "rna_over_chromatin"
                    ],
                    "canonical": "rna_over_chromatin"
                  }
                }
              }
            ]
          },
          {
            "name": "regulator_set_no_extras_gate",
            "role": "gate",
            "field": "$.regulator_program_map[*]",
            "kind": "other",
            "expected": null
          },
          {
            "name": "program_identity_conclusion_gate",
            "role": "gate",
            "field": "program_identity_conclusion",
            "kind": "value",
            "expected": "distinct_state_shared_brake_divergent_activators"
          },
          {
            "name": "residency_install_logic_gate",
            "role": "gate",
            "field": "residency_install_logic",
            "kind": "value",
            "expected": "brake_release_permits_context_matched_activator"
          },
          {
            "name": "activator_context_dependence_gate",
            "role": "gate",
            "field": "activator_context_dependence",
            "kind": "value",
            "expected": "context_matched_crossover"
          },
          {
            "name": "evidence_ceiling_gate",
            "role": "gate",
            "field": "evidence_ceiling",
            "kind": "value",
            "expected": "specificity_dissociations_require_perturbation"
          },
          {
            "name": "cf_install_logic_flat_or_single_step",
            "role": "hard_fail",
            "field": "residency_install_logic",
            "kind": "value",
            "op": "in",
            "expected": [
              "flat_shared_residency_toolkit",
              "activator_induction_sufficient",
              "brake_release_alone_sufficient"
            ]
          },
          {
            "name": "cf_program_identity_lumped_or_wrong",
            "role": "hard_fail",
            "field": "program_identity_conclusion",
            "kind": "value",
            "op": "in",
            "expected": [
              "trm_like_til_is_tumor_trm",
              "trm_like_til_is_exhausted_subset",
              "trm_like_til_is_effector_state",
              "trm_like_til_is_transitional_intermediate",
              "trm_like_til_is_shared_program_distinct_niche"
            ]
          },
          {
            "name": "cf_ahr_assigned_to_til",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Ahr",
            "valueField": "program_specificity",
            "expected": [
              "trm_like_til_specific"
            ]
          },
          {
            "name": "cf_batf_assigned_to_trm_or_shared",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Batf",
            "valueField": "program_specificity",
            "expected": [
              "trm_specific",
              "both"
            ]
          },
          {
            "name": "cf_klf2_called_activator",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Klf2",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          },
          {
            "name": "cf_evidence_ceiling_overclaim",
            "role": "hard_fail",
            "field": "evidence_ceiling",
            "kind": "value",
            "op": "in",
            "expected": [
              "specificity_established_from_regulon_activity",
              "specificity_established_in_dataset",
              "causation_established_by_cross_sectional_concordance",
              "necessity_and_sufficiency_established"
            ]
          },
          {
            "name": "cf_activator_context_shared_or_single",
            "role": "hard_fail",
            "field": "activator_context_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "shared_activators_both_contexts",
              "single_global_activator",
              "redundant_activators"
            ]
          },
          {
            "name": "cf_single_modality_rule",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_hic1_motif_brake_mislabel",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_runx3_anticanonical_as_controller",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Runx3",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-6-claude-code",
      "modelName": "claude-opus-4-6",
      "harness": "claude-code",
      "task": "green_shared_vs_distinct_regulators_trm_vs_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1554,
      "cost": 2.069407,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_program_map",
            "kind": "list",
            "matchKey": "regulator",
            "k": 3,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "both"
                    ],
                    "canonical": "both"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "represses"
                    ],
                    "canonical": "represses"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Batf",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "trm_like_til_specific"
                    ],
                    "canonical": "trm_like_til_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "chromatin_over_rna"
                    ],
                    "canonical": "chromatin_over_rna"
                  }
                }
              },
              {
                "key": "Ahr",
                "fields": {
                  "program_specificity": {
                    "op": "in",
                    "values": [
                      "trm_specific",
                      "both"
                    ],
                    "canonical": "trm_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "rna_over_chromatin"
                    ],
                    "canonical": "rna_over_chromatin"
                  }
                }
              }
            ]
          },
          {
            "name": "regulator_set_no_extras_gate",
            "role": "gate",
            "field": "$.regulator_program_map[*]",
            "kind": "other",
            "expected": null
          },
          {
            "name": "program_identity_conclusion_gate",
            "role": "gate",
            "field": "program_identity_conclusion",
            "kind": "value",
            "expected": "distinct_state_shared_brake_divergent_activators"
          },
          {
            "name": "residency_install_logic_gate",
            "role": "gate",
            "field": "residency_install_logic",
            "kind": "value",
            "expected": "brake_release_permits_context_matched_activator"
          },
          {
            "name": "activator_context_dependence_gate",
            "role": "gate",
            "field": "activator_context_dependence",
            "kind": "value",
            "expected": "context_matched_crossover"
          },
          {
            "name": "evidence_ceiling_gate",
            "role": "gate",
            "field": "evidence_ceiling",
            "kind": "value",
            "expected": "specificity_dissociations_require_perturbation"
          },
          {
            "name": "cf_install_logic_flat_or_single_step",
            "role": "hard_fail",
            "field": "residency_install_logic",
            "kind": "value",
            "op": "in",
            "expected": [
              "flat_shared_residency_toolkit",
              "activator_induction_sufficient",
              "brake_release_alone_sufficient"
            ]
          },
          {
            "name": "cf_program_identity_lumped_or_wrong",
            "role": "hard_fail",
            "field": "program_identity_conclusion",
            "kind": "value",
            "op": "in",
            "expected": [
              "trm_like_til_is_tumor_trm",
              "trm_like_til_is_exhausted_subset",
              "trm_like_til_is_effector_state",
              "trm_like_til_is_transitional_intermediate",
              "trm_like_til_is_shared_program_distinct_niche"
            ]
          },
          {
            "name": "cf_ahr_assigned_to_til",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Ahr",
            "valueField": "program_specificity",
            "expected": [
              "trm_like_til_specific"
            ]
          },
          {
            "name": "cf_batf_assigned_to_trm_or_shared",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Batf",
            "valueField": "program_specificity",
            "expected": [
              "trm_specific",
              "both"
            ]
          },
          {
            "name": "cf_klf2_called_activator",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Klf2",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          },
          {
            "name": "cf_evidence_ceiling_overclaim",
            "role": "hard_fail",
            "field": "evidence_ceiling",
            "kind": "value",
            "op": "in",
            "expected": [
              "specificity_established_from_regulon_activity",
              "specificity_established_in_dataset",
              "causation_established_by_cross_sectional_concordance",
              "necessity_and_sufficiency_established"
            ]
          },
          {
            "name": "cf_activator_context_shared_or_single",
            "role": "hard_fail",
            "field": "activator_context_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "shared_activators_both_contexts",
              "single_global_activator",
              "redundant_activators"
            ]
          },
          {
            "name": "cf_single_modality_rule",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_hic1_motif_brake_mislabel",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_runx3_anticanonical_as_controller",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Runx3",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-6-pi",
      "modelName": "claude-opus-4-6",
      "harness": "pi",
      "task": "green_shared_vs_distinct_regulators_trm_vs_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 4106,
      "cost": 2.6619855000000006,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_program_map",
            "kind": "list",
            "matchKey": "regulator",
            "k": 3,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "both"
                    ],
                    "canonical": "both"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "represses"
                    ],
                    "canonical": "represses"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Batf",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "trm_like_til_specific"
                    ],
                    "canonical": "trm_like_til_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "chromatin_over_rna"
                    ],
                    "canonical": "chromatin_over_rna"
                  }
                }
              },
              {
                "key": "Ahr",
                "fields": {
                  "program_specificity": {
                    "op": "in",
                    "values": [
                      "trm_specific",
                      "both"
                    ],
                    "canonical": "trm_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "rna_over_chromatin"
                    ],
                    "canonical": "rna_over_chromatin"
                  }
                }
              }
            ]
          },
          {
            "name": "regulator_set_no_extras_gate",
            "role": "gate",
            "field": "$.regulator_program_map[*]",
            "kind": "other",
            "expected": null
          },
          {
            "name": "program_identity_conclusion_gate",
            "role": "gate",
            "field": "program_identity_conclusion",
            "kind": "value",
            "expected": "distinct_state_shared_brake_divergent_activators"
          },
          {
            "name": "residency_install_logic_gate",
            "role": "gate",
            "field": "residency_install_logic",
            "kind": "value",
            "expected": "brake_release_permits_context_matched_activator"
          },
          {
            "name": "activator_context_dependence_gate",
            "role": "gate",
            "field": "activator_context_dependence",
            "kind": "value",
            "expected": "context_matched_crossover"
          },
          {
            "name": "evidence_ceiling_gate",
            "role": "gate",
            "field": "evidence_ceiling",
            "kind": "value",
            "expected": "specificity_dissociations_require_perturbation"
          },
          {
            "name": "cf_install_logic_flat_or_single_step",
            "role": "hard_fail",
            "field": "residency_install_logic",
            "kind": "value",
            "op": "in",
            "expected": [
              "flat_shared_residency_toolkit",
              "activator_induction_sufficient",
              "brake_release_alone_sufficient"
            ]
          },
          {
            "name": "cf_program_identity_lumped_or_wrong",
            "role": "hard_fail",
            "field": "program_identity_conclusion",
            "kind": "value",
            "op": "in",
            "expected": [
              "trm_like_til_is_tumor_trm",
              "trm_like_til_is_exhausted_subset",
              "trm_like_til_is_effector_state",
              "trm_like_til_is_transitional_intermediate",
              "trm_like_til_is_shared_program_distinct_niche"
            ]
          },
          {
            "name": "cf_ahr_assigned_to_til",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Ahr",
            "valueField": "program_specificity",
            "expected": [
              "trm_like_til_specific"
            ]
          },
          {
            "name": "cf_batf_assigned_to_trm_or_shared",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Batf",
            "valueField": "program_specificity",
            "expected": [
              "trm_specific",
              "both"
            ]
          },
          {
            "name": "cf_klf2_called_activator",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Klf2",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          },
          {
            "name": "cf_evidence_ceiling_overclaim",
            "role": "hard_fail",
            "field": "evidence_ceiling",
            "kind": "value",
            "op": "in",
            "expected": [
              "specificity_established_from_regulon_activity",
              "specificity_established_in_dataset",
              "causation_established_by_cross_sectional_concordance",
              "necessity_and_sufficiency_established"
            ]
          },
          {
            "name": "cf_activator_context_shared_or_single",
            "role": "hard_fail",
            "field": "activator_context_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "shared_activators_both_contexts",
              "single_global_activator",
              "redundant_activators"
            ]
          },
          {
            "name": "cf_single_modality_rule",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_hic1_motif_brake_mislabel",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_runx3_anticanonical_as_controller",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Runx3",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-6-pi",
      "modelName": "claude-opus-4-6",
      "harness": "pi",
      "task": "green_shared_vs_distinct_regulators_trm_vs_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 2151,
      "cost": 2.3105007500000005,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_program_map",
            "kind": "list",
            "matchKey": "regulator",
            "k": 3,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "both"
                    ],
                    "canonical": "both"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "represses"
                    ],
                    "canonical": "represses"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Batf",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "trm_like_til_specific"
                    ],
                    "canonical": "trm_like_til_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "chromatin_over_rna"
                    ],
                    "canonical": "chromatin_over_rna"
                  }
                }
              },
              {
                "key": "Ahr",
                "fields": {
                  "program_specificity": {
                    "op": "in",
                    "values": [
                      "trm_specific",
                      "both"
                    ],
                    "canonical": "trm_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "rna_over_chromatin"
                    ],
                    "canonical": "rna_over_chromatin"
                  }
                }
              }
            ]
          },
          {
            "name": "regulator_set_no_extras_gate",
            "role": "gate",
            "field": "$.regulator_program_map[*]",
            "kind": "other",
            "expected": null
          },
          {
            "name": "program_identity_conclusion_gate",
            "role": "gate",
            "field": "program_identity_conclusion",
            "kind": "value",
            "expected": "distinct_state_shared_brake_divergent_activators"
          },
          {
            "name": "residency_install_logic_gate",
            "role": "gate",
            "field": "residency_install_logic",
            "kind": "value",
            "expected": "brake_release_permits_context_matched_activator"
          },
          {
            "name": "activator_context_dependence_gate",
            "role": "gate",
            "field": "activator_context_dependence",
            "kind": "value",
            "expected": "context_matched_crossover"
          },
          {
            "name": "evidence_ceiling_gate",
            "role": "gate",
            "field": "evidence_ceiling",
            "kind": "value",
            "expected": "specificity_dissociations_require_perturbation"
          },
          {
            "name": "cf_install_logic_flat_or_single_step",
            "role": "hard_fail",
            "field": "residency_install_logic",
            "kind": "value",
            "op": "in",
            "expected": [
              "flat_shared_residency_toolkit",
              "activator_induction_sufficient",
              "brake_release_alone_sufficient"
            ]
          },
          {
            "name": "cf_program_identity_lumped_or_wrong",
            "role": "hard_fail",
            "field": "program_identity_conclusion",
            "kind": "value",
            "op": "in",
            "expected": [
              "trm_like_til_is_tumor_trm",
              "trm_like_til_is_exhausted_subset",
              "trm_like_til_is_effector_state",
              "trm_like_til_is_transitional_intermediate",
              "trm_like_til_is_shared_program_distinct_niche"
            ]
          },
          {
            "name": "cf_ahr_assigned_to_til",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Ahr",
            "valueField": "program_specificity",
            "expected": [
              "trm_like_til_specific"
            ]
          },
          {
            "name": "cf_batf_assigned_to_trm_or_shared",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Batf",
            "valueField": "program_specificity",
            "expected": [
              "trm_specific",
              "both"
            ]
          },
          {
            "name": "cf_klf2_called_activator",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Klf2",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          },
          {
            "name": "cf_evidence_ceiling_overclaim",
            "role": "hard_fail",
            "field": "evidence_ceiling",
            "kind": "value",
            "op": "in",
            "expected": [
              "specificity_established_from_regulon_activity",
              "specificity_established_in_dataset",
              "causation_established_by_cross_sectional_concordance",
              "necessity_and_sufficiency_established"
            ]
          },
          {
            "name": "cf_activator_context_shared_or_single",
            "role": "hard_fail",
            "field": "activator_context_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "shared_activators_both_contexts",
              "single_global_activator",
              "redundant_activators"
            ]
          },
          {
            "name": "cf_single_modality_rule",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_hic1_motif_brake_mislabel",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_runx3_anticanonical_as_controller",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Runx3",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-6-pi",
      "modelName": "claude-opus-4-6",
      "harness": "pi",
      "task": "green_shared_vs_distinct_regulators_trm_vs_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 2494,
      "cost": 2.1532915,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_program_map",
            "kind": "list",
            "matchKey": "regulator",
            "k": 3,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "both"
                    ],
                    "canonical": "both"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "represses"
                    ],
                    "canonical": "represses"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Batf",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "trm_like_til_specific"
                    ],
                    "canonical": "trm_like_til_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "chromatin_over_rna"
                    ],
                    "canonical": "chromatin_over_rna"
                  }
                }
              },
              {
                "key": "Ahr",
                "fields": {
                  "program_specificity": {
                    "op": "in",
                    "values": [
                      "trm_specific",
                      "both"
                    ],
                    "canonical": "trm_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "rna_over_chromatin"
                    ],
                    "canonical": "rna_over_chromatin"
                  }
                }
              }
            ]
          },
          {
            "name": "regulator_set_no_extras_gate",
            "role": "gate",
            "field": "$.regulator_program_map[*]",
            "kind": "other",
            "expected": null
          },
          {
            "name": "program_identity_conclusion_gate",
            "role": "gate",
            "field": "program_identity_conclusion",
            "kind": "value",
            "expected": "distinct_state_shared_brake_divergent_activators"
          },
          {
            "name": "residency_install_logic_gate",
            "role": "gate",
            "field": "residency_install_logic",
            "kind": "value",
            "expected": "brake_release_permits_context_matched_activator"
          },
          {
            "name": "activator_context_dependence_gate",
            "role": "gate",
            "field": "activator_context_dependence",
            "kind": "value",
            "expected": "context_matched_crossover"
          },
          {
            "name": "evidence_ceiling_gate",
            "role": "gate",
            "field": "evidence_ceiling",
            "kind": "value",
            "expected": "specificity_dissociations_require_perturbation"
          },
          {
            "name": "cf_install_logic_flat_or_single_step",
            "role": "hard_fail",
            "field": "residency_install_logic",
            "kind": "value",
            "op": "in",
            "expected": [
              "flat_shared_residency_toolkit",
              "activator_induction_sufficient",
              "brake_release_alone_sufficient"
            ]
          },
          {
            "name": "cf_program_identity_lumped_or_wrong",
            "role": "hard_fail",
            "field": "program_identity_conclusion",
            "kind": "value",
            "op": "in",
            "expected": [
              "trm_like_til_is_tumor_trm",
              "trm_like_til_is_exhausted_subset",
              "trm_like_til_is_effector_state",
              "trm_like_til_is_transitional_intermediate",
              "trm_like_til_is_shared_program_distinct_niche"
            ]
          },
          {
            "name": "cf_ahr_assigned_to_til",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Ahr",
            "valueField": "program_specificity",
            "expected": [
              "trm_like_til_specific"
            ]
          },
          {
            "name": "cf_batf_assigned_to_trm_or_shared",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Batf",
            "valueField": "program_specificity",
            "expected": [
              "trm_specific",
              "both"
            ]
          },
          {
            "name": "cf_klf2_called_activator",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Klf2",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          },
          {
            "name": "cf_evidence_ceiling_overclaim",
            "role": "hard_fail",
            "field": "evidence_ceiling",
            "kind": "value",
            "op": "in",
            "expected": [
              "specificity_established_from_regulon_activity",
              "specificity_established_in_dataset",
              "causation_established_by_cross_sectional_concordance",
              "necessity_and_sufficiency_established"
            ]
          },
          {
            "name": "cf_activator_context_shared_or_single",
            "role": "hard_fail",
            "field": "activator_context_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "shared_activators_both_contexts",
              "single_global_activator",
              "redundant_activators"
            ]
          },
          {
            "name": "cf_single_modality_rule",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_hic1_motif_brake_mislabel",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_runx3_anticanonical_as_controller",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Runx3",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-7-claude-code",
      "modelName": "claude-opus-4-7",
      "harness": "claude-code",
      "task": "green_shared_vs_distinct_regulators_trm_vs_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 849,
      "cost": 2.71493,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_program_map",
            "kind": "list",
            "matchKey": "regulator",
            "k": 3,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "both"
                    ],
                    "canonical": "both"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "represses"
                    ],
                    "canonical": "represses"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Batf",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "trm_like_til_specific"
                    ],
                    "canonical": "trm_like_til_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "chromatin_over_rna"
                    ],
                    "canonical": "chromatin_over_rna"
                  }
                }
              },
              {
                "key": "Ahr",
                "fields": {
                  "program_specificity": {
                    "op": "in",
                    "values": [
                      "trm_specific",
                      "both"
                    ],
                    "canonical": "trm_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "rna_over_chromatin"
                    ],
                    "canonical": "rna_over_chromatin"
                  }
                }
              }
            ]
          },
          {
            "name": "regulator_set_no_extras_gate",
            "role": "gate",
            "field": "$.regulator_program_map[*]",
            "kind": "other",
            "expected": null
          },
          {
            "name": "program_identity_conclusion_gate",
            "role": "gate",
            "field": "program_identity_conclusion",
            "kind": "value",
            "expected": "distinct_state_shared_brake_divergent_activators"
          },
          {
            "name": "residency_install_logic_gate",
            "role": "gate",
            "field": "residency_install_logic",
            "kind": "value",
            "expected": "brake_release_permits_context_matched_activator"
          },
          {
            "name": "activator_context_dependence_gate",
            "role": "gate",
            "field": "activator_context_dependence",
            "kind": "value",
            "expected": "context_matched_crossover"
          },
          {
            "name": "evidence_ceiling_gate",
            "role": "gate",
            "field": "evidence_ceiling",
            "kind": "value",
            "expected": "specificity_dissociations_require_perturbation"
          },
          {
            "name": "cf_install_logic_flat_or_single_step",
            "role": "hard_fail",
            "field": "residency_install_logic",
            "kind": "value",
            "op": "in",
            "expected": [
              "flat_shared_residency_toolkit",
              "activator_induction_sufficient",
              "brake_release_alone_sufficient"
            ]
          },
          {
            "name": "cf_program_identity_lumped_or_wrong",
            "role": "hard_fail",
            "field": "program_identity_conclusion",
            "kind": "value",
            "op": "in",
            "expected": [
              "trm_like_til_is_tumor_trm",
              "trm_like_til_is_exhausted_subset",
              "trm_like_til_is_effector_state",
              "trm_like_til_is_transitional_intermediate",
              "trm_like_til_is_shared_program_distinct_niche"
            ]
          },
          {
            "name": "cf_ahr_assigned_to_til",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Ahr",
            "valueField": "program_specificity",
            "expected": [
              "trm_like_til_specific"
            ]
          },
          {
            "name": "cf_batf_assigned_to_trm_or_shared",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Batf",
            "valueField": "program_specificity",
            "expected": [
              "trm_specific",
              "both"
            ]
          },
          {
            "name": "cf_klf2_called_activator",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Klf2",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          },
          {
            "name": "cf_evidence_ceiling_overclaim",
            "role": "hard_fail",
            "field": "evidence_ceiling",
            "kind": "value",
            "op": "in",
            "expected": [
              "specificity_established_from_regulon_activity",
              "specificity_established_in_dataset",
              "causation_established_by_cross_sectional_concordance",
              "necessity_and_sufficiency_established"
            ]
          },
          {
            "name": "cf_activator_context_shared_or_single",
            "role": "hard_fail",
            "field": "activator_context_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "shared_activators_both_contexts",
              "single_global_activator",
              "redundant_activators"
            ]
          },
          {
            "name": "cf_single_modality_rule",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_hic1_motif_brake_mislabel",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_runx3_anticanonical_as_controller",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Runx3",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-7-claude-code",
      "modelName": "claude-opus-4-7",
      "harness": "claude-code",
      "task": "green_shared_vs_distinct_regulators_trm_vs_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 1468,
      "cost": 3.9205547499999995,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_program_map",
            "kind": "list",
            "matchKey": "regulator",
            "k": 3,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "both"
                    ],
                    "canonical": "both"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "represses"
                    ],
                    "canonical": "represses"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Batf",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "trm_like_til_specific"
                    ],
                    "canonical": "trm_like_til_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "chromatin_over_rna"
                    ],
                    "canonical": "chromatin_over_rna"
                  }
                }
              },
              {
                "key": "Ahr",
                "fields": {
                  "program_specificity": {
                    "op": "in",
                    "values": [
                      "trm_specific",
                      "both"
                    ],
                    "canonical": "trm_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "rna_over_chromatin"
                    ],
                    "canonical": "rna_over_chromatin"
                  }
                }
              }
            ]
          },
          {
            "name": "regulator_set_no_extras_gate",
            "role": "gate",
            "field": "$.regulator_program_map[*]",
            "kind": "other",
            "expected": null
          },
          {
            "name": "program_identity_conclusion_gate",
            "role": "gate",
            "field": "program_identity_conclusion",
            "kind": "value",
            "expected": "distinct_state_shared_brake_divergent_activators"
          },
          {
            "name": "residency_install_logic_gate",
            "role": "gate",
            "field": "residency_install_logic",
            "kind": "value",
            "expected": "brake_release_permits_context_matched_activator"
          },
          {
            "name": "activator_context_dependence_gate",
            "role": "gate",
            "field": "activator_context_dependence",
            "kind": "value",
            "expected": "context_matched_crossover"
          },
          {
            "name": "evidence_ceiling_gate",
            "role": "gate",
            "field": "evidence_ceiling",
            "kind": "value",
            "expected": "specificity_dissociations_require_perturbation"
          },
          {
            "name": "cf_install_logic_flat_or_single_step",
            "role": "hard_fail",
            "field": "residency_install_logic",
            "kind": "value",
            "op": "in",
            "expected": [
              "flat_shared_residency_toolkit",
              "activator_induction_sufficient",
              "brake_release_alone_sufficient"
            ]
          },
          {
            "name": "cf_program_identity_lumped_or_wrong",
            "role": "hard_fail",
            "field": "program_identity_conclusion",
            "kind": "value",
            "op": "in",
            "expected": [
              "trm_like_til_is_tumor_trm",
              "trm_like_til_is_exhausted_subset",
              "trm_like_til_is_effector_state",
              "trm_like_til_is_transitional_intermediate",
              "trm_like_til_is_shared_program_distinct_niche"
            ]
          },
          {
            "name": "cf_ahr_assigned_to_til",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Ahr",
            "valueField": "program_specificity",
            "expected": [
              "trm_like_til_specific"
            ]
          },
          {
            "name": "cf_batf_assigned_to_trm_or_shared",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Batf",
            "valueField": "program_specificity",
            "expected": [
              "trm_specific",
              "both"
            ]
          },
          {
            "name": "cf_klf2_called_activator",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Klf2",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          },
          {
            "name": "cf_evidence_ceiling_overclaim",
            "role": "hard_fail",
            "field": "evidence_ceiling",
            "kind": "value",
            "op": "in",
            "expected": [
              "specificity_established_from_regulon_activity",
              "specificity_established_in_dataset",
              "causation_established_by_cross_sectional_concordance",
              "necessity_and_sufficiency_established"
            ]
          },
          {
            "name": "cf_activator_context_shared_or_single",
            "role": "hard_fail",
            "field": "activator_context_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "shared_activators_both_contexts",
              "single_global_activator",
              "redundant_activators"
            ]
          },
          {
            "name": "cf_single_modality_rule",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_hic1_motif_brake_mislabel",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_runx3_anticanonical_as_controller",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Runx3",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-7-claude-code",
      "modelName": "claude-opus-4-7",
      "harness": "claude-code",
      "task": "green_shared_vs_distinct_regulators_trm_vs_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1789,
      "cost": 5.592284500000001,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_program_map",
            "kind": "list",
            "matchKey": "regulator",
            "k": 3,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "both"
                    ],
                    "canonical": "both"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "represses"
                    ],
                    "canonical": "represses"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Batf",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "trm_like_til_specific"
                    ],
                    "canonical": "trm_like_til_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "chromatin_over_rna"
                    ],
                    "canonical": "chromatin_over_rna"
                  }
                }
              },
              {
                "key": "Ahr",
                "fields": {
                  "program_specificity": {
                    "op": "in",
                    "values": [
                      "trm_specific",
                      "both"
                    ],
                    "canonical": "trm_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "rna_over_chromatin"
                    ],
                    "canonical": "rna_over_chromatin"
                  }
                }
              }
            ]
          },
          {
            "name": "regulator_set_no_extras_gate",
            "role": "gate",
            "field": "$.regulator_program_map[*]",
            "kind": "other",
            "expected": null
          },
          {
            "name": "program_identity_conclusion_gate",
            "role": "gate",
            "field": "program_identity_conclusion",
            "kind": "value",
            "expected": "distinct_state_shared_brake_divergent_activators"
          },
          {
            "name": "residency_install_logic_gate",
            "role": "gate",
            "field": "residency_install_logic",
            "kind": "value",
            "expected": "brake_release_permits_context_matched_activator"
          },
          {
            "name": "activator_context_dependence_gate",
            "role": "gate",
            "field": "activator_context_dependence",
            "kind": "value",
            "expected": "context_matched_crossover"
          },
          {
            "name": "evidence_ceiling_gate",
            "role": "gate",
            "field": "evidence_ceiling",
            "kind": "value",
            "expected": "specificity_dissociations_require_perturbation"
          },
          {
            "name": "cf_install_logic_flat_or_single_step",
            "role": "hard_fail",
            "field": "residency_install_logic",
            "kind": "value",
            "op": "in",
            "expected": [
              "flat_shared_residency_toolkit",
              "activator_induction_sufficient",
              "brake_release_alone_sufficient"
            ]
          },
          {
            "name": "cf_program_identity_lumped_or_wrong",
            "role": "hard_fail",
            "field": "program_identity_conclusion",
            "kind": "value",
            "op": "in",
            "expected": [
              "trm_like_til_is_tumor_trm",
              "trm_like_til_is_exhausted_subset",
              "trm_like_til_is_effector_state",
              "trm_like_til_is_transitional_intermediate",
              "trm_like_til_is_shared_program_distinct_niche"
            ]
          },
          {
            "name": "cf_ahr_assigned_to_til",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Ahr",
            "valueField": "program_specificity",
            "expected": [
              "trm_like_til_specific"
            ]
          },
          {
            "name": "cf_batf_assigned_to_trm_or_shared",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Batf",
            "valueField": "program_specificity",
            "expected": [
              "trm_specific",
              "both"
            ]
          },
          {
            "name": "cf_klf2_called_activator",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Klf2",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          },
          {
            "name": "cf_evidence_ceiling_overclaim",
            "role": "hard_fail",
            "field": "evidence_ceiling",
            "kind": "value",
            "op": "in",
            "expected": [
              "specificity_established_from_regulon_activity",
              "specificity_established_in_dataset",
              "causation_established_by_cross_sectional_concordance",
              "necessity_and_sufficiency_established"
            ]
          },
          {
            "name": "cf_activator_context_shared_or_single",
            "role": "hard_fail",
            "field": "activator_context_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "shared_activators_both_contexts",
              "single_global_activator",
              "redundant_activators"
            ]
          },
          {
            "name": "cf_single_modality_rule",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_hic1_motif_brake_mislabel",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_runx3_anticanonical_as_controller",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Runx3",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-7-pi",
      "modelName": "claude-opus-4-7",
      "harness": "pi",
      "task": "green_shared_vs_distinct_regulators_trm_vs_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 2710,
      "cost": 4.50714425,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_program_map",
            "kind": "list",
            "matchKey": "regulator",
            "k": 3,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "both"
                    ],
                    "canonical": "both"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "represses"
                    ],
                    "canonical": "represses"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Batf",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "trm_like_til_specific"
                    ],
                    "canonical": "trm_like_til_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "chromatin_over_rna"
                    ],
                    "canonical": "chromatin_over_rna"
                  }
                }
              },
              {
                "key": "Ahr",
                "fields": {
                  "program_specificity": {
                    "op": "in",
                    "values": [
                      "trm_specific",
                      "both"
                    ],
                    "canonical": "trm_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "rna_over_chromatin"
                    ],
                    "canonical": "rna_over_chromatin"
                  }
                }
              }
            ]
          },
          {
            "name": "regulator_set_no_extras_gate",
            "role": "gate",
            "field": "$.regulator_program_map[*]",
            "kind": "other",
            "expected": null
          },
          {
            "name": "program_identity_conclusion_gate",
            "role": "gate",
            "field": "program_identity_conclusion",
            "kind": "value",
            "expected": "distinct_state_shared_brake_divergent_activators"
          },
          {
            "name": "residency_install_logic_gate",
            "role": "gate",
            "field": "residency_install_logic",
            "kind": "value",
            "expected": "brake_release_permits_context_matched_activator"
          },
          {
            "name": "activator_context_dependence_gate",
            "role": "gate",
            "field": "activator_context_dependence",
            "kind": "value",
            "expected": "context_matched_crossover"
          },
          {
            "name": "evidence_ceiling_gate",
            "role": "gate",
            "field": "evidence_ceiling",
            "kind": "value",
            "expected": "specificity_dissociations_require_perturbation"
          },
          {
            "name": "cf_install_logic_flat_or_single_step",
            "role": "hard_fail",
            "field": "residency_install_logic",
            "kind": "value",
            "op": "in",
            "expected": [
              "flat_shared_residency_toolkit",
              "activator_induction_sufficient",
              "brake_release_alone_sufficient"
            ]
          },
          {
            "name": "cf_program_identity_lumped_or_wrong",
            "role": "hard_fail",
            "field": "program_identity_conclusion",
            "kind": "value",
            "op": "in",
            "expected": [
              "trm_like_til_is_tumor_trm",
              "trm_like_til_is_exhausted_subset",
              "trm_like_til_is_effector_state",
              "trm_like_til_is_transitional_intermediate",
              "trm_like_til_is_shared_program_distinct_niche"
            ]
          },
          {
            "name": "cf_ahr_assigned_to_til",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Ahr",
            "valueField": "program_specificity",
            "expected": [
              "trm_like_til_specific"
            ]
          },
          {
            "name": "cf_batf_assigned_to_trm_or_shared",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Batf",
            "valueField": "program_specificity",
            "expected": [
              "trm_specific",
              "both"
            ]
          },
          {
            "name": "cf_klf2_called_activator",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Klf2",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          },
          {
            "name": "cf_evidence_ceiling_overclaim",
            "role": "hard_fail",
            "field": "evidence_ceiling",
            "kind": "value",
            "op": "in",
            "expected": [
              "specificity_established_from_regulon_activity",
              "specificity_established_in_dataset",
              "causation_established_by_cross_sectional_concordance",
              "necessity_and_sufficiency_established"
            ]
          },
          {
            "name": "cf_activator_context_shared_or_single",
            "role": "hard_fail",
            "field": "activator_context_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "shared_activators_both_contexts",
              "single_global_activator",
              "redundant_activators"
            ]
          },
          {
            "name": "cf_single_modality_rule",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_hic1_motif_brake_mislabel",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_runx3_anticanonical_as_controller",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Runx3",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-7-pi",
      "modelName": "claude-opus-4-7",
      "harness": "pi",
      "task": "green_shared_vs_distinct_regulators_trm_vs_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 1898,
      "cost": 2.6665400000000004,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_program_map",
            "kind": "list",
            "matchKey": "regulator",
            "k": 3,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "both"
                    ],
                    "canonical": "both"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "represses"
                    ],
                    "canonical": "represses"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Batf",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "trm_like_til_specific"
                    ],
                    "canonical": "trm_like_til_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "chromatin_over_rna"
                    ],
                    "canonical": "chromatin_over_rna"
                  }
                }
              },
              {
                "key": "Ahr",
                "fields": {
                  "program_specificity": {
                    "op": "in",
                    "values": [
                      "trm_specific",
                      "both"
                    ],
                    "canonical": "trm_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "rna_over_chromatin"
                    ],
                    "canonical": "rna_over_chromatin"
                  }
                }
              }
            ]
          },
          {
            "name": "regulator_set_no_extras_gate",
            "role": "gate",
            "field": "$.regulator_program_map[*]",
            "kind": "other",
            "expected": null
          },
          {
            "name": "program_identity_conclusion_gate",
            "role": "gate",
            "field": "program_identity_conclusion",
            "kind": "value",
            "expected": "distinct_state_shared_brake_divergent_activators"
          },
          {
            "name": "residency_install_logic_gate",
            "role": "gate",
            "field": "residency_install_logic",
            "kind": "value",
            "expected": "brake_release_permits_context_matched_activator"
          },
          {
            "name": "activator_context_dependence_gate",
            "role": "gate",
            "field": "activator_context_dependence",
            "kind": "value",
            "expected": "context_matched_crossover"
          },
          {
            "name": "evidence_ceiling_gate",
            "role": "gate",
            "field": "evidence_ceiling",
            "kind": "value",
            "expected": "specificity_dissociations_require_perturbation"
          },
          {
            "name": "cf_install_logic_flat_or_single_step",
            "role": "hard_fail",
            "field": "residency_install_logic",
            "kind": "value",
            "op": "in",
            "expected": [
              "flat_shared_residency_toolkit",
              "activator_induction_sufficient",
              "brake_release_alone_sufficient"
            ]
          },
          {
            "name": "cf_program_identity_lumped_or_wrong",
            "role": "hard_fail",
            "field": "program_identity_conclusion",
            "kind": "value",
            "op": "in",
            "expected": [
              "trm_like_til_is_tumor_trm",
              "trm_like_til_is_exhausted_subset",
              "trm_like_til_is_effector_state",
              "trm_like_til_is_transitional_intermediate",
              "trm_like_til_is_shared_program_distinct_niche"
            ]
          },
          {
            "name": "cf_ahr_assigned_to_til",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Ahr",
            "valueField": "program_specificity",
            "expected": [
              "trm_like_til_specific"
            ]
          },
          {
            "name": "cf_batf_assigned_to_trm_or_shared",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Batf",
            "valueField": "program_specificity",
            "expected": [
              "trm_specific",
              "both"
            ]
          },
          {
            "name": "cf_klf2_called_activator",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Klf2",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          },
          {
            "name": "cf_evidence_ceiling_overclaim",
            "role": "hard_fail",
            "field": "evidence_ceiling",
            "kind": "value",
            "op": "in",
            "expected": [
              "specificity_established_from_regulon_activity",
              "specificity_established_in_dataset",
              "causation_established_by_cross_sectional_concordance",
              "necessity_and_sufficiency_established"
            ]
          },
          {
            "name": "cf_activator_context_shared_or_single",
            "role": "hard_fail",
            "field": "activator_context_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "shared_activators_both_contexts",
              "single_global_activator",
              "redundant_activators"
            ]
          },
          {
            "name": "cf_single_modality_rule",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_hic1_motif_brake_mislabel",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_runx3_anticanonical_as_controller",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Runx3",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-7-pi",
      "modelName": "claude-opus-4-7",
      "harness": "pi",
      "task": "green_shared_vs_distinct_regulators_trm_vs_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 6001,
      "cost": 4.59812775,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_program_map",
            "kind": "list",
            "matchKey": "regulator",
            "k": 3,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "both"
                    ],
                    "canonical": "both"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "represses"
                    ],
                    "canonical": "represses"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Batf",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "trm_like_til_specific"
                    ],
                    "canonical": "trm_like_til_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "chromatin_over_rna"
                    ],
                    "canonical": "chromatin_over_rna"
                  }
                }
              },
              {
                "key": "Ahr",
                "fields": {
                  "program_specificity": {
                    "op": "in",
                    "values": [
                      "trm_specific",
                      "both"
                    ],
                    "canonical": "trm_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "rna_over_chromatin"
                    ],
                    "canonical": "rna_over_chromatin"
                  }
                }
              }
            ]
          },
          {
            "name": "regulator_set_no_extras_gate",
            "role": "gate",
            "field": "$.regulator_program_map[*]",
            "kind": "other",
            "expected": null
          },
          {
            "name": "program_identity_conclusion_gate",
            "role": "gate",
            "field": "program_identity_conclusion",
            "kind": "value",
            "expected": "distinct_state_shared_brake_divergent_activators"
          },
          {
            "name": "residency_install_logic_gate",
            "role": "gate",
            "field": "residency_install_logic",
            "kind": "value",
            "expected": "brake_release_permits_context_matched_activator"
          },
          {
            "name": "activator_context_dependence_gate",
            "role": "gate",
            "field": "activator_context_dependence",
            "kind": "value",
            "expected": "context_matched_crossover"
          },
          {
            "name": "evidence_ceiling_gate",
            "role": "gate",
            "field": "evidence_ceiling",
            "kind": "value",
            "expected": "specificity_dissociations_require_perturbation"
          },
          {
            "name": "cf_install_logic_flat_or_single_step",
            "role": "hard_fail",
            "field": "residency_install_logic",
            "kind": "value",
            "op": "in",
            "expected": [
              "flat_shared_residency_toolkit",
              "activator_induction_sufficient",
              "brake_release_alone_sufficient"
            ]
          },
          {
            "name": "cf_program_identity_lumped_or_wrong",
            "role": "hard_fail",
            "field": "program_identity_conclusion",
            "kind": "value",
            "op": "in",
            "expected": [
              "trm_like_til_is_tumor_trm",
              "trm_like_til_is_exhausted_subset",
              "trm_like_til_is_effector_state",
              "trm_like_til_is_transitional_intermediate",
              "trm_like_til_is_shared_program_distinct_niche"
            ]
          },
          {
            "name": "cf_ahr_assigned_to_til",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Ahr",
            "valueField": "program_specificity",
            "expected": [
              "trm_like_til_specific"
            ]
          },
          {
            "name": "cf_batf_assigned_to_trm_or_shared",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Batf",
            "valueField": "program_specificity",
            "expected": [
              "trm_specific",
              "both"
            ]
          },
          {
            "name": "cf_klf2_called_activator",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Klf2",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          },
          {
            "name": "cf_evidence_ceiling_overclaim",
            "role": "hard_fail",
            "field": "evidence_ceiling",
            "kind": "value",
            "op": "in",
            "expected": [
              "specificity_established_from_regulon_activity",
              "specificity_established_in_dataset",
              "causation_established_by_cross_sectional_concordance",
              "necessity_and_sufficiency_established"
            ]
          },
          {
            "name": "cf_activator_context_shared_or_single",
            "role": "hard_fail",
            "field": "activator_context_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "shared_activators_both_contexts",
              "single_global_activator",
              "redundant_activators"
            ]
          },
          {
            "name": "cf_single_modality_rule",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_hic1_motif_brake_mislabel",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_runx3_anticanonical_as_controller",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Runx3",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-8-claude-code",
      "modelName": "claude-opus-4-8",
      "harness": "claude-code",
      "task": "green_shared_vs_distinct_regulators_trm_vs_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 2863,
      "cost": 9.273847000000002,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_program_map",
            "kind": "list",
            "matchKey": "regulator",
            "k": 3,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "both"
                    ],
                    "canonical": "both"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "represses"
                    ],
                    "canonical": "represses"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Batf",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "trm_like_til_specific"
                    ],
                    "canonical": "trm_like_til_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "chromatin_over_rna"
                    ],
                    "canonical": "chromatin_over_rna"
                  }
                }
              },
              {
                "key": "Ahr",
                "fields": {
                  "program_specificity": {
                    "op": "in",
                    "values": [
                      "trm_specific",
                      "both"
                    ],
                    "canonical": "trm_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "rna_over_chromatin"
                    ],
                    "canonical": "rna_over_chromatin"
                  }
                }
              }
            ]
          },
          {
            "name": "regulator_set_no_extras_gate",
            "role": "gate",
            "field": "$.regulator_program_map[*]",
            "kind": "other",
            "expected": null
          },
          {
            "name": "program_identity_conclusion_gate",
            "role": "gate",
            "field": "program_identity_conclusion",
            "kind": "value",
            "expected": "distinct_state_shared_brake_divergent_activators"
          },
          {
            "name": "residency_install_logic_gate",
            "role": "gate",
            "field": "residency_install_logic",
            "kind": "value",
            "expected": "brake_release_permits_context_matched_activator"
          },
          {
            "name": "activator_context_dependence_gate",
            "role": "gate",
            "field": "activator_context_dependence",
            "kind": "value",
            "expected": "context_matched_crossover"
          },
          {
            "name": "evidence_ceiling_gate",
            "role": "gate",
            "field": "evidence_ceiling",
            "kind": "value",
            "expected": "specificity_dissociations_require_perturbation"
          },
          {
            "name": "cf_install_logic_flat_or_single_step",
            "role": "hard_fail",
            "field": "residency_install_logic",
            "kind": "value",
            "op": "in",
            "expected": [
              "flat_shared_residency_toolkit",
              "activator_induction_sufficient",
              "brake_release_alone_sufficient"
            ]
          },
          {
            "name": "cf_program_identity_lumped_or_wrong",
            "role": "hard_fail",
            "field": "program_identity_conclusion",
            "kind": "value",
            "op": "in",
            "expected": [
              "trm_like_til_is_tumor_trm",
              "trm_like_til_is_exhausted_subset",
              "trm_like_til_is_effector_state",
              "trm_like_til_is_transitional_intermediate",
              "trm_like_til_is_shared_program_distinct_niche"
            ]
          },
          {
            "name": "cf_ahr_assigned_to_til",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Ahr",
            "valueField": "program_specificity",
            "expected": [
              "trm_like_til_specific"
            ]
          },
          {
            "name": "cf_batf_assigned_to_trm_or_shared",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Batf",
            "valueField": "program_specificity",
            "expected": [
              "trm_specific",
              "both"
            ]
          },
          {
            "name": "cf_klf2_called_activator",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Klf2",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          },
          {
            "name": "cf_evidence_ceiling_overclaim",
            "role": "hard_fail",
            "field": "evidence_ceiling",
            "kind": "value",
            "op": "in",
            "expected": [
              "specificity_established_from_regulon_activity",
              "specificity_established_in_dataset",
              "causation_established_by_cross_sectional_concordance",
              "necessity_and_sufficiency_established"
            ]
          },
          {
            "name": "cf_activator_context_shared_or_single",
            "role": "hard_fail",
            "field": "activator_context_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "shared_activators_both_contexts",
              "single_global_activator",
              "redundant_activators"
            ]
          },
          {
            "name": "cf_single_modality_rule",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_hic1_motif_brake_mislabel",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_runx3_anticanonical_as_controller",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Runx3",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-8-claude-code",
      "modelName": "claude-opus-4-8",
      "harness": "claude-code",
      "task": "green_shared_vs_distinct_regulators_trm_vs_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 3493,
      "cost": 11.633500000000003,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_program_map",
            "kind": "list",
            "matchKey": "regulator",
            "k": 3,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "both"
                    ],
                    "canonical": "both"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "represses"
                    ],
                    "canonical": "represses"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Batf",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "trm_like_til_specific"
                    ],
                    "canonical": "trm_like_til_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "chromatin_over_rna"
                    ],
                    "canonical": "chromatin_over_rna"
                  }
                }
              },
              {
                "key": "Ahr",
                "fields": {
                  "program_specificity": {
                    "op": "in",
                    "values": [
                      "trm_specific",
                      "both"
                    ],
                    "canonical": "trm_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "rna_over_chromatin"
                    ],
                    "canonical": "rna_over_chromatin"
                  }
                }
              }
            ]
          },
          {
            "name": "regulator_set_no_extras_gate",
            "role": "gate",
            "field": "$.regulator_program_map[*]",
            "kind": "other",
            "expected": null
          },
          {
            "name": "program_identity_conclusion_gate",
            "role": "gate",
            "field": "program_identity_conclusion",
            "kind": "value",
            "expected": "distinct_state_shared_brake_divergent_activators"
          },
          {
            "name": "residency_install_logic_gate",
            "role": "gate",
            "field": "residency_install_logic",
            "kind": "value",
            "expected": "brake_release_permits_context_matched_activator"
          },
          {
            "name": "activator_context_dependence_gate",
            "role": "gate",
            "field": "activator_context_dependence",
            "kind": "value",
            "expected": "context_matched_crossover"
          },
          {
            "name": "evidence_ceiling_gate",
            "role": "gate",
            "field": "evidence_ceiling",
            "kind": "value",
            "expected": "specificity_dissociations_require_perturbation"
          },
          {
            "name": "cf_install_logic_flat_or_single_step",
            "role": "hard_fail",
            "field": "residency_install_logic",
            "kind": "value",
            "op": "in",
            "expected": [
              "flat_shared_residency_toolkit",
              "activator_induction_sufficient",
              "brake_release_alone_sufficient"
            ]
          },
          {
            "name": "cf_program_identity_lumped_or_wrong",
            "role": "hard_fail",
            "field": "program_identity_conclusion",
            "kind": "value",
            "op": "in",
            "expected": [
              "trm_like_til_is_tumor_trm",
              "trm_like_til_is_exhausted_subset",
              "trm_like_til_is_effector_state",
              "trm_like_til_is_transitional_intermediate",
              "trm_like_til_is_shared_program_distinct_niche"
            ]
          },
          {
            "name": "cf_ahr_assigned_to_til",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Ahr",
            "valueField": "program_specificity",
            "expected": [
              "trm_like_til_specific"
            ]
          },
          {
            "name": "cf_batf_assigned_to_trm_or_shared",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Batf",
            "valueField": "program_specificity",
            "expected": [
              "trm_specific",
              "both"
            ]
          },
          {
            "name": "cf_klf2_called_activator",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Klf2",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          },
          {
            "name": "cf_evidence_ceiling_overclaim",
            "role": "hard_fail",
            "field": "evidence_ceiling",
            "kind": "value",
            "op": "in",
            "expected": [
              "specificity_established_from_regulon_activity",
              "specificity_established_in_dataset",
              "causation_established_by_cross_sectional_concordance",
              "necessity_and_sufficiency_established"
            ]
          },
          {
            "name": "cf_activator_context_shared_or_single",
            "role": "hard_fail",
            "field": "activator_context_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "shared_activators_both_contexts",
              "single_global_activator",
              "redundant_activators"
            ]
          },
          {
            "name": "cf_single_modality_rule",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_hic1_motif_brake_mislabel",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_runx3_anticanonical_as_controller",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Runx3",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-8-claude-code",
      "modelName": "claude-opus-4-8",
      "harness": "claude-code",
      "task": "green_shared_vs_distinct_regulators_trm_vs_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 3253,
      "cost": 9.0457485,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_program_map",
            "kind": "list",
            "matchKey": "regulator",
            "k": 3,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "both"
                    ],
                    "canonical": "both"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "represses"
                    ],
                    "canonical": "represses"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Batf",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "trm_like_til_specific"
                    ],
                    "canonical": "trm_like_til_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "chromatin_over_rna"
                    ],
                    "canonical": "chromatin_over_rna"
                  }
                }
              },
              {
                "key": "Ahr",
                "fields": {
                  "program_specificity": {
                    "op": "in",
                    "values": [
                      "trm_specific",
                      "both"
                    ],
                    "canonical": "trm_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "rna_over_chromatin"
                    ],
                    "canonical": "rna_over_chromatin"
                  }
                }
              }
            ]
          },
          {
            "name": "regulator_set_no_extras_gate",
            "role": "gate",
            "field": "$.regulator_program_map[*]",
            "kind": "other",
            "expected": null
          },
          {
            "name": "program_identity_conclusion_gate",
            "role": "gate",
            "field": "program_identity_conclusion",
            "kind": "value",
            "expected": "distinct_state_shared_brake_divergent_activators"
          },
          {
            "name": "residency_install_logic_gate",
            "role": "gate",
            "field": "residency_install_logic",
            "kind": "value",
            "expected": "brake_release_permits_context_matched_activator"
          },
          {
            "name": "activator_context_dependence_gate",
            "role": "gate",
            "field": "activator_context_dependence",
            "kind": "value",
            "expected": "context_matched_crossover"
          },
          {
            "name": "evidence_ceiling_gate",
            "role": "gate",
            "field": "evidence_ceiling",
            "kind": "value",
            "expected": "specificity_dissociations_require_perturbation"
          },
          {
            "name": "cf_install_logic_flat_or_single_step",
            "role": "hard_fail",
            "field": "residency_install_logic",
            "kind": "value",
            "op": "in",
            "expected": [
              "flat_shared_residency_toolkit",
              "activator_induction_sufficient",
              "brake_release_alone_sufficient"
            ]
          },
          {
            "name": "cf_program_identity_lumped_or_wrong",
            "role": "hard_fail",
            "field": "program_identity_conclusion",
            "kind": "value",
            "op": "in",
            "expected": [
              "trm_like_til_is_tumor_trm",
              "trm_like_til_is_exhausted_subset",
              "trm_like_til_is_effector_state",
              "trm_like_til_is_transitional_intermediate",
              "trm_like_til_is_shared_program_distinct_niche"
            ]
          },
          {
            "name": "cf_ahr_assigned_to_til",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Ahr",
            "valueField": "program_specificity",
            "expected": [
              "trm_like_til_specific"
            ]
          },
          {
            "name": "cf_batf_assigned_to_trm_or_shared",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Batf",
            "valueField": "program_specificity",
            "expected": [
              "trm_specific",
              "both"
            ]
          },
          {
            "name": "cf_klf2_called_activator",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Klf2",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          },
          {
            "name": "cf_evidence_ceiling_overclaim",
            "role": "hard_fail",
            "field": "evidence_ceiling",
            "kind": "value",
            "op": "in",
            "expected": [
              "specificity_established_from_regulon_activity",
              "specificity_established_in_dataset",
              "causation_established_by_cross_sectional_concordance",
              "necessity_and_sufficiency_established"
            ]
          },
          {
            "name": "cf_activator_context_shared_or_single",
            "role": "hard_fail",
            "field": "activator_context_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "shared_activators_both_contexts",
              "single_global_activator",
              "redundant_activators"
            ]
          },
          {
            "name": "cf_single_modality_rule",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_hic1_motif_brake_mislabel",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_runx3_anticanonical_as_controller",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Runx3",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-8-pi",
      "modelName": "claude-opus-4-8",
      "harness": "pi",
      "task": "green_shared_vs_distinct_regulators_trm_vs_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 4829,
      "cost": 7.733942000000001,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_program_map",
            "kind": "list",
            "matchKey": "regulator",
            "k": 3,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "both"
                    ],
                    "canonical": "both"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "represses"
                    ],
                    "canonical": "represses"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Batf",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "trm_like_til_specific"
                    ],
                    "canonical": "trm_like_til_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "chromatin_over_rna"
                    ],
                    "canonical": "chromatin_over_rna"
                  }
                }
              },
              {
                "key": "Ahr",
                "fields": {
                  "program_specificity": {
                    "op": "in",
                    "values": [
                      "trm_specific",
                      "both"
                    ],
                    "canonical": "trm_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "rna_over_chromatin"
                    ],
                    "canonical": "rna_over_chromatin"
                  }
                }
              }
            ]
          },
          {
            "name": "regulator_set_no_extras_gate",
            "role": "gate",
            "field": "$.regulator_program_map[*]",
            "kind": "other",
            "expected": null
          },
          {
            "name": "program_identity_conclusion_gate",
            "role": "gate",
            "field": "program_identity_conclusion",
            "kind": "value",
            "expected": "distinct_state_shared_brake_divergent_activators"
          },
          {
            "name": "residency_install_logic_gate",
            "role": "gate",
            "field": "residency_install_logic",
            "kind": "value",
            "expected": "brake_release_permits_context_matched_activator"
          },
          {
            "name": "activator_context_dependence_gate",
            "role": "gate",
            "field": "activator_context_dependence",
            "kind": "value",
            "expected": "context_matched_crossover"
          },
          {
            "name": "evidence_ceiling_gate",
            "role": "gate",
            "field": "evidence_ceiling",
            "kind": "value",
            "expected": "specificity_dissociations_require_perturbation"
          },
          {
            "name": "cf_install_logic_flat_or_single_step",
            "role": "hard_fail",
            "field": "residency_install_logic",
            "kind": "value",
            "op": "in",
            "expected": [
              "flat_shared_residency_toolkit",
              "activator_induction_sufficient",
              "brake_release_alone_sufficient"
            ]
          },
          {
            "name": "cf_program_identity_lumped_or_wrong",
            "role": "hard_fail",
            "field": "program_identity_conclusion",
            "kind": "value",
            "op": "in",
            "expected": [
              "trm_like_til_is_tumor_trm",
              "trm_like_til_is_exhausted_subset",
              "trm_like_til_is_effector_state",
              "trm_like_til_is_transitional_intermediate",
              "trm_like_til_is_shared_program_distinct_niche"
            ]
          },
          {
            "name": "cf_ahr_assigned_to_til",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Ahr",
            "valueField": "program_specificity",
            "expected": [
              "trm_like_til_specific"
            ]
          },
          {
            "name": "cf_batf_assigned_to_trm_or_shared",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Batf",
            "valueField": "program_specificity",
            "expected": [
              "trm_specific",
              "both"
            ]
          },
          {
            "name": "cf_klf2_called_activator",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Klf2",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          },
          {
            "name": "cf_evidence_ceiling_overclaim",
            "role": "hard_fail",
            "field": "evidence_ceiling",
            "kind": "value",
            "op": "in",
            "expected": [
              "specificity_established_from_regulon_activity",
              "specificity_established_in_dataset",
              "causation_established_by_cross_sectional_concordance",
              "necessity_and_sufficiency_established"
            ]
          },
          {
            "name": "cf_activator_context_shared_or_single",
            "role": "hard_fail",
            "field": "activator_context_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "shared_activators_both_contexts",
              "single_global_activator",
              "redundant_activators"
            ]
          },
          {
            "name": "cf_single_modality_rule",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_hic1_motif_brake_mislabel",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_runx3_anticanonical_as_controller",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Runx3",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-8-pi",
      "modelName": "claude-opus-4-8",
      "harness": "pi",
      "task": "green_shared_vs_distinct_regulators_trm_vs_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 4381,
      "cost": 7.94044575,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_program_map",
            "kind": "list",
            "matchKey": "regulator",
            "k": 3,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "both"
                    ],
                    "canonical": "both"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "represses"
                    ],
                    "canonical": "represses"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Batf",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "trm_like_til_specific"
                    ],
                    "canonical": "trm_like_til_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "chromatin_over_rna"
                    ],
                    "canonical": "chromatin_over_rna"
                  }
                }
              },
              {
                "key": "Ahr",
                "fields": {
                  "program_specificity": {
                    "op": "in",
                    "values": [
                      "trm_specific",
                      "both"
                    ],
                    "canonical": "trm_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "rna_over_chromatin"
                    ],
                    "canonical": "rna_over_chromatin"
                  }
                }
              }
            ]
          },
          {
            "name": "regulator_set_no_extras_gate",
            "role": "gate",
            "field": "$.regulator_program_map[*]",
            "kind": "other",
            "expected": null
          },
          {
            "name": "program_identity_conclusion_gate",
            "role": "gate",
            "field": "program_identity_conclusion",
            "kind": "value",
            "expected": "distinct_state_shared_brake_divergent_activators"
          },
          {
            "name": "residency_install_logic_gate",
            "role": "gate",
            "field": "residency_install_logic",
            "kind": "value",
            "expected": "brake_release_permits_context_matched_activator"
          },
          {
            "name": "activator_context_dependence_gate",
            "role": "gate",
            "field": "activator_context_dependence",
            "kind": "value",
            "expected": "context_matched_crossover"
          },
          {
            "name": "evidence_ceiling_gate",
            "role": "gate",
            "field": "evidence_ceiling",
            "kind": "value",
            "expected": "specificity_dissociations_require_perturbation"
          },
          {
            "name": "cf_install_logic_flat_or_single_step",
            "role": "hard_fail",
            "field": "residency_install_logic",
            "kind": "value",
            "op": "in",
            "expected": [
              "flat_shared_residency_toolkit",
              "activator_induction_sufficient",
              "brake_release_alone_sufficient"
            ]
          },
          {
            "name": "cf_program_identity_lumped_or_wrong",
            "role": "hard_fail",
            "field": "program_identity_conclusion",
            "kind": "value",
            "op": "in",
            "expected": [
              "trm_like_til_is_tumor_trm",
              "trm_like_til_is_exhausted_subset",
              "trm_like_til_is_effector_state",
              "trm_like_til_is_transitional_intermediate",
              "trm_like_til_is_shared_program_distinct_niche"
            ]
          },
          {
            "name": "cf_ahr_assigned_to_til",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Ahr",
            "valueField": "program_specificity",
            "expected": [
              "trm_like_til_specific"
            ]
          },
          {
            "name": "cf_batf_assigned_to_trm_or_shared",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Batf",
            "valueField": "program_specificity",
            "expected": [
              "trm_specific",
              "both"
            ]
          },
          {
            "name": "cf_klf2_called_activator",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Klf2",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          },
          {
            "name": "cf_evidence_ceiling_overclaim",
            "role": "hard_fail",
            "field": "evidence_ceiling",
            "kind": "value",
            "op": "in",
            "expected": [
              "specificity_established_from_regulon_activity",
              "specificity_established_in_dataset",
              "causation_established_by_cross_sectional_concordance",
              "necessity_and_sufficiency_established"
            ]
          },
          {
            "name": "cf_activator_context_shared_or_single",
            "role": "hard_fail",
            "field": "activator_context_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "shared_activators_both_contexts",
              "single_global_activator",
              "redundant_activators"
            ]
          },
          {
            "name": "cf_single_modality_rule",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_hic1_motif_brake_mislabel",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_runx3_anticanonical_as_controller",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Runx3",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-8-pi",
      "modelName": "claude-opus-4-8",
      "harness": "pi",
      "task": "green_shared_vs_distinct_regulators_trm_vs_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 13050,
      "cost": 12.66089375,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_program_map",
            "kind": "list",
            "matchKey": "regulator",
            "k": 3,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "both"
                    ],
                    "canonical": "both"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "represses"
                    ],
                    "canonical": "represses"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Batf",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "trm_like_til_specific"
                    ],
                    "canonical": "trm_like_til_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "chromatin_over_rna"
                    ],
                    "canonical": "chromatin_over_rna"
                  }
                }
              },
              {
                "key": "Ahr",
                "fields": {
                  "program_specificity": {
                    "op": "in",
                    "values": [
                      "trm_specific",
                      "both"
                    ],
                    "canonical": "trm_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "rna_over_chromatin"
                    ],
                    "canonical": "rna_over_chromatin"
                  }
                }
              }
            ]
          },
          {
            "name": "regulator_set_no_extras_gate",
            "role": "gate",
            "field": "$.regulator_program_map[*]",
            "kind": "other",
            "expected": null
          },
          {
            "name": "program_identity_conclusion_gate",
            "role": "gate",
            "field": "program_identity_conclusion",
            "kind": "value",
            "expected": "distinct_state_shared_brake_divergent_activators"
          },
          {
            "name": "residency_install_logic_gate",
            "role": "gate",
            "field": "residency_install_logic",
            "kind": "value",
            "expected": "brake_release_permits_context_matched_activator"
          },
          {
            "name": "activator_context_dependence_gate",
            "role": "gate",
            "field": "activator_context_dependence",
            "kind": "value",
            "expected": "context_matched_crossover"
          },
          {
            "name": "evidence_ceiling_gate",
            "role": "gate",
            "field": "evidence_ceiling",
            "kind": "value",
            "expected": "specificity_dissociations_require_perturbation"
          },
          {
            "name": "cf_install_logic_flat_or_single_step",
            "role": "hard_fail",
            "field": "residency_install_logic",
            "kind": "value",
            "op": "in",
            "expected": [
              "flat_shared_residency_toolkit",
              "activator_induction_sufficient",
              "brake_release_alone_sufficient"
            ]
          },
          {
            "name": "cf_program_identity_lumped_or_wrong",
            "role": "hard_fail",
            "field": "program_identity_conclusion",
            "kind": "value",
            "op": "in",
            "expected": [
              "trm_like_til_is_tumor_trm",
              "trm_like_til_is_exhausted_subset",
              "trm_like_til_is_effector_state",
              "trm_like_til_is_transitional_intermediate",
              "trm_like_til_is_shared_program_distinct_niche"
            ]
          },
          {
            "name": "cf_ahr_assigned_to_til",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Ahr",
            "valueField": "program_specificity",
            "expected": [
              "trm_like_til_specific"
            ]
          },
          {
            "name": "cf_batf_assigned_to_trm_or_shared",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Batf",
            "valueField": "program_specificity",
            "expected": [
              "trm_specific",
              "both"
            ]
          },
          {
            "name": "cf_klf2_called_activator",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Klf2",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          },
          {
            "name": "cf_evidence_ceiling_overclaim",
            "role": "hard_fail",
            "field": "evidence_ceiling",
            "kind": "value",
            "op": "in",
            "expected": [
              "specificity_established_from_regulon_activity",
              "specificity_established_in_dataset",
              "causation_established_by_cross_sectional_concordance",
              "necessity_and_sufficiency_established"
            ]
          },
          {
            "name": "cf_activator_context_shared_or_single",
            "role": "hard_fail",
            "field": "activator_context_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "shared_activators_both_contexts",
              "single_global_activator",
              "redundant_activators"
            ]
          },
          {
            "name": "cf_single_modality_rule",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_hic1_motif_brake_mislabel",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_runx3_anticanonical_as_controller",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Runx3",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-4-6-pi",
      "modelName": "claude-sonnet-4-6",
      "harness": "pi",
      "task": "green_shared_vs_distinct_regulators_trm_vs_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 6145,
      "cost": 4.48809405,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_program_map",
            "kind": "list",
            "matchKey": "regulator",
            "k": 3,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "both"
                    ],
                    "canonical": "both"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "represses"
                    ],
                    "canonical": "represses"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Batf",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "trm_like_til_specific"
                    ],
                    "canonical": "trm_like_til_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "chromatin_over_rna"
                    ],
                    "canonical": "chromatin_over_rna"
                  }
                }
              },
              {
                "key": "Ahr",
                "fields": {
                  "program_specificity": {
                    "op": "in",
                    "values": [
                      "trm_specific",
                      "both"
                    ],
                    "canonical": "trm_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "rna_over_chromatin"
                    ],
                    "canonical": "rna_over_chromatin"
                  }
                }
              }
            ]
          },
          {
            "name": "regulator_set_no_extras_gate",
            "role": "gate",
            "field": "$.regulator_program_map[*]",
            "kind": "other",
            "expected": null
          },
          {
            "name": "program_identity_conclusion_gate",
            "role": "gate",
            "field": "program_identity_conclusion",
            "kind": "value",
            "expected": "distinct_state_shared_brake_divergent_activators"
          },
          {
            "name": "residency_install_logic_gate",
            "role": "gate",
            "field": "residency_install_logic",
            "kind": "value",
            "expected": "brake_release_permits_context_matched_activator"
          },
          {
            "name": "activator_context_dependence_gate",
            "role": "gate",
            "field": "activator_context_dependence",
            "kind": "value",
            "expected": "context_matched_crossover"
          },
          {
            "name": "evidence_ceiling_gate",
            "role": "gate",
            "field": "evidence_ceiling",
            "kind": "value",
            "expected": "specificity_dissociations_require_perturbation"
          },
          {
            "name": "cf_install_logic_flat_or_single_step",
            "role": "hard_fail",
            "field": "residency_install_logic",
            "kind": "value",
            "op": "in",
            "expected": [
              "flat_shared_residency_toolkit",
              "activator_induction_sufficient",
              "brake_release_alone_sufficient"
            ]
          },
          {
            "name": "cf_program_identity_lumped_or_wrong",
            "role": "hard_fail",
            "field": "program_identity_conclusion",
            "kind": "value",
            "op": "in",
            "expected": [
              "trm_like_til_is_tumor_trm",
              "trm_like_til_is_exhausted_subset",
              "trm_like_til_is_effector_state",
              "trm_like_til_is_transitional_intermediate",
              "trm_like_til_is_shared_program_distinct_niche"
            ]
          },
          {
            "name": "cf_ahr_assigned_to_til",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Ahr",
            "valueField": "program_specificity",
            "expected": [
              "trm_like_til_specific"
            ]
          },
          {
            "name": "cf_batf_assigned_to_trm_or_shared",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Batf",
            "valueField": "program_specificity",
            "expected": [
              "trm_specific",
              "both"
            ]
          },
          {
            "name": "cf_klf2_called_activator",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Klf2",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          },
          {
            "name": "cf_evidence_ceiling_overclaim",
            "role": "hard_fail",
            "field": "evidence_ceiling",
            "kind": "value",
            "op": "in",
            "expected": [
              "specificity_established_from_regulon_activity",
              "specificity_established_in_dataset",
              "causation_established_by_cross_sectional_concordance",
              "necessity_and_sufficiency_established"
            ]
          },
          {
            "name": "cf_activator_context_shared_or_single",
            "role": "hard_fail",
            "field": "activator_context_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "shared_activators_both_contexts",
              "single_global_activator",
              "redundant_activators"
            ]
          },
          {
            "name": "cf_single_modality_rule",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_hic1_motif_brake_mislabel",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_runx3_anticanonical_as_controller",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Runx3",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-4-6-pi",
      "modelName": "claude-sonnet-4-6",
      "harness": "pi",
      "task": "green_shared_vs_distinct_regulators_trm_vs_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 2999,
      "cost": 2.3564865,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_program_map",
            "kind": "list",
            "matchKey": "regulator",
            "k": 3,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "both"
                    ],
                    "canonical": "both"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "represses"
                    ],
                    "canonical": "represses"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Batf",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "trm_like_til_specific"
                    ],
                    "canonical": "trm_like_til_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "chromatin_over_rna"
                    ],
                    "canonical": "chromatin_over_rna"
                  }
                }
              },
              {
                "key": "Ahr",
                "fields": {
                  "program_specificity": {
                    "op": "in",
                    "values": [
                      "trm_specific",
                      "both"
                    ],
                    "canonical": "trm_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "rna_over_chromatin"
                    ],
                    "canonical": "rna_over_chromatin"
                  }
                }
              }
            ]
          },
          {
            "name": "regulator_set_no_extras_gate",
            "role": "gate",
            "field": "$.regulator_program_map[*]",
            "kind": "other",
            "expected": null
          },
          {
            "name": "program_identity_conclusion_gate",
            "role": "gate",
            "field": "program_identity_conclusion",
            "kind": "value",
            "expected": "distinct_state_shared_brake_divergent_activators"
          },
          {
            "name": "residency_install_logic_gate",
            "role": "gate",
            "field": "residency_install_logic",
            "kind": "value",
            "expected": "brake_release_permits_context_matched_activator"
          },
          {
            "name": "activator_context_dependence_gate",
            "role": "gate",
            "field": "activator_context_dependence",
            "kind": "value",
            "expected": "context_matched_crossover"
          },
          {
            "name": "evidence_ceiling_gate",
            "role": "gate",
            "field": "evidence_ceiling",
            "kind": "value",
            "expected": "specificity_dissociations_require_perturbation"
          },
          {
            "name": "cf_install_logic_flat_or_single_step",
            "role": "hard_fail",
            "field": "residency_install_logic",
            "kind": "value",
            "op": "in",
            "expected": [
              "flat_shared_residency_toolkit",
              "activator_induction_sufficient",
              "brake_release_alone_sufficient"
            ]
          },
          {
            "name": "cf_program_identity_lumped_or_wrong",
            "role": "hard_fail",
            "field": "program_identity_conclusion",
            "kind": "value",
            "op": "in",
            "expected": [
              "trm_like_til_is_tumor_trm",
              "trm_like_til_is_exhausted_subset",
              "trm_like_til_is_effector_state",
              "trm_like_til_is_transitional_intermediate",
              "trm_like_til_is_shared_program_distinct_niche"
            ]
          },
          {
            "name": "cf_ahr_assigned_to_til",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Ahr",
            "valueField": "program_specificity",
            "expected": [
              "trm_like_til_specific"
            ]
          },
          {
            "name": "cf_batf_assigned_to_trm_or_shared",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Batf",
            "valueField": "program_specificity",
            "expected": [
              "trm_specific",
              "both"
            ]
          },
          {
            "name": "cf_klf2_called_activator",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Klf2",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          },
          {
            "name": "cf_evidence_ceiling_overclaim",
            "role": "hard_fail",
            "field": "evidence_ceiling",
            "kind": "value",
            "op": "in",
            "expected": [
              "specificity_established_from_regulon_activity",
              "specificity_established_in_dataset",
              "causation_established_by_cross_sectional_concordance",
              "necessity_and_sufficiency_established"
            ]
          },
          {
            "name": "cf_activator_context_shared_or_single",
            "role": "hard_fail",
            "field": "activator_context_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "shared_activators_both_contexts",
              "single_global_activator",
              "redundant_activators"
            ]
          },
          {
            "name": "cf_single_modality_rule",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_hic1_motif_brake_mislabel",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_runx3_anticanonical_as_controller",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Runx3",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-4-6-pi",
      "modelName": "claude-sonnet-4-6",
      "harness": "pi",
      "task": "green_shared_vs_distinct_regulators_trm_vs_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 3619,
      "cost": 2.87269215,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_program_map",
            "kind": "list",
            "matchKey": "regulator",
            "k": 3,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "both"
                    ],
                    "canonical": "both"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "represses"
                    ],
                    "canonical": "represses"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Batf",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "trm_like_til_specific"
                    ],
                    "canonical": "trm_like_til_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "chromatin_over_rna"
                    ],
                    "canonical": "chromatin_over_rna"
                  }
                }
              },
              {
                "key": "Ahr",
                "fields": {
                  "program_specificity": {
                    "op": "in",
                    "values": [
                      "trm_specific",
                      "both"
                    ],
                    "canonical": "trm_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "rna_over_chromatin"
                    ],
                    "canonical": "rna_over_chromatin"
                  }
                }
              }
            ]
          },
          {
            "name": "regulator_set_no_extras_gate",
            "role": "gate",
            "field": "$.regulator_program_map[*]",
            "kind": "other",
            "expected": null
          },
          {
            "name": "program_identity_conclusion_gate",
            "role": "gate",
            "field": "program_identity_conclusion",
            "kind": "value",
            "expected": "distinct_state_shared_brake_divergent_activators"
          },
          {
            "name": "residency_install_logic_gate",
            "role": "gate",
            "field": "residency_install_logic",
            "kind": "value",
            "expected": "brake_release_permits_context_matched_activator"
          },
          {
            "name": "activator_context_dependence_gate",
            "role": "gate",
            "field": "activator_context_dependence",
            "kind": "value",
            "expected": "context_matched_crossover"
          },
          {
            "name": "evidence_ceiling_gate",
            "role": "gate",
            "field": "evidence_ceiling",
            "kind": "value",
            "expected": "specificity_dissociations_require_perturbation"
          },
          {
            "name": "cf_install_logic_flat_or_single_step",
            "role": "hard_fail",
            "field": "residency_install_logic",
            "kind": "value",
            "op": "in",
            "expected": [
              "flat_shared_residency_toolkit",
              "activator_induction_sufficient",
              "brake_release_alone_sufficient"
            ]
          },
          {
            "name": "cf_program_identity_lumped_or_wrong",
            "role": "hard_fail",
            "field": "program_identity_conclusion",
            "kind": "value",
            "op": "in",
            "expected": [
              "trm_like_til_is_tumor_trm",
              "trm_like_til_is_exhausted_subset",
              "trm_like_til_is_effector_state",
              "trm_like_til_is_transitional_intermediate",
              "trm_like_til_is_shared_program_distinct_niche"
            ]
          },
          {
            "name": "cf_ahr_assigned_to_til",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Ahr",
            "valueField": "program_specificity",
            "expected": [
              "trm_like_til_specific"
            ]
          },
          {
            "name": "cf_batf_assigned_to_trm_or_shared",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Batf",
            "valueField": "program_specificity",
            "expected": [
              "trm_specific",
              "both"
            ]
          },
          {
            "name": "cf_klf2_called_activator",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Klf2",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          },
          {
            "name": "cf_evidence_ceiling_overclaim",
            "role": "hard_fail",
            "field": "evidence_ceiling",
            "kind": "value",
            "op": "in",
            "expected": [
              "specificity_established_from_regulon_activity",
              "specificity_established_in_dataset",
              "causation_established_by_cross_sectional_concordance",
              "necessity_and_sufficiency_established"
            ]
          },
          {
            "name": "cf_activator_context_shared_or_single",
            "role": "hard_fail",
            "field": "activator_context_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "shared_activators_both_contexts",
              "single_global_activator",
              "redundant_activators"
            ]
          },
          {
            "name": "cf_single_modality_rule",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_hic1_motif_brake_mislabel",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_runx3_anticanonical_as_controller",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Runx3",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gemini-3-1-pro-preview-pi",
      "modelName": "gemini-3.1-pro-preview",
      "harness": "pi",
      "task": "green_shared_vs_distinct_regulators_trm_vs_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 582,
      "cost": 0.9742494000000002,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_program_map",
            "kind": "list",
            "matchKey": "regulator",
            "k": 3,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "both"
                    ],
                    "canonical": "both"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "represses"
                    ],
                    "canonical": "represses"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Batf",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "trm_like_til_specific"
                    ],
                    "canonical": "trm_like_til_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "chromatin_over_rna"
                    ],
                    "canonical": "chromatin_over_rna"
                  }
                }
              },
              {
                "key": "Ahr",
                "fields": {
                  "program_specificity": {
                    "op": "in",
                    "values": [
                      "trm_specific",
                      "both"
                    ],
                    "canonical": "trm_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "rna_over_chromatin"
                    ],
                    "canonical": "rna_over_chromatin"
                  }
                }
              }
            ]
          },
          {
            "name": "regulator_set_no_extras_gate",
            "role": "gate",
            "field": "$.regulator_program_map[*]",
            "kind": "other",
            "expected": null
          },
          {
            "name": "program_identity_conclusion_gate",
            "role": "gate",
            "field": "program_identity_conclusion",
            "kind": "value",
            "expected": "distinct_state_shared_brake_divergent_activators"
          },
          {
            "name": "residency_install_logic_gate",
            "role": "gate",
            "field": "residency_install_logic",
            "kind": "value",
            "expected": "brake_release_permits_context_matched_activator"
          },
          {
            "name": "activator_context_dependence_gate",
            "role": "gate",
            "field": "activator_context_dependence",
            "kind": "value",
            "expected": "context_matched_crossover"
          },
          {
            "name": "evidence_ceiling_gate",
            "role": "gate",
            "field": "evidence_ceiling",
            "kind": "value",
            "expected": "specificity_dissociations_require_perturbation"
          },
          {
            "name": "cf_install_logic_flat_or_single_step",
            "role": "hard_fail",
            "field": "residency_install_logic",
            "kind": "value",
            "op": "in",
            "expected": [
              "flat_shared_residency_toolkit",
              "activator_induction_sufficient",
              "brake_release_alone_sufficient"
            ]
          },
          {
            "name": "cf_program_identity_lumped_or_wrong",
            "role": "hard_fail",
            "field": "program_identity_conclusion",
            "kind": "value",
            "op": "in",
            "expected": [
              "trm_like_til_is_tumor_trm",
              "trm_like_til_is_exhausted_subset",
              "trm_like_til_is_effector_state",
              "trm_like_til_is_transitional_intermediate",
              "trm_like_til_is_shared_program_distinct_niche"
            ]
          },
          {
            "name": "cf_ahr_assigned_to_til",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Ahr",
            "valueField": "program_specificity",
            "expected": [
              "trm_like_til_specific"
            ]
          },
          {
            "name": "cf_batf_assigned_to_trm_or_shared",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Batf",
            "valueField": "program_specificity",
            "expected": [
              "trm_specific",
              "both"
            ]
          },
          {
            "name": "cf_klf2_called_activator",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Klf2",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          },
          {
            "name": "cf_evidence_ceiling_overclaim",
            "role": "hard_fail",
            "field": "evidence_ceiling",
            "kind": "value",
            "op": "in",
            "expected": [
              "specificity_established_from_regulon_activity",
              "specificity_established_in_dataset",
              "causation_established_by_cross_sectional_concordance",
              "necessity_and_sufficiency_established"
            ]
          },
          {
            "name": "cf_activator_context_shared_or_single",
            "role": "hard_fail",
            "field": "activator_context_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "shared_activators_both_contexts",
              "single_global_activator",
              "redundant_activators"
            ]
          },
          {
            "name": "cf_single_modality_rule",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_hic1_motif_brake_mislabel",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_runx3_anticanonical_as_controller",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Runx3",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gemini-3-1-pro-preview-pi",
      "modelName": "gemini-3.1-pro-preview",
      "harness": "pi",
      "task": "green_shared_vs_distinct_regulators_trm_vs_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 478,
      "cost": 0.9203922000000001,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_program_map",
            "kind": "list",
            "matchKey": "regulator",
            "k": 3,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "both"
                    ],
                    "canonical": "both"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "represses"
                    ],
                    "canonical": "represses"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Batf",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "trm_like_til_specific"
                    ],
                    "canonical": "trm_like_til_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "chromatin_over_rna"
                    ],
                    "canonical": "chromatin_over_rna"
                  }
                }
              },
              {
                "key": "Ahr",
                "fields": {
                  "program_specificity": {
                    "op": "in",
                    "values": [
                      "trm_specific",
                      "both"
                    ],
                    "canonical": "trm_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "rna_over_chromatin"
                    ],
                    "canonical": "rna_over_chromatin"
                  }
                }
              }
            ]
          },
          {
            "name": "regulator_set_no_extras_gate",
            "role": "gate",
            "field": "$.regulator_program_map[*]",
            "kind": "other",
            "expected": null
          },
          {
            "name": "program_identity_conclusion_gate",
            "role": "gate",
            "field": "program_identity_conclusion",
            "kind": "value",
            "expected": "distinct_state_shared_brake_divergent_activators"
          },
          {
            "name": "residency_install_logic_gate",
            "role": "gate",
            "field": "residency_install_logic",
            "kind": "value",
            "expected": "brake_release_permits_context_matched_activator"
          },
          {
            "name": "activator_context_dependence_gate",
            "role": "gate",
            "field": "activator_context_dependence",
            "kind": "value",
            "expected": "context_matched_crossover"
          },
          {
            "name": "evidence_ceiling_gate",
            "role": "gate",
            "field": "evidence_ceiling",
            "kind": "value",
            "expected": "specificity_dissociations_require_perturbation"
          },
          {
            "name": "cf_install_logic_flat_or_single_step",
            "role": "hard_fail",
            "field": "residency_install_logic",
            "kind": "value",
            "op": "in",
            "expected": [
              "flat_shared_residency_toolkit",
              "activator_induction_sufficient",
              "brake_release_alone_sufficient"
            ]
          },
          {
            "name": "cf_program_identity_lumped_or_wrong",
            "role": "hard_fail",
            "field": "program_identity_conclusion",
            "kind": "value",
            "op": "in",
            "expected": [
              "trm_like_til_is_tumor_trm",
              "trm_like_til_is_exhausted_subset",
              "trm_like_til_is_effector_state",
              "trm_like_til_is_transitional_intermediate",
              "trm_like_til_is_shared_program_distinct_niche"
            ]
          },
          {
            "name": "cf_ahr_assigned_to_til",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Ahr",
            "valueField": "program_specificity",
            "expected": [
              "trm_like_til_specific"
            ]
          },
          {
            "name": "cf_batf_assigned_to_trm_or_shared",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Batf",
            "valueField": "program_specificity",
            "expected": [
              "trm_specific",
              "both"
            ]
          },
          {
            "name": "cf_klf2_called_activator",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Klf2",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          },
          {
            "name": "cf_evidence_ceiling_overclaim",
            "role": "hard_fail",
            "field": "evidence_ceiling",
            "kind": "value",
            "op": "in",
            "expected": [
              "specificity_established_from_regulon_activity",
              "specificity_established_in_dataset",
              "causation_established_by_cross_sectional_concordance",
              "necessity_and_sufficiency_established"
            ]
          },
          {
            "name": "cf_activator_context_shared_or_single",
            "role": "hard_fail",
            "field": "activator_context_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "shared_activators_both_contexts",
              "single_global_activator",
              "redundant_activators"
            ]
          },
          {
            "name": "cf_single_modality_rule",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_hic1_motif_brake_mislabel",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_runx3_anticanonical_as_controller",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Runx3",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gemini-3-1-pro-preview-pi",
      "modelName": "gemini-3.1-pro-preview",
      "harness": "pi",
      "task": "green_shared_vs_distinct_regulators_trm_vs_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 962,
      "cost": 0.4844897999999999,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_program_map",
            "kind": "list",
            "matchKey": "regulator",
            "k": 3,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "both"
                    ],
                    "canonical": "both"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "represses"
                    ],
                    "canonical": "represses"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Batf",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "trm_like_til_specific"
                    ],
                    "canonical": "trm_like_til_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "chromatin_over_rna"
                    ],
                    "canonical": "chromatin_over_rna"
                  }
                }
              },
              {
                "key": "Ahr",
                "fields": {
                  "program_specificity": {
                    "op": "in",
                    "values": [
                      "trm_specific",
                      "both"
                    ],
                    "canonical": "trm_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "rna_over_chromatin"
                    ],
                    "canonical": "rna_over_chromatin"
                  }
                }
              }
            ]
          },
          {
            "name": "regulator_set_no_extras_gate",
            "role": "gate",
            "field": "$.regulator_program_map[*]",
            "kind": "other",
            "expected": null
          },
          {
            "name": "program_identity_conclusion_gate",
            "role": "gate",
            "field": "program_identity_conclusion",
            "kind": "value",
            "expected": "distinct_state_shared_brake_divergent_activators"
          },
          {
            "name": "residency_install_logic_gate",
            "role": "gate",
            "field": "residency_install_logic",
            "kind": "value",
            "expected": "brake_release_permits_context_matched_activator"
          },
          {
            "name": "activator_context_dependence_gate",
            "role": "gate",
            "field": "activator_context_dependence",
            "kind": "value",
            "expected": "context_matched_crossover"
          },
          {
            "name": "evidence_ceiling_gate",
            "role": "gate",
            "field": "evidence_ceiling",
            "kind": "value",
            "expected": "specificity_dissociations_require_perturbation"
          },
          {
            "name": "cf_install_logic_flat_or_single_step",
            "role": "hard_fail",
            "field": "residency_install_logic",
            "kind": "value",
            "op": "in",
            "expected": [
              "flat_shared_residency_toolkit",
              "activator_induction_sufficient",
              "brake_release_alone_sufficient"
            ]
          },
          {
            "name": "cf_program_identity_lumped_or_wrong",
            "role": "hard_fail",
            "field": "program_identity_conclusion",
            "kind": "value",
            "op": "in",
            "expected": [
              "trm_like_til_is_tumor_trm",
              "trm_like_til_is_exhausted_subset",
              "trm_like_til_is_effector_state",
              "trm_like_til_is_transitional_intermediate",
              "trm_like_til_is_shared_program_distinct_niche"
            ]
          },
          {
            "name": "cf_ahr_assigned_to_til",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Ahr",
            "valueField": "program_specificity",
            "expected": [
              "trm_like_til_specific"
            ]
          },
          {
            "name": "cf_batf_assigned_to_trm_or_shared",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Batf",
            "valueField": "program_specificity",
            "expected": [
              "trm_specific",
              "both"
            ]
          },
          {
            "name": "cf_klf2_called_activator",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Klf2",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          },
          {
            "name": "cf_evidence_ceiling_overclaim",
            "role": "hard_fail",
            "field": "evidence_ceiling",
            "kind": "value",
            "op": "in",
            "expected": [
              "specificity_established_from_regulon_activity",
              "specificity_established_in_dataset",
              "causation_established_by_cross_sectional_concordance",
              "necessity_and_sufficiency_established"
            ]
          },
          {
            "name": "cf_activator_context_shared_or_single",
            "role": "hard_fail",
            "field": "activator_context_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "shared_activators_both_contexts",
              "single_global_activator",
              "redundant_activators"
            ]
          },
          {
            "name": "cf_single_modality_rule",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_hic1_motif_brake_mislabel",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_runx3_anticanonical_as_controller",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Runx3",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gemini-3-5-flash-pi",
      "modelName": "gemini-3.5-flash",
      "harness": "pi",
      "task": "green_shared_vs_distinct_regulators_trm_vs_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 755,
      "cost": 0.7651911000000001,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_program_map",
            "kind": "list",
            "matchKey": "regulator",
            "k": 3,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "both"
                    ],
                    "canonical": "both"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "represses"
                    ],
                    "canonical": "represses"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Batf",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "trm_like_til_specific"
                    ],
                    "canonical": "trm_like_til_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "chromatin_over_rna"
                    ],
                    "canonical": "chromatin_over_rna"
                  }
                }
              },
              {
                "key": "Ahr",
                "fields": {
                  "program_specificity": {
                    "op": "in",
                    "values": [
                      "trm_specific",
                      "both"
                    ],
                    "canonical": "trm_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "rna_over_chromatin"
                    ],
                    "canonical": "rna_over_chromatin"
                  }
                }
              }
            ]
          },
          {
            "name": "regulator_set_no_extras_gate",
            "role": "gate",
            "field": "$.regulator_program_map[*]",
            "kind": "other",
            "expected": null
          },
          {
            "name": "program_identity_conclusion_gate",
            "role": "gate",
            "field": "program_identity_conclusion",
            "kind": "value",
            "expected": "distinct_state_shared_brake_divergent_activators"
          },
          {
            "name": "residency_install_logic_gate",
            "role": "gate",
            "field": "residency_install_logic",
            "kind": "value",
            "expected": "brake_release_permits_context_matched_activator"
          },
          {
            "name": "activator_context_dependence_gate",
            "role": "gate",
            "field": "activator_context_dependence",
            "kind": "value",
            "expected": "context_matched_crossover"
          },
          {
            "name": "evidence_ceiling_gate",
            "role": "gate",
            "field": "evidence_ceiling",
            "kind": "value",
            "expected": "specificity_dissociations_require_perturbation"
          },
          {
            "name": "cf_install_logic_flat_or_single_step",
            "role": "hard_fail",
            "field": "residency_install_logic",
            "kind": "value",
            "op": "in",
            "expected": [
              "flat_shared_residency_toolkit",
              "activator_induction_sufficient",
              "brake_release_alone_sufficient"
            ]
          },
          {
            "name": "cf_program_identity_lumped_or_wrong",
            "role": "hard_fail",
            "field": "program_identity_conclusion",
            "kind": "value",
            "op": "in",
            "expected": [
              "trm_like_til_is_tumor_trm",
              "trm_like_til_is_exhausted_subset",
              "trm_like_til_is_effector_state",
              "trm_like_til_is_transitional_intermediate",
              "trm_like_til_is_shared_program_distinct_niche"
            ]
          },
          {
            "name": "cf_ahr_assigned_to_til",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Ahr",
            "valueField": "program_specificity",
            "expected": [
              "trm_like_til_specific"
            ]
          },
          {
            "name": "cf_batf_assigned_to_trm_or_shared",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Batf",
            "valueField": "program_specificity",
            "expected": [
              "trm_specific",
              "both"
            ]
          },
          {
            "name": "cf_klf2_called_activator",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Klf2",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          },
          {
            "name": "cf_evidence_ceiling_overclaim",
            "role": "hard_fail",
            "field": "evidence_ceiling",
            "kind": "value",
            "op": "in",
            "expected": [
              "specificity_established_from_regulon_activity",
              "specificity_established_in_dataset",
              "causation_established_by_cross_sectional_concordance",
              "necessity_and_sufficiency_established"
            ]
          },
          {
            "name": "cf_activator_context_shared_or_single",
            "role": "hard_fail",
            "field": "activator_context_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "shared_activators_both_contexts",
              "single_global_activator",
              "redundant_activators"
            ]
          },
          {
            "name": "cf_single_modality_rule",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_hic1_motif_brake_mislabel",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_runx3_anticanonical_as_controller",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Runx3",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gemini-3-5-flash-pi",
      "modelName": "gemini-3.5-flash",
      "harness": "pi",
      "task": "green_shared_vs_distinct_regulators_trm_vs_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 425,
      "cost": 0.7607039999999998,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_program_map",
            "kind": "list",
            "matchKey": "regulator",
            "k": 3,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "both"
                    ],
                    "canonical": "both"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "represses"
                    ],
                    "canonical": "represses"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Batf",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "trm_like_til_specific"
                    ],
                    "canonical": "trm_like_til_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "chromatin_over_rna"
                    ],
                    "canonical": "chromatin_over_rna"
                  }
                }
              },
              {
                "key": "Ahr",
                "fields": {
                  "program_specificity": {
                    "op": "in",
                    "values": [
                      "trm_specific",
                      "both"
                    ],
                    "canonical": "trm_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "rna_over_chromatin"
                    ],
                    "canonical": "rna_over_chromatin"
                  }
                }
              }
            ]
          },
          {
            "name": "regulator_set_no_extras_gate",
            "role": "gate",
            "field": "$.regulator_program_map[*]",
            "kind": "other",
            "expected": null
          },
          {
            "name": "program_identity_conclusion_gate",
            "role": "gate",
            "field": "program_identity_conclusion",
            "kind": "value",
            "expected": "distinct_state_shared_brake_divergent_activators"
          },
          {
            "name": "residency_install_logic_gate",
            "role": "gate",
            "field": "residency_install_logic",
            "kind": "value",
            "expected": "brake_release_permits_context_matched_activator"
          },
          {
            "name": "activator_context_dependence_gate",
            "role": "gate",
            "field": "activator_context_dependence",
            "kind": "value",
            "expected": "context_matched_crossover"
          },
          {
            "name": "evidence_ceiling_gate",
            "role": "gate",
            "field": "evidence_ceiling",
            "kind": "value",
            "expected": "specificity_dissociations_require_perturbation"
          },
          {
            "name": "cf_install_logic_flat_or_single_step",
            "role": "hard_fail",
            "field": "residency_install_logic",
            "kind": "value",
            "op": "in",
            "expected": [
              "flat_shared_residency_toolkit",
              "activator_induction_sufficient",
              "brake_release_alone_sufficient"
            ]
          },
          {
            "name": "cf_program_identity_lumped_or_wrong",
            "role": "hard_fail",
            "field": "program_identity_conclusion",
            "kind": "value",
            "op": "in",
            "expected": [
              "trm_like_til_is_tumor_trm",
              "trm_like_til_is_exhausted_subset",
              "trm_like_til_is_effector_state",
              "trm_like_til_is_transitional_intermediate",
              "trm_like_til_is_shared_program_distinct_niche"
            ]
          },
          {
            "name": "cf_ahr_assigned_to_til",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Ahr",
            "valueField": "program_specificity",
            "expected": [
              "trm_like_til_specific"
            ]
          },
          {
            "name": "cf_batf_assigned_to_trm_or_shared",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Batf",
            "valueField": "program_specificity",
            "expected": [
              "trm_specific",
              "both"
            ]
          },
          {
            "name": "cf_klf2_called_activator",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Klf2",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          },
          {
            "name": "cf_evidence_ceiling_overclaim",
            "role": "hard_fail",
            "field": "evidence_ceiling",
            "kind": "value",
            "op": "in",
            "expected": [
              "specificity_established_from_regulon_activity",
              "specificity_established_in_dataset",
              "causation_established_by_cross_sectional_concordance",
              "necessity_and_sufficiency_established"
            ]
          },
          {
            "name": "cf_activator_context_shared_or_single",
            "role": "hard_fail",
            "field": "activator_context_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "shared_activators_both_contexts",
              "single_global_activator",
              "redundant_activators"
            ]
          },
          {
            "name": "cf_single_modality_rule",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_hic1_motif_brake_mislabel",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_runx3_anticanonical_as_controller",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Runx3",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gemini-3-5-flash-pi",
      "modelName": "gemini-3.5-flash",
      "harness": "pi",
      "task": "green_shared_vs_distinct_regulators_trm_vs_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 524,
      "cost": 1.6583776499999998,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_program_map",
            "kind": "list",
            "matchKey": "regulator",
            "k": 3,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "both"
                    ],
                    "canonical": "both"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "represses"
                    ],
                    "canonical": "represses"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Batf",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "trm_like_til_specific"
                    ],
                    "canonical": "trm_like_til_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "chromatin_over_rna"
                    ],
                    "canonical": "chromatin_over_rna"
                  }
                }
              },
              {
                "key": "Ahr",
                "fields": {
                  "program_specificity": {
                    "op": "in",
                    "values": [
                      "trm_specific",
                      "both"
                    ],
                    "canonical": "trm_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "rna_over_chromatin"
                    ],
                    "canonical": "rna_over_chromatin"
                  }
                }
              }
            ]
          },
          {
            "name": "regulator_set_no_extras_gate",
            "role": "gate",
            "field": "$.regulator_program_map[*]",
            "kind": "other",
            "expected": null
          },
          {
            "name": "program_identity_conclusion_gate",
            "role": "gate",
            "field": "program_identity_conclusion",
            "kind": "value",
            "expected": "distinct_state_shared_brake_divergent_activators"
          },
          {
            "name": "residency_install_logic_gate",
            "role": "gate",
            "field": "residency_install_logic",
            "kind": "value",
            "expected": "brake_release_permits_context_matched_activator"
          },
          {
            "name": "activator_context_dependence_gate",
            "role": "gate",
            "field": "activator_context_dependence",
            "kind": "value",
            "expected": "context_matched_crossover"
          },
          {
            "name": "evidence_ceiling_gate",
            "role": "gate",
            "field": "evidence_ceiling",
            "kind": "value",
            "expected": "specificity_dissociations_require_perturbation"
          },
          {
            "name": "cf_install_logic_flat_or_single_step",
            "role": "hard_fail",
            "field": "residency_install_logic",
            "kind": "value",
            "op": "in",
            "expected": [
              "flat_shared_residency_toolkit",
              "activator_induction_sufficient",
              "brake_release_alone_sufficient"
            ]
          },
          {
            "name": "cf_program_identity_lumped_or_wrong",
            "role": "hard_fail",
            "field": "program_identity_conclusion",
            "kind": "value",
            "op": "in",
            "expected": [
              "trm_like_til_is_tumor_trm",
              "trm_like_til_is_exhausted_subset",
              "trm_like_til_is_effector_state",
              "trm_like_til_is_transitional_intermediate",
              "trm_like_til_is_shared_program_distinct_niche"
            ]
          },
          {
            "name": "cf_ahr_assigned_to_til",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Ahr",
            "valueField": "program_specificity",
            "expected": [
              "trm_like_til_specific"
            ]
          },
          {
            "name": "cf_batf_assigned_to_trm_or_shared",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Batf",
            "valueField": "program_specificity",
            "expected": [
              "trm_specific",
              "both"
            ]
          },
          {
            "name": "cf_klf2_called_activator",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Klf2",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          },
          {
            "name": "cf_evidence_ceiling_overclaim",
            "role": "hard_fail",
            "field": "evidence_ceiling",
            "kind": "value",
            "op": "in",
            "expected": [
              "specificity_established_from_regulon_activity",
              "specificity_established_in_dataset",
              "causation_established_by_cross_sectional_concordance",
              "necessity_and_sufficiency_established"
            ]
          },
          {
            "name": "cf_activator_context_shared_or_single",
            "role": "hard_fail",
            "field": "activator_context_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "shared_activators_both_contexts",
              "single_global_activator",
              "redundant_activators"
            ]
          },
          {
            "name": "cf_single_modality_rule",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_hic1_motif_brake_mislabel",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_runx3_anticanonical_as_controller",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Runx3",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-glm-5p2-pi",
      "modelName": "glm-5p2",
      "harness": "pi",
      "task": "green_shared_vs_distinct_regulators_trm_vs_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 4131,
      "cost": 5.59233202,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_program_map",
            "kind": "list",
            "matchKey": "regulator",
            "k": 3,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "both"
                    ],
                    "canonical": "both"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "represses"
                    ],
                    "canonical": "represses"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Batf",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "trm_like_til_specific"
                    ],
                    "canonical": "trm_like_til_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "chromatin_over_rna"
                    ],
                    "canonical": "chromatin_over_rna"
                  }
                }
              },
              {
                "key": "Ahr",
                "fields": {
                  "program_specificity": {
                    "op": "in",
                    "values": [
                      "trm_specific",
                      "both"
                    ],
                    "canonical": "trm_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "rna_over_chromatin"
                    ],
                    "canonical": "rna_over_chromatin"
                  }
                }
              }
            ]
          },
          {
            "name": "regulator_set_no_extras_gate",
            "role": "gate",
            "field": "$.regulator_program_map[*]",
            "kind": "other",
            "expected": null
          },
          {
            "name": "program_identity_conclusion_gate",
            "role": "gate",
            "field": "program_identity_conclusion",
            "kind": "value",
            "expected": "distinct_state_shared_brake_divergent_activators"
          },
          {
            "name": "residency_install_logic_gate",
            "role": "gate",
            "field": "residency_install_logic",
            "kind": "value",
            "expected": "brake_release_permits_context_matched_activator"
          },
          {
            "name": "activator_context_dependence_gate",
            "role": "gate",
            "field": "activator_context_dependence",
            "kind": "value",
            "expected": "context_matched_crossover"
          },
          {
            "name": "evidence_ceiling_gate",
            "role": "gate",
            "field": "evidence_ceiling",
            "kind": "value",
            "expected": "specificity_dissociations_require_perturbation"
          },
          {
            "name": "cf_install_logic_flat_or_single_step",
            "role": "hard_fail",
            "field": "residency_install_logic",
            "kind": "value",
            "op": "in",
            "expected": [
              "flat_shared_residency_toolkit",
              "activator_induction_sufficient",
              "brake_release_alone_sufficient"
            ]
          },
          {
            "name": "cf_program_identity_lumped_or_wrong",
            "role": "hard_fail",
            "field": "program_identity_conclusion",
            "kind": "value",
            "op": "in",
            "expected": [
              "trm_like_til_is_tumor_trm",
              "trm_like_til_is_exhausted_subset",
              "trm_like_til_is_effector_state",
              "trm_like_til_is_transitional_intermediate",
              "trm_like_til_is_shared_program_distinct_niche"
            ]
          },
          {
            "name": "cf_ahr_assigned_to_til",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Ahr",
            "valueField": "program_specificity",
            "expected": [
              "trm_like_til_specific"
            ]
          },
          {
            "name": "cf_batf_assigned_to_trm_or_shared",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Batf",
            "valueField": "program_specificity",
            "expected": [
              "trm_specific",
              "both"
            ]
          },
          {
            "name": "cf_klf2_called_activator",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Klf2",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          },
          {
            "name": "cf_evidence_ceiling_overclaim",
            "role": "hard_fail",
            "field": "evidence_ceiling",
            "kind": "value",
            "op": "in",
            "expected": [
              "specificity_established_from_regulon_activity",
              "specificity_established_in_dataset",
              "causation_established_by_cross_sectional_concordance",
              "necessity_and_sufficiency_established"
            ]
          },
          {
            "name": "cf_activator_context_shared_or_single",
            "role": "hard_fail",
            "field": "activator_context_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "shared_activators_both_contexts",
              "single_global_activator",
              "redundant_activators"
            ]
          },
          {
            "name": "cf_single_modality_rule",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_hic1_motif_brake_mislabel",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_runx3_anticanonical_as_controller",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Runx3",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-glm-5p2-pi",
      "modelName": "glm-5p2",
      "harness": "pi",
      "task": "green_shared_vs_distinct_regulators_trm_vs_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 9347,
      "cost": 12.714151460000004,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_program_map",
            "kind": "list",
            "matchKey": "regulator",
            "k": 3,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "both"
                    ],
                    "canonical": "both"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "represses"
                    ],
                    "canonical": "represses"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Batf",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "trm_like_til_specific"
                    ],
                    "canonical": "trm_like_til_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "chromatin_over_rna"
                    ],
                    "canonical": "chromatin_over_rna"
                  }
                }
              },
              {
                "key": "Ahr",
                "fields": {
                  "program_specificity": {
                    "op": "in",
                    "values": [
                      "trm_specific",
                      "both"
                    ],
                    "canonical": "trm_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "rna_over_chromatin"
                    ],
                    "canonical": "rna_over_chromatin"
                  }
                }
              }
            ]
          },
          {
            "name": "regulator_set_no_extras_gate",
            "role": "gate",
            "field": "$.regulator_program_map[*]",
            "kind": "other",
            "expected": null
          },
          {
            "name": "program_identity_conclusion_gate",
            "role": "gate",
            "field": "program_identity_conclusion",
            "kind": "value",
            "expected": "distinct_state_shared_brake_divergent_activators"
          },
          {
            "name": "residency_install_logic_gate",
            "role": "gate",
            "field": "residency_install_logic",
            "kind": "value",
            "expected": "brake_release_permits_context_matched_activator"
          },
          {
            "name": "activator_context_dependence_gate",
            "role": "gate",
            "field": "activator_context_dependence",
            "kind": "value",
            "expected": "context_matched_crossover"
          },
          {
            "name": "evidence_ceiling_gate",
            "role": "gate",
            "field": "evidence_ceiling",
            "kind": "value",
            "expected": "specificity_dissociations_require_perturbation"
          },
          {
            "name": "cf_install_logic_flat_or_single_step",
            "role": "hard_fail",
            "field": "residency_install_logic",
            "kind": "value",
            "op": "in",
            "expected": [
              "flat_shared_residency_toolkit",
              "activator_induction_sufficient",
              "brake_release_alone_sufficient"
            ]
          },
          {
            "name": "cf_program_identity_lumped_or_wrong",
            "role": "hard_fail",
            "field": "program_identity_conclusion",
            "kind": "value",
            "op": "in",
            "expected": [
              "trm_like_til_is_tumor_trm",
              "trm_like_til_is_exhausted_subset",
              "trm_like_til_is_effector_state",
              "trm_like_til_is_transitional_intermediate",
              "trm_like_til_is_shared_program_distinct_niche"
            ]
          },
          {
            "name": "cf_ahr_assigned_to_til",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Ahr",
            "valueField": "program_specificity",
            "expected": [
              "trm_like_til_specific"
            ]
          },
          {
            "name": "cf_batf_assigned_to_trm_or_shared",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Batf",
            "valueField": "program_specificity",
            "expected": [
              "trm_specific",
              "both"
            ]
          },
          {
            "name": "cf_klf2_called_activator",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Klf2",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          },
          {
            "name": "cf_evidence_ceiling_overclaim",
            "role": "hard_fail",
            "field": "evidence_ceiling",
            "kind": "value",
            "op": "in",
            "expected": [
              "specificity_established_from_regulon_activity",
              "specificity_established_in_dataset",
              "causation_established_by_cross_sectional_concordance",
              "necessity_and_sufficiency_established"
            ]
          },
          {
            "name": "cf_activator_context_shared_or_single",
            "role": "hard_fail",
            "field": "activator_context_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "shared_activators_both_contexts",
              "single_global_activator",
              "redundant_activators"
            ]
          },
          {
            "name": "cf_single_modality_rule",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_hic1_motif_brake_mislabel",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_runx3_anticanonical_as_controller",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Runx3",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-glm-5p2-pi",
      "modelName": "glm-5p2",
      "harness": "pi",
      "task": "green_shared_vs_distinct_regulators_trm_vs_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 6524,
      "cost": 6.821225640000001,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_program_map",
            "kind": "list",
            "matchKey": "regulator",
            "k": 3,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "both"
                    ],
                    "canonical": "both"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "represses"
                    ],
                    "canonical": "represses"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Batf",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "trm_like_til_specific"
                    ],
                    "canonical": "trm_like_til_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "chromatin_over_rna"
                    ],
                    "canonical": "chromatin_over_rna"
                  }
                }
              },
              {
                "key": "Ahr",
                "fields": {
                  "program_specificity": {
                    "op": "in",
                    "values": [
                      "trm_specific",
                      "both"
                    ],
                    "canonical": "trm_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "rna_over_chromatin"
                    ],
                    "canonical": "rna_over_chromatin"
                  }
                }
              }
            ]
          },
          {
            "name": "regulator_set_no_extras_gate",
            "role": "gate",
            "field": "$.regulator_program_map[*]",
            "kind": "other",
            "expected": null
          },
          {
            "name": "program_identity_conclusion_gate",
            "role": "gate",
            "field": "program_identity_conclusion",
            "kind": "value",
            "expected": "distinct_state_shared_brake_divergent_activators"
          },
          {
            "name": "residency_install_logic_gate",
            "role": "gate",
            "field": "residency_install_logic",
            "kind": "value",
            "expected": "brake_release_permits_context_matched_activator"
          },
          {
            "name": "activator_context_dependence_gate",
            "role": "gate",
            "field": "activator_context_dependence",
            "kind": "value",
            "expected": "context_matched_crossover"
          },
          {
            "name": "evidence_ceiling_gate",
            "role": "gate",
            "field": "evidence_ceiling",
            "kind": "value",
            "expected": "specificity_dissociations_require_perturbation"
          },
          {
            "name": "cf_install_logic_flat_or_single_step",
            "role": "hard_fail",
            "field": "residency_install_logic",
            "kind": "value",
            "op": "in",
            "expected": [
              "flat_shared_residency_toolkit",
              "activator_induction_sufficient",
              "brake_release_alone_sufficient"
            ]
          },
          {
            "name": "cf_program_identity_lumped_or_wrong",
            "role": "hard_fail",
            "field": "program_identity_conclusion",
            "kind": "value",
            "op": "in",
            "expected": [
              "trm_like_til_is_tumor_trm",
              "trm_like_til_is_exhausted_subset",
              "trm_like_til_is_effector_state",
              "trm_like_til_is_transitional_intermediate",
              "trm_like_til_is_shared_program_distinct_niche"
            ]
          },
          {
            "name": "cf_ahr_assigned_to_til",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Ahr",
            "valueField": "program_specificity",
            "expected": [
              "trm_like_til_specific"
            ]
          },
          {
            "name": "cf_batf_assigned_to_trm_or_shared",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Batf",
            "valueField": "program_specificity",
            "expected": [
              "trm_specific",
              "both"
            ]
          },
          {
            "name": "cf_klf2_called_activator",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Klf2",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          },
          {
            "name": "cf_evidence_ceiling_overclaim",
            "role": "hard_fail",
            "field": "evidence_ceiling",
            "kind": "value",
            "op": "in",
            "expected": [
              "specificity_established_from_regulon_activity",
              "specificity_established_in_dataset",
              "causation_established_by_cross_sectional_concordance",
              "necessity_and_sufficiency_established"
            ]
          },
          {
            "name": "cf_activator_context_shared_or_single",
            "role": "hard_fail",
            "field": "activator_context_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "shared_activators_both_contexts",
              "single_global_activator",
              "redundant_activators"
            ]
          },
          {
            "name": "cf_single_modality_rule",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_hic1_motif_brake_mislabel",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_runx3_anticanonical_as_controller",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Runx3",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-4-codex",
      "modelName": "gpt-5.4",
      "harness": "openai-codex",
      "task": "green_shared_vs_distinct_regulators_trm_vs_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 1255,
      "cost": 2.4944965,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_program_map",
            "kind": "list",
            "matchKey": "regulator",
            "k": 3,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "both"
                    ],
                    "canonical": "both"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "represses"
                    ],
                    "canonical": "represses"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Batf",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "trm_like_til_specific"
                    ],
                    "canonical": "trm_like_til_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "chromatin_over_rna"
                    ],
                    "canonical": "chromatin_over_rna"
                  }
                }
              },
              {
                "key": "Ahr",
                "fields": {
                  "program_specificity": {
                    "op": "in",
                    "values": [
                      "trm_specific",
                      "both"
                    ],
                    "canonical": "trm_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "rna_over_chromatin"
                    ],
                    "canonical": "rna_over_chromatin"
                  }
                }
              }
            ]
          },
          {
            "name": "regulator_set_no_extras_gate",
            "role": "gate",
            "field": "$.regulator_program_map[*]",
            "kind": "other",
            "expected": null
          },
          {
            "name": "program_identity_conclusion_gate",
            "role": "gate",
            "field": "program_identity_conclusion",
            "kind": "value",
            "expected": "distinct_state_shared_brake_divergent_activators"
          },
          {
            "name": "residency_install_logic_gate",
            "role": "gate",
            "field": "residency_install_logic",
            "kind": "value",
            "expected": "brake_release_permits_context_matched_activator"
          },
          {
            "name": "activator_context_dependence_gate",
            "role": "gate",
            "field": "activator_context_dependence",
            "kind": "value",
            "expected": "context_matched_crossover"
          },
          {
            "name": "evidence_ceiling_gate",
            "role": "gate",
            "field": "evidence_ceiling",
            "kind": "value",
            "expected": "specificity_dissociations_require_perturbation"
          },
          {
            "name": "cf_install_logic_flat_or_single_step",
            "role": "hard_fail",
            "field": "residency_install_logic",
            "kind": "value",
            "op": "in",
            "expected": [
              "flat_shared_residency_toolkit",
              "activator_induction_sufficient",
              "brake_release_alone_sufficient"
            ]
          },
          {
            "name": "cf_program_identity_lumped_or_wrong",
            "role": "hard_fail",
            "field": "program_identity_conclusion",
            "kind": "value",
            "op": "in",
            "expected": [
              "trm_like_til_is_tumor_trm",
              "trm_like_til_is_exhausted_subset",
              "trm_like_til_is_effector_state",
              "trm_like_til_is_transitional_intermediate",
              "trm_like_til_is_shared_program_distinct_niche"
            ]
          },
          {
            "name": "cf_ahr_assigned_to_til",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Ahr",
            "valueField": "program_specificity",
            "expected": [
              "trm_like_til_specific"
            ]
          },
          {
            "name": "cf_batf_assigned_to_trm_or_shared",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Batf",
            "valueField": "program_specificity",
            "expected": [
              "trm_specific",
              "both"
            ]
          },
          {
            "name": "cf_klf2_called_activator",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Klf2",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          },
          {
            "name": "cf_evidence_ceiling_overclaim",
            "role": "hard_fail",
            "field": "evidence_ceiling",
            "kind": "value",
            "op": "in",
            "expected": [
              "specificity_established_from_regulon_activity",
              "specificity_established_in_dataset",
              "causation_established_by_cross_sectional_concordance",
              "necessity_and_sufficiency_established"
            ]
          },
          {
            "name": "cf_activator_context_shared_or_single",
            "role": "hard_fail",
            "field": "activator_context_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "shared_activators_both_contexts",
              "single_global_activator",
              "redundant_activators"
            ]
          },
          {
            "name": "cf_single_modality_rule",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_hic1_motif_brake_mislabel",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_runx3_anticanonical_as_controller",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Runx3",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-4-codex",
      "modelName": "gpt-5.4",
      "harness": "openai-codex",
      "task": "green_shared_vs_distinct_regulators_trm_vs_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 2583,
      "cost": 3.4103874999999997,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_program_map",
            "kind": "list",
            "matchKey": "regulator",
            "k": 3,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "both"
                    ],
                    "canonical": "both"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "represses"
                    ],
                    "canonical": "represses"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Batf",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "trm_like_til_specific"
                    ],
                    "canonical": "trm_like_til_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "chromatin_over_rna"
                    ],
                    "canonical": "chromatin_over_rna"
                  }
                }
              },
              {
                "key": "Ahr",
                "fields": {
                  "program_specificity": {
                    "op": "in",
                    "values": [
                      "trm_specific",
                      "both"
                    ],
                    "canonical": "trm_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "rna_over_chromatin"
                    ],
                    "canonical": "rna_over_chromatin"
                  }
                }
              }
            ]
          },
          {
            "name": "regulator_set_no_extras_gate",
            "role": "gate",
            "field": "$.regulator_program_map[*]",
            "kind": "other",
            "expected": null
          },
          {
            "name": "program_identity_conclusion_gate",
            "role": "gate",
            "field": "program_identity_conclusion",
            "kind": "value",
            "expected": "distinct_state_shared_brake_divergent_activators"
          },
          {
            "name": "residency_install_logic_gate",
            "role": "gate",
            "field": "residency_install_logic",
            "kind": "value",
            "expected": "brake_release_permits_context_matched_activator"
          },
          {
            "name": "activator_context_dependence_gate",
            "role": "gate",
            "field": "activator_context_dependence",
            "kind": "value",
            "expected": "context_matched_crossover"
          },
          {
            "name": "evidence_ceiling_gate",
            "role": "gate",
            "field": "evidence_ceiling",
            "kind": "value",
            "expected": "specificity_dissociations_require_perturbation"
          },
          {
            "name": "cf_install_logic_flat_or_single_step",
            "role": "hard_fail",
            "field": "residency_install_logic",
            "kind": "value",
            "op": "in",
            "expected": [
              "flat_shared_residency_toolkit",
              "activator_induction_sufficient",
              "brake_release_alone_sufficient"
            ]
          },
          {
            "name": "cf_program_identity_lumped_or_wrong",
            "role": "hard_fail",
            "field": "program_identity_conclusion",
            "kind": "value",
            "op": "in",
            "expected": [
              "trm_like_til_is_tumor_trm",
              "trm_like_til_is_exhausted_subset",
              "trm_like_til_is_effector_state",
              "trm_like_til_is_transitional_intermediate",
              "trm_like_til_is_shared_program_distinct_niche"
            ]
          },
          {
            "name": "cf_ahr_assigned_to_til",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Ahr",
            "valueField": "program_specificity",
            "expected": [
              "trm_like_til_specific"
            ]
          },
          {
            "name": "cf_batf_assigned_to_trm_or_shared",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Batf",
            "valueField": "program_specificity",
            "expected": [
              "trm_specific",
              "both"
            ]
          },
          {
            "name": "cf_klf2_called_activator",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Klf2",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          },
          {
            "name": "cf_evidence_ceiling_overclaim",
            "role": "hard_fail",
            "field": "evidence_ceiling",
            "kind": "value",
            "op": "in",
            "expected": [
              "specificity_established_from_regulon_activity",
              "specificity_established_in_dataset",
              "causation_established_by_cross_sectional_concordance",
              "necessity_and_sufficiency_established"
            ]
          },
          {
            "name": "cf_activator_context_shared_or_single",
            "role": "hard_fail",
            "field": "activator_context_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "shared_activators_both_contexts",
              "single_global_activator",
              "redundant_activators"
            ]
          },
          {
            "name": "cf_single_modality_rule",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_hic1_motif_brake_mislabel",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_runx3_anticanonical_as_controller",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Runx3",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-4-codex",
      "modelName": "gpt-5.4",
      "harness": "openai-codex",
      "task": "green_shared_vs_distinct_regulators_trm_vs_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1934,
      "cost": 2.745984,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_program_map",
            "kind": "list",
            "matchKey": "regulator",
            "k": 3,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "both"
                    ],
                    "canonical": "both"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "represses"
                    ],
                    "canonical": "represses"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Batf",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "trm_like_til_specific"
                    ],
                    "canonical": "trm_like_til_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "chromatin_over_rna"
                    ],
                    "canonical": "chromatin_over_rna"
                  }
                }
              },
              {
                "key": "Ahr",
                "fields": {
                  "program_specificity": {
                    "op": "in",
                    "values": [
                      "trm_specific",
                      "both"
                    ],
                    "canonical": "trm_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "rna_over_chromatin"
                    ],
                    "canonical": "rna_over_chromatin"
                  }
                }
              }
            ]
          },
          {
            "name": "regulator_set_no_extras_gate",
            "role": "gate",
            "field": "$.regulator_program_map[*]",
            "kind": "other",
            "expected": null
          },
          {
            "name": "program_identity_conclusion_gate",
            "role": "gate",
            "field": "program_identity_conclusion",
            "kind": "value",
            "expected": "distinct_state_shared_brake_divergent_activators"
          },
          {
            "name": "residency_install_logic_gate",
            "role": "gate",
            "field": "residency_install_logic",
            "kind": "value",
            "expected": "brake_release_permits_context_matched_activator"
          },
          {
            "name": "activator_context_dependence_gate",
            "role": "gate",
            "field": "activator_context_dependence",
            "kind": "value",
            "expected": "context_matched_crossover"
          },
          {
            "name": "evidence_ceiling_gate",
            "role": "gate",
            "field": "evidence_ceiling",
            "kind": "value",
            "expected": "specificity_dissociations_require_perturbation"
          },
          {
            "name": "cf_install_logic_flat_or_single_step",
            "role": "hard_fail",
            "field": "residency_install_logic",
            "kind": "value",
            "op": "in",
            "expected": [
              "flat_shared_residency_toolkit",
              "activator_induction_sufficient",
              "brake_release_alone_sufficient"
            ]
          },
          {
            "name": "cf_program_identity_lumped_or_wrong",
            "role": "hard_fail",
            "field": "program_identity_conclusion",
            "kind": "value",
            "op": "in",
            "expected": [
              "trm_like_til_is_tumor_trm",
              "trm_like_til_is_exhausted_subset",
              "trm_like_til_is_effector_state",
              "trm_like_til_is_transitional_intermediate",
              "trm_like_til_is_shared_program_distinct_niche"
            ]
          },
          {
            "name": "cf_ahr_assigned_to_til",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Ahr",
            "valueField": "program_specificity",
            "expected": [
              "trm_like_til_specific"
            ]
          },
          {
            "name": "cf_batf_assigned_to_trm_or_shared",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Batf",
            "valueField": "program_specificity",
            "expected": [
              "trm_specific",
              "both"
            ]
          },
          {
            "name": "cf_klf2_called_activator",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Klf2",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          },
          {
            "name": "cf_evidence_ceiling_overclaim",
            "role": "hard_fail",
            "field": "evidence_ceiling",
            "kind": "value",
            "op": "in",
            "expected": [
              "specificity_established_from_regulon_activity",
              "specificity_established_in_dataset",
              "causation_established_by_cross_sectional_concordance",
              "necessity_and_sufficiency_established"
            ]
          },
          {
            "name": "cf_activator_context_shared_or_single",
            "role": "hard_fail",
            "field": "activator_context_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "shared_activators_both_contexts",
              "single_global_activator",
              "redundant_activators"
            ]
          },
          {
            "name": "cf_single_modality_rule",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_hic1_motif_brake_mislabel",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_runx3_anticanonical_as_controller",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Runx3",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-4-pi",
      "modelName": "gpt-5.4",
      "harness": "pi",
      "task": "green_shared_vs_distinct_regulators_trm_vs_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 3691,
      "cost": 2.2065680000000003,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_program_map",
            "kind": "list",
            "matchKey": "regulator",
            "k": 3,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "both"
                    ],
                    "canonical": "both"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "represses"
                    ],
                    "canonical": "represses"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Batf",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "trm_like_til_specific"
                    ],
                    "canonical": "trm_like_til_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "chromatin_over_rna"
                    ],
                    "canonical": "chromatin_over_rna"
                  }
                }
              },
              {
                "key": "Ahr",
                "fields": {
                  "program_specificity": {
                    "op": "in",
                    "values": [
                      "trm_specific",
                      "both"
                    ],
                    "canonical": "trm_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "rna_over_chromatin"
                    ],
                    "canonical": "rna_over_chromatin"
                  }
                }
              }
            ]
          },
          {
            "name": "regulator_set_no_extras_gate",
            "role": "gate",
            "field": "$.regulator_program_map[*]",
            "kind": "other",
            "expected": null
          },
          {
            "name": "program_identity_conclusion_gate",
            "role": "gate",
            "field": "program_identity_conclusion",
            "kind": "value",
            "expected": "distinct_state_shared_brake_divergent_activators"
          },
          {
            "name": "residency_install_logic_gate",
            "role": "gate",
            "field": "residency_install_logic",
            "kind": "value",
            "expected": "brake_release_permits_context_matched_activator"
          },
          {
            "name": "activator_context_dependence_gate",
            "role": "gate",
            "field": "activator_context_dependence",
            "kind": "value",
            "expected": "context_matched_crossover"
          },
          {
            "name": "evidence_ceiling_gate",
            "role": "gate",
            "field": "evidence_ceiling",
            "kind": "value",
            "expected": "specificity_dissociations_require_perturbation"
          },
          {
            "name": "cf_install_logic_flat_or_single_step",
            "role": "hard_fail",
            "field": "residency_install_logic",
            "kind": "value",
            "op": "in",
            "expected": [
              "flat_shared_residency_toolkit",
              "activator_induction_sufficient",
              "brake_release_alone_sufficient"
            ]
          },
          {
            "name": "cf_program_identity_lumped_or_wrong",
            "role": "hard_fail",
            "field": "program_identity_conclusion",
            "kind": "value",
            "op": "in",
            "expected": [
              "trm_like_til_is_tumor_trm",
              "trm_like_til_is_exhausted_subset",
              "trm_like_til_is_effector_state",
              "trm_like_til_is_transitional_intermediate",
              "trm_like_til_is_shared_program_distinct_niche"
            ]
          },
          {
            "name": "cf_ahr_assigned_to_til",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Ahr",
            "valueField": "program_specificity",
            "expected": [
              "trm_like_til_specific"
            ]
          },
          {
            "name": "cf_batf_assigned_to_trm_or_shared",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Batf",
            "valueField": "program_specificity",
            "expected": [
              "trm_specific",
              "both"
            ]
          },
          {
            "name": "cf_klf2_called_activator",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Klf2",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          },
          {
            "name": "cf_evidence_ceiling_overclaim",
            "role": "hard_fail",
            "field": "evidence_ceiling",
            "kind": "value",
            "op": "in",
            "expected": [
              "specificity_established_from_regulon_activity",
              "specificity_established_in_dataset",
              "causation_established_by_cross_sectional_concordance",
              "necessity_and_sufficiency_established"
            ]
          },
          {
            "name": "cf_activator_context_shared_or_single",
            "role": "hard_fail",
            "field": "activator_context_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "shared_activators_both_contexts",
              "single_global_activator",
              "redundant_activators"
            ]
          },
          {
            "name": "cf_single_modality_rule",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_hic1_motif_brake_mislabel",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_runx3_anticanonical_as_controller",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Runx3",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-4-pi",
      "modelName": "gpt-5.4",
      "harness": "pi",
      "task": "green_shared_vs_distinct_regulators_trm_vs_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 5857,
      "cost": 2.5091520000000007,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_program_map",
            "kind": "list",
            "matchKey": "regulator",
            "k": 3,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "both"
                    ],
                    "canonical": "both"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "represses"
                    ],
                    "canonical": "represses"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Batf",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "trm_like_til_specific"
                    ],
                    "canonical": "trm_like_til_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "chromatin_over_rna"
                    ],
                    "canonical": "chromatin_over_rna"
                  }
                }
              },
              {
                "key": "Ahr",
                "fields": {
                  "program_specificity": {
                    "op": "in",
                    "values": [
                      "trm_specific",
                      "both"
                    ],
                    "canonical": "trm_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "rna_over_chromatin"
                    ],
                    "canonical": "rna_over_chromatin"
                  }
                }
              }
            ]
          },
          {
            "name": "regulator_set_no_extras_gate",
            "role": "gate",
            "field": "$.regulator_program_map[*]",
            "kind": "other",
            "expected": null
          },
          {
            "name": "program_identity_conclusion_gate",
            "role": "gate",
            "field": "program_identity_conclusion",
            "kind": "value",
            "expected": "distinct_state_shared_brake_divergent_activators"
          },
          {
            "name": "residency_install_logic_gate",
            "role": "gate",
            "field": "residency_install_logic",
            "kind": "value",
            "expected": "brake_release_permits_context_matched_activator"
          },
          {
            "name": "activator_context_dependence_gate",
            "role": "gate",
            "field": "activator_context_dependence",
            "kind": "value",
            "expected": "context_matched_crossover"
          },
          {
            "name": "evidence_ceiling_gate",
            "role": "gate",
            "field": "evidence_ceiling",
            "kind": "value",
            "expected": "specificity_dissociations_require_perturbation"
          },
          {
            "name": "cf_install_logic_flat_or_single_step",
            "role": "hard_fail",
            "field": "residency_install_logic",
            "kind": "value",
            "op": "in",
            "expected": [
              "flat_shared_residency_toolkit",
              "activator_induction_sufficient",
              "brake_release_alone_sufficient"
            ]
          },
          {
            "name": "cf_program_identity_lumped_or_wrong",
            "role": "hard_fail",
            "field": "program_identity_conclusion",
            "kind": "value",
            "op": "in",
            "expected": [
              "trm_like_til_is_tumor_trm",
              "trm_like_til_is_exhausted_subset",
              "trm_like_til_is_effector_state",
              "trm_like_til_is_transitional_intermediate",
              "trm_like_til_is_shared_program_distinct_niche"
            ]
          },
          {
            "name": "cf_ahr_assigned_to_til",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Ahr",
            "valueField": "program_specificity",
            "expected": [
              "trm_like_til_specific"
            ]
          },
          {
            "name": "cf_batf_assigned_to_trm_or_shared",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Batf",
            "valueField": "program_specificity",
            "expected": [
              "trm_specific",
              "both"
            ]
          },
          {
            "name": "cf_klf2_called_activator",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Klf2",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          },
          {
            "name": "cf_evidence_ceiling_overclaim",
            "role": "hard_fail",
            "field": "evidence_ceiling",
            "kind": "value",
            "op": "in",
            "expected": [
              "specificity_established_from_regulon_activity",
              "specificity_established_in_dataset",
              "causation_established_by_cross_sectional_concordance",
              "necessity_and_sufficiency_established"
            ]
          },
          {
            "name": "cf_activator_context_shared_or_single",
            "role": "hard_fail",
            "field": "activator_context_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "shared_activators_both_contexts",
              "single_global_activator",
              "redundant_activators"
            ]
          },
          {
            "name": "cf_single_modality_rule",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_hic1_motif_brake_mislabel",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_runx3_anticanonical_as_controller",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Runx3",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-4-pi",
      "modelName": "gpt-5.4",
      "harness": "pi",
      "task": "green_shared_vs_distinct_regulators_trm_vs_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 7903,
      "cost": 2.466253,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_program_map",
            "kind": "list",
            "matchKey": "regulator",
            "k": 3,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "both"
                    ],
                    "canonical": "both"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "represses"
                    ],
                    "canonical": "represses"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Batf",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "trm_like_til_specific"
                    ],
                    "canonical": "trm_like_til_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "chromatin_over_rna"
                    ],
                    "canonical": "chromatin_over_rna"
                  }
                }
              },
              {
                "key": "Ahr",
                "fields": {
                  "program_specificity": {
                    "op": "in",
                    "values": [
                      "trm_specific",
                      "both"
                    ],
                    "canonical": "trm_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "rna_over_chromatin"
                    ],
                    "canonical": "rna_over_chromatin"
                  }
                }
              }
            ]
          },
          {
            "name": "regulator_set_no_extras_gate",
            "role": "gate",
            "field": "$.regulator_program_map[*]",
            "kind": "other",
            "expected": null
          },
          {
            "name": "program_identity_conclusion_gate",
            "role": "gate",
            "field": "program_identity_conclusion",
            "kind": "value",
            "expected": "distinct_state_shared_brake_divergent_activators"
          },
          {
            "name": "residency_install_logic_gate",
            "role": "gate",
            "field": "residency_install_logic",
            "kind": "value",
            "expected": "brake_release_permits_context_matched_activator"
          },
          {
            "name": "activator_context_dependence_gate",
            "role": "gate",
            "field": "activator_context_dependence",
            "kind": "value",
            "expected": "context_matched_crossover"
          },
          {
            "name": "evidence_ceiling_gate",
            "role": "gate",
            "field": "evidence_ceiling",
            "kind": "value",
            "expected": "specificity_dissociations_require_perturbation"
          },
          {
            "name": "cf_install_logic_flat_or_single_step",
            "role": "hard_fail",
            "field": "residency_install_logic",
            "kind": "value",
            "op": "in",
            "expected": [
              "flat_shared_residency_toolkit",
              "activator_induction_sufficient",
              "brake_release_alone_sufficient"
            ]
          },
          {
            "name": "cf_program_identity_lumped_or_wrong",
            "role": "hard_fail",
            "field": "program_identity_conclusion",
            "kind": "value",
            "op": "in",
            "expected": [
              "trm_like_til_is_tumor_trm",
              "trm_like_til_is_exhausted_subset",
              "trm_like_til_is_effector_state",
              "trm_like_til_is_transitional_intermediate",
              "trm_like_til_is_shared_program_distinct_niche"
            ]
          },
          {
            "name": "cf_ahr_assigned_to_til",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Ahr",
            "valueField": "program_specificity",
            "expected": [
              "trm_like_til_specific"
            ]
          },
          {
            "name": "cf_batf_assigned_to_trm_or_shared",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Batf",
            "valueField": "program_specificity",
            "expected": [
              "trm_specific",
              "both"
            ]
          },
          {
            "name": "cf_klf2_called_activator",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Klf2",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          },
          {
            "name": "cf_evidence_ceiling_overclaim",
            "role": "hard_fail",
            "field": "evidence_ceiling",
            "kind": "value",
            "op": "in",
            "expected": [
              "specificity_established_from_regulon_activity",
              "specificity_established_in_dataset",
              "causation_established_by_cross_sectional_concordance",
              "necessity_and_sufficiency_established"
            ]
          },
          {
            "name": "cf_activator_context_shared_or_single",
            "role": "hard_fail",
            "field": "activator_context_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "shared_activators_both_contexts",
              "single_global_activator",
              "redundant_activators"
            ]
          },
          {
            "name": "cf_single_modality_rule",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_hic1_motif_brake_mislabel",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_runx3_anticanonical_as_controller",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Runx3",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-5-codex",
      "modelName": "gpt-5.5",
      "harness": "openai-codex",
      "task": "green_shared_vs_distinct_regulators_trm_vs_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 472,
      "cost": 3.41996,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_program_map",
            "kind": "list",
            "matchKey": "regulator",
            "k": 3,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "both"
                    ],
                    "canonical": "both"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "represses"
                    ],
                    "canonical": "represses"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Batf",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "trm_like_til_specific"
                    ],
                    "canonical": "trm_like_til_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "chromatin_over_rna"
                    ],
                    "canonical": "chromatin_over_rna"
                  }
                }
              },
              {
                "key": "Ahr",
                "fields": {
                  "program_specificity": {
                    "op": "in",
                    "values": [
                      "trm_specific",
                      "both"
                    ],
                    "canonical": "trm_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "rna_over_chromatin"
                    ],
                    "canonical": "rna_over_chromatin"
                  }
                }
              }
            ]
          },
          {
            "name": "regulator_set_no_extras_gate",
            "role": "gate",
            "field": "$.regulator_program_map[*]",
            "kind": "other",
            "expected": null
          },
          {
            "name": "program_identity_conclusion_gate",
            "role": "gate",
            "field": "program_identity_conclusion",
            "kind": "value",
            "expected": "distinct_state_shared_brake_divergent_activators"
          },
          {
            "name": "residency_install_logic_gate",
            "role": "gate",
            "field": "residency_install_logic",
            "kind": "value",
            "expected": "brake_release_permits_context_matched_activator"
          },
          {
            "name": "activator_context_dependence_gate",
            "role": "gate",
            "field": "activator_context_dependence",
            "kind": "value",
            "expected": "context_matched_crossover"
          },
          {
            "name": "evidence_ceiling_gate",
            "role": "gate",
            "field": "evidence_ceiling",
            "kind": "value",
            "expected": "specificity_dissociations_require_perturbation"
          },
          {
            "name": "cf_install_logic_flat_or_single_step",
            "role": "hard_fail",
            "field": "residency_install_logic",
            "kind": "value",
            "op": "in",
            "expected": [
              "flat_shared_residency_toolkit",
              "activator_induction_sufficient",
              "brake_release_alone_sufficient"
            ]
          },
          {
            "name": "cf_program_identity_lumped_or_wrong",
            "role": "hard_fail",
            "field": "program_identity_conclusion",
            "kind": "value",
            "op": "in",
            "expected": [
              "trm_like_til_is_tumor_trm",
              "trm_like_til_is_exhausted_subset",
              "trm_like_til_is_effector_state",
              "trm_like_til_is_transitional_intermediate",
              "trm_like_til_is_shared_program_distinct_niche"
            ]
          },
          {
            "name": "cf_ahr_assigned_to_til",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Ahr",
            "valueField": "program_specificity",
            "expected": [
              "trm_like_til_specific"
            ]
          },
          {
            "name": "cf_batf_assigned_to_trm_or_shared",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Batf",
            "valueField": "program_specificity",
            "expected": [
              "trm_specific",
              "both"
            ]
          },
          {
            "name": "cf_klf2_called_activator",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Klf2",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          },
          {
            "name": "cf_evidence_ceiling_overclaim",
            "role": "hard_fail",
            "field": "evidence_ceiling",
            "kind": "value",
            "op": "in",
            "expected": [
              "specificity_established_from_regulon_activity",
              "specificity_established_in_dataset",
              "causation_established_by_cross_sectional_concordance",
              "necessity_and_sufficiency_established"
            ]
          },
          {
            "name": "cf_activator_context_shared_or_single",
            "role": "hard_fail",
            "field": "activator_context_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "shared_activators_both_contexts",
              "single_global_activator",
              "redundant_activators"
            ]
          },
          {
            "name": "cf_single_modality_rule",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_hic1_motif_brake_mislabel",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_runx3_anticanonical_as_controller",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Runx3",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-5-codex",
      "modelName": "gpt-5.5",
      "harness": "openai-codex",
      "task": "green_shared_vs_distinct_regulators_trm_vs_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 673,
      "cost": 2.2473899999999998,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_program_map",
            "kind": "list",
            "matchKey": "regulator",
            "k": 3,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "both"
                    ],
                    "canonical": "both"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "represses"
                    ],
                    "canonical": "represses"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Batf",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "trm_like_til_specific"
                    ],
                    "canonical": "trm_like_til_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "chromatin_over_rna"
                    ],
                    "canonical": "chromatin_over_rna"
                  }
                }
              },
              {
                "key": "Ahr",
                "fields": {
                  "program_specificity": {
                    "op": "in",
                    "values": [
                      "trm_specific",
                      "both"
                    ],
                    "canonical": "trm_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "rna_over_chromatin"
                    ],
                    "canonical": "rna_over_chromatin"
                  }
                }
              }
            ]
          },
          {
            "name": "regulator_set_no_extras_gate",
            "role": "gate",
            "field": "$.regulator_program_map[*]",
            "kind": "other",
            "expected": null
          },
          {
            "name": "program_identity_conclusion_gate",
            "role": "gate",
            "field": "program_identity_conclusion",
            "kind": "value",
            "expected": "distinct_state_shared_brake_divergent_activators"
          },
          {
            "name": "residency_install_logic_gate",
            "role": "gate",
            "field": "residency_install_logic",
            "kind": "value",
            "expected": "brake_release_permits_context_matched_activator"
          },
          {
            "name": "activator_context_dependence_gate",
            "role": "gate",
            "field": "activator_context_dependence",
            "kind": "value",
            "expected": "context_matched_crossover"
          },
          {
            "name": "evidence_ceiling_gate",
            "role": "gate",
            "field": "evidence_ceiling",
            "kind": "value",
            "expected": "specificity_dissociations_require_perturbation"
          },
          {
            "name": "cf_install_logic_flat_or_single_step",
            "role": "hard_fail",
            "field": "residency_install_logic",
            "kind": "value",
            "op": "in",
            "expected": [
              "flat_shared_residency_toolkit",
              "activator_induction_sufficient",
              "brake_release_alone_sufficient"
            ]
          },
          {
            "name": "cf_program_identity_lumped_or_wrong",
            "role": "hard_fail",
            "field": "program_identity_conclusion",
            "kind": "value",
            "op": "in",
            "expected": [
              "trm_like_til_is_tumor_trm",
              "trm_like_til_is_exhausted_subset",
              "trm_like_til_is_effector_state",
              "trm_like_til_is_transitional_intermediate",
              "trm_like_til_is_shared_program_distinct_niche"
            ]
          },
          {
            "name": "cf_ahr_assigned_to_til",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Ahr",
            "valueField": "program_specificity",
            "expected": [
              "trm_like_til_specific"
            ]
          },
          {
            "name": "cf_batf_assigned_to_trm_or_shared",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Batf",
            "valueField": "program_specificity",
            "expected": [
              "trm_specific",
              "both"
            ]
          },
          {
            "name": "cf_klf2_called_activator",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Klf2",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          },
          {
            "name": "cf_evidence_ceiling_overclaim",
            "role": "hard_fail",
            "field": "evidence_ceiling",
            "kind": "value",
            "op": "in",
            "expected": [
              "specificity_established_from_regulon_activity",
              "specificity_established_in_dataset",
              "causation_established_by_cross_sectional_concordance",
              "necessity_and_sufficiency_established"
            ]
          },
          {
            "name": "cf_activator_context_shared_or_single",
            "role": "hard_fail",
            "field": "activator_context_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "shared_activators_both_contexts",
              "single_global_activator",
              "redundant_activators"
            ]
          },
          {
            "name": "cf_single_modality_rule",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_hic1_motif_brake_mislabel",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_runx3_anticanonical_as_controller",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Runx3",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-5-codex",
      "modelName": "gpt-5.5",
      "harness": "openai-codex",
      "task": "green_shared_vs_distinct_regulators_trm_vs_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 2101,
      "cost": 2.7952269999999997,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_program_map",
            "kind": "list",
            "matchKey": "regulator",
            "k": 3,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "both"
                    ],
                    "canonical": "both"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "represses"
                    ],
                    "canonical": "represses"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Batf",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "trm_like_til_specific"
                    ],
                    "canonical": "trm_like_til_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "chromatin_over_rna"
                    ],
                    "canonical": "chromatin_over_rna"
                  }
                }
              },
              {
                "key": "Ahr",
                "fields": {
                  "program_specificity": {
                    "op": "in",
                    "values": [
                      "trm_specific",
                      "both"
                    ],
                    "canonical": "trm_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "rna_over_chromatin"
                    ],
                    "canonical": "rna_over_chromatin"
                  }
                }
              }
            ]
          },
          {
            "name": "regulator_set_no_extras_gate",
            "role": "gate",
            "field": "$.regulator_program_map[*]",
            "kind": "other",
            "expected": null
          },
          {
            "name": "program_identity_conclusion_gate",
            "role": "gate",
            "field": "program_identity_conclusion",
            "kind": "value",
            "expected": "distinct_state_shared_brake_divergent_activators"
          },
          {
            "name": "residency_install_logic_gate",
            "role": "gate",
            "field": "residency_install_logic",
            "kind": "value",
            "expected": "brake_release_permits_context_matched_activator"
          },
          {
            "name": "activator_context_dependence_gate",
            "role": "gate",
            "field": "activator_context_dependence",
            "kind": "value",
            "expected": "context_matched_crossover"
          },
          {
            "name": "evidence_ceiling_gate",
            "role": "gate",
            "field": "evidence_ceiling",
            "kind": "value",
            "expected": "specificity_dissociations_require_perturbation"
          },
          {
            "name": "cf_install_logic_flat_or_single_step",
            "role": "hard_fail",
            "field": "residency_install_logic",
            "kind": "value",
            "op": "in",
            "expected": [
              "flat_shared_residency_toolkit",
              "activator_induction_sufficient",
              "brake_release_alone_sufficient"
            ]
          },
          {
            "name": "cf_program_identity_lumped_or_wrong",
            "role": "hard_fail",
            "field": "program_identity_conclusion",
            "kind": "value",
            "op": "in",
            "expected": [
              "trm_like_til_is_tumor_trm",
              "trm_like_til_is_exhausted_subset",
              "trm_like_til_is_effector_state",
              "trm_like_til_is_transitional_intermediate",
              "trm_like_til_is_shared_program_distinct_niche"
            ]
          },
          {
            "name": "cf_ahr_assigned_to_til",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Ahr",
            "valueField": "program_specificity",
            "expected": [
              "trm_like_til_specific"
            ]
          },
          {
            "name": "cf_batf_assigned_to_trm_or_shared",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Batf",
            "valueField": "program_specificity",
            "expected": [
              "trm_specific",
              "both"
            ]
          },
          {
            "name": "cf_klf2_called_activator",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Klf2",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          },
          {
            "name": "cf_evidence_ceiling_overclaim",
            "role": "hard_fail",
            "field": "evidence_ceiling",
            "kind": "value",
            "op": "in",
            "expected": [
              "specificity_established_from_regulon_activity",
              "specificity_established_in_dataset",
              "causation_established_by_cross_sectional_concordance",
              "necessity_and_sufficiency_established"
            ]
          },
          {
            "name": "cf_activator_context_shared_or_single",
            "role": "hard_fail",
            "field": "activator_context_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "shared_activators_both_contexts",
              "single_global_activator",
              "redundant_activators"
            ]
          },
          {
            "name": "cf_single_modality_rule",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_hic1_motif_brake_mislabel",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_runx3_anticanonical_as_controller",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Runx3",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-5-pi",
      "modelName": "gpt-5.5",
      "harness": "pi",
      "task": "green_shared_vs_distinct_regulators_trm_vs_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 2981,
      "cost": 2.791939,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_program_map",
            "kind": "list",
            "matchKey": "regulator",
            "k": 3,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "both"
                    ],
                    "canonical": "both"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "represses"
                    ],
                    "canonical": "represses"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Batf",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "trm_like_til_specific"
                    ],
                    "canonical": "trm_like_til_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "chromatin_over_rna"
                    ],
                    "canonical": "chromatin_over_rna"
                  }
                }
              },
              {
                "key": "Ahr",
                "fields": {
                  "program_specificity": {
                    "op": "in",
                    "values": [
                      "trm_specific",
                      "both"
                    ],
                    "canonical": "trm_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "rna_over_chromatin"
                    ],
                    "canonical": "rna_over_chromatin"
                  }
                }
              }
            ]
          },
          {
            "name": "regulator_set_no_extras_gate",
            "role": "gate",
            "field": "$.regulator_program_map[*]",
            "kind": "other",
            "expected": null
          },
          {
            "name": "program_identity_conclusion_gate",
            "role": "gate",
            "field": "program_identity_conclusion",
            "kind": "value",
            "expected": "distinct_state_shared_brake_divergent_activators"
          },
          {
            "name": "residency_install_logic_gate",
            "role": "gate",
            "field": "residency_install_logic",
            "kind": "value",
            "expected": "brake_release_permits_context_matched_activator"
          },
          {
            "name": "activator_context_dependence_gate",
            "role": "gate",
            "field": "activator_context_dependence",
            "kind": "value",
            "expected": "context_matched_crossover"
          },
          {
            "name": "evidence_ceiling_gate",
            "role": "gate",
            "field": "evidence_ceiling",
            "kind": "value",
            "expected": "specificity_dissociations_require_perturbation"
          },
          {
            "name": "cf_install_logic_flat_or_single_step",
            "role": "hard_fail",
            "field": "residency_install_logic",
            "kind": "value",
            "op": "in",
            "expected": [
              "flat_shared_residency_toolkit",
              "activator_induction_sufficient",
              "brake_release_alone_sufficient"
            ]
          },
          {
            "name": "cf_program_identity_lumped_or_wrong",
            "role": "hard_fail",
            "field": "program_identity_conclusion",
            "kind": "value",
            "op": "in",
            "expected": [
              "trm_like_til_is_tumor_trm",
              "trm_like_til_is_exhausted_subset",
              "trm_like_til_is_effector_state",
              "trm_like_til_is_transitional_intermediate",
              "trm_like_til_is_shared_program_distinct_niche"
            ]
          },
          {
            "name": "cf_ahr_assigned_to_til",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Ahr",
            "valueField": "program_specificity",
            "expected": [
              "trm_like_til_specific"
            ]
          },
          {
            "name": "cf_batf_assigned_to_trm_or_shared",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Batf",
            "valueField": "program_specificity",
            "expected": [
              "trm_specific",
              "both"
            ]
          },
          {
            "name": "cf_klf2_called_activator",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Klf2",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          },
          {
            "name": "cf_evidence_ceiling_overclaim",
            "role": "hard_fail",
            "field": "evidence_ceiling",
            "kind": "value",
            "op": "in",
            "expected": [
              "specificity_established_from_regulon_activity",
              "specificity_established_in_dataset",
              "causation_established_by_cross_sectional_concordance",
              "necessity_and_sufficiency_established"
            ]
          },
          {
            "name": "cf_activator_context_shared_or_single",
            "role": "hard_fail",
            "field": "activator_context_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "shared_activators_both_contexts",
              "single_global_activator",
              "redundant_activators"
            ]
          },
          {
            "name": "cf_single_modality_rule",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_hic1_motif_brake_mislabel",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_runx3_anticanonical_as_controller",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Runx3",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-5-pi",
      "modelName": "gpt-5.5",
      "harness": "pi",
      "task": "green_shared_vs_distinct_regulators_trm_vs_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 2010,
      "cost": 2.9173999999999993,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_program_map",
            "kind": "list",
            "matchKey": "regulator",
            "k": 3,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "both"
                    ],
                    "canonical": "both"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "represses"
                    ],
                    "canonical": "represses"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Batf",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "trm_like_til_specific"
                    ],
                    "canonical": "trm_like_til_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "chromatin_over_rna"
                    ],
                    "canonical": "chromatin_over_rna"
                  }
                }
              },
              {
                "key": "Ahr",
                "fields": {
                  "program_specificity": {
                    "op": "in",
                    "values": [
                      "trm_specific",
                      "both"
                    ],
                    "canonical": "trm_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "rna_over_chromatin"
                    ],
                    "canonical": "rna_over_chromatin"
                  }
                }
              }
            ]
          },
          {
            "name": "regulator_set_no_extras_gate",
            "role": "gate",
            "field": "$.regulator_program_map[*]",
            "kind": "other",
            "expected": null
          },
          {
            "name": "program_identity_conclusion_gate",
            "role": "gate",
            "field": "program_identity_conclusion",
            "kind": "value",
            "expected": "distinct_state_shared_brake_divergent_activators"
          },
          {
            "name": "residency_install_logic_gate",
            "role": "gate",
            "field": "residency_install_logic",
            "kind": "value",
            "expected": "brake_release_permits_context_matched_activator"
          },
          {
            "name": "activator_context_dependence_gate",
            "role": "gate",
            "field": "activator_context_dependence",
            "kind": "value",
            "expected": "context_matched_crossover"
          },
          {
            "name": "evidence_ceiling_gate",
            "role": "gate",
            "field": "evidence_ceiling",
            "kind": "value",
            "expected": "specificity_dissociations_require_perturbation"
          },
          {
            "name": "cf_install_logic_flat_or_single_step",
            "role": "hard_fail",
            "field": "residency_install_logic",
            "kind": "value",
            "op": "in",
            "expected": [
              "flat_shared_residency_toolkit",
              "activator_induction_sufficient",
              "brake_release_alone_sufficient"
            ]
          },
          {
            "name": "cf_program_identity_lumped_or_wrong",
            "role": "hard_fail",
            "field": "program_identity_conclusion",
            "kind": "value",
            "op": "in",
            "expected": [
              "trm_like_til_is_tumor_trm",
              "trm_like_til_is_exhausted_subset",
              "trm_like_til_is_effector_state",
              "trm_like_til_is_transitional_intermediate",
              "trm_like_til_is_shared_program_distinct_niche"
            ]
          },
          {
            "name": "cf_ahr_assigned_to_til",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Ahr",
            "valueField": "program_specificity",
            "expected": [
              "trm_like_til_specific"
            ]
          },
          {
            "name": "cf_batf_assigned_to_trm_or_shared",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Batf",
            "valueField": "program_specificity",
            "expected": [
              "trm_specific",
              "both"
            ]
          },
          {
            "name": "cf_klf2_called_activator",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Klf2",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          },
          {
            "name": "cf_evidence_ceiling_overclaim",
            "role": "hard_fail",
            "field": "evidence_ceiling",
            "kind": "value",
            "op": "in",
            "expected": [
              "specificity_established_from_regulon_activity",
              "specificity_established_in_dataset",
              "causation_established_by_cross_sectional_concordance",
              "necessity_and_sufficiency_established"
            ]
          },
          {
            "name": "cf_activator_context_shared_or_single",
            "role": "hard_fail",
            "field": "activator_context_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "shared_activators_both_contexts",
              "single_global_activator",
              "redundant_activators"
            ]
          },
          {
            "name": "cf_single_modality_rule",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_hic1_motif_brake_mislabel",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_runx3_anticanonical_as_controller",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Runx3",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-5-pi",
      "modelName": "gpt-5.5",
      "harness": "pi",
      "task": "green_shared_vs_distinct_regulators_trm_vs_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 4840,
      "cost": 3.4614800000000003,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_program_map",
            "kind": "list",
            "matchKey": "regulator",
            "k": 3,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "both"
                    ],
                    "canonical": "both"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "represses"
                    ],
                    "canonical": "represses"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Batf",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "trm_like_til_specific"
                    ],
                    "canonical": "trm_like_til_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "chromatin_over_rna"
                    ],
                    "canonical": "chromatin_over_rna"
                  }
                }
              },
              {
                "key": "Ahr",
                "fields": {
                  "program_specificity": {
                    "op": "in",
                    "values": [
                      "trm_specific",
                      "both"
                    ],
                    "canonical": "trm_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "rna_over_chromatin"
                    ],
                    "canonical": "rna_over_chromatin"
                  }
                }
              }
            ]
          },
          {
            "name": "regulator_set_no_extras_gate",
            "role": "gate",
            "field": "$.regulator_program_map[*]",
            "kind": "other",
            "expected": null
          },
          {
            "name": "program_identity_conclusion_gate",
            "role": "gate",
            "field": "program_identity_conclusion",
            "kind": "value",
            "expected": "distinct_state_shared_brake_divergent_activators"
          },
          {
            "name": "residency_install_logic_gate",
            "role": "gate",
            "field": "residency_install_logic",
            "kind": "value",
            "expected": "brake_release_permits_context_matched_activator"
          },
          {
            "name": "activator_context_dependence_gate",
            "role": "gate",
            "field": "activator_context_dependence",
            "kind": "value",
            "expected": "context_matched_crossover"
          },
          {
            "name": "evidence_ceiling_gate",
            "role": "gate",
            "field": "evidence_ceiling",
            "kind": "value",
            "expected": "specificity_dissociations_require_perturbation"
          },
          {
            "name": "cf_install_logic_flat_or_single_step",
            "role": "hard_fail",
            "field": "residency_install_logic",
            "kind": "value",
            "op": "in",
            "expected": [
              "flat_shared_residency_toolkit",
              "activator_induction_sufficient",
              "brake_release_alone_sufficient"
            ]
          },
          {
            "name": "cf_program_identity_lumped_or_wrong",
            "role": "hard_fail",
            "field": "program_identity_conclusion",
            "kind": "value",
            "op": "in",
            "expected": [
              "trm_like_til_is_tumor_trm",
              "trm_like_til_is_exhausted_subset",
              "trm_like_til_is_effector_state",
              "trm_like_til_is_transitional_intermediate",
              "trm_like_til_is_shared_program_distinct_niche"
            ]
          },
          {
            "name": "cf_ahr_assigned_to_til",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Ahr",
            "valueField": "program_specificity",
            "expected": [
              "trm_like_til_specific"
            ]
          },
          {
            "name": "cf_batf_assigned_to_trm_or_shared",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Batf",
            "valueField": "program_specificity",
            "expected": [
              "trm_specific",
              "both"
            ]
          },
          {
            "name": "cf_klf2_called_activator",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Klf2",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          },
          {
            "name": "cf_evidence_ceiling_overclaim",
            "role": "hard_fail",
            "field": "evidence_ceiling",
            "kind": "value",
            "op": "in",
            "expected": [
              "specificity_established_from_regulon_activity",
              "specificity_established_in_dataset",
              "causation_established_by_cross_sectional_concordance",
              "necessity_and_sufficiency_established"
            ]
          },
          {
            "name": "cf_activator_context_shared_or_single",
            "role": "hard_fail",
            "field": "activator_context_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "shared_activators_both_contexts",
              "single_global_activator",
              "redundant_activators"
            ]
          },
          {
            "name": "cf_single_modality_rule",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_hic1_motif_brake_mislabel",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_runx3_anticanonical_as_controller",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Runx3",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-20-beta-reasoning-pi",
      "modelName": "grok-4.20-beta-0309-reasoning",
      "harness": "pi",
      "task": "green_shared_vs_distinct_regulators_trm_vs_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 373,
      "cost": 0.0327914,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_program_map",
            "kind": "list",
            "matchKey": "regulator",
            "k": 3,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "both"
                    ],
                    "canonical": "both"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "represses"
                    ],
                    "canonical": "represses"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Batf",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "trm_like_til_specific"
                    ],
                    "canonical": "trm_like_til_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "chromatin_over_rna"
                    ],
                    "canonical": "chromatin_over_rna"
                  }
                }
              },
              {
                "key": "Ahr",
                "fields": {
                  "program_specificity": {
                    "op": "in",
                    "values": [
                      "trm_specific",
                      "both"
                    ],
                    "canonical": "trm_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "rna_over_chromatin"
                    ],
                    "canonical": "rna_over_chromatin"
                  }
                }
              }
            ]
          },
          {
            "name": "regulator_set_no_extras_gate",
            "role": "gate",
            "field": "$.regulator_program_map[*]",
            "kind": "other",
            "expected": null
          },
          {
            "name": "program_identity_conclusion_gate",
            "role": "gate",
            "field": "program_identity_conclusion",
            "kind": "value",
            "expected": "distinct_state_shared_brake_divergent_activators"
          },
          {
            "name": "residency_install_logic_gate",
            "role": "gate",
            "field": "residency_install_logic",
            "kind": "value",
            "expected": "brake_release_permits_context_matched_activator"
          },
          {
            "name": "activator_context_dependence_gate",
            "role": "gate",
            "field": "activator_context_dependence",
            "kind": "value",
            "expected": "context_matched_crossover"
          },
          {
            "name": "evidence_ceiling_gate",
            "role": "gate",
            "field": "evidence_ceiling",
            "kind": "value",
            "expected": "specificity_dissociations_require_perturbation"
          },
          {
            "name": "cf_install_logic_flat_or_single_step",
            "role": "hard_fail",
            "field": "residency_install_logic",
            "kind": "value",
            "op": "in",
            "expected": [
              "flat_shared_residency_toolkit",
              "activator_induction_sufficient",
              "brake_release_alone_sufficient"
            ]
          },
          {
            "name": "cf_program_identity_lumped_or_wrong",
            "role": "hard_fail",
            "field": "program_identity_conclusion",
            "kind": "value",
            "op": "in",
            "expected": [
              "trm_like_til_is_tumor_trm",
              "trm_like_til_is_exhausted_subset",
              "trm_like_til_is_effector_state",
              "trm_like_til_is_transitional_intermediate",
              "trm_like_til_is_shared_program_distinct_niche"
            ]
          },
          {
            "name": "cf_ahr_assigned_to_til",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Ahr",
            "valueField": "program_specificity",
            "expected": [
              "trm_like_til_specific"
            ]
          },
          {
            "name": "cf_batf_assigned_to_trm_or_shared",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Batf",
            "valueField": "program_specificity",
            "expected": [
              "trm_specific",
              "both"
            ]
          },
          {
            "name": "cf_klf2_called_activator",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Klf2",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          },
          {
            "name": "cf_evidence_ceiling_overclaim",
            "role": "hard_fail",
            "field": "evidence_ceiling",
            "kind": "value",
            "op": "in",
            "expected": [
              "specificity_established_from_regulon_activity",
              "specificity_established_in_dataset",
              "causation_established_by_cross_sectional_concordance",
              "necessity_and_sufficiency_established"
            ]
          },
          {
            "name": "cf_activator_context_shared_or_single",
            "role": "hard_fail",
            "field": "activator_context_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "shared_activators_both_contexts",
              "single_global_activator",
              "redundant_activators"
            ]
          },
          {
            "name": "cf_single_modality_rule",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_hic1_motif_brake_mislabel",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_runx3_anticanonical_as_controller",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Runx3",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-20-beta-reasoning-pi",
      "modelName": "grok-4.20-beta-0309-reasoning",
      "harness": "pi",
      "task": "green_shared_vs_distinct_regulators_trm_vs_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 791,
      "cost": 0.05095835,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_program_map",
            "kind": "list",
            "matchKey": "regulator",
            "k": 3,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "both"
                    ],
                    "canonical": "both"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "represses"
                    ],
                    "canonical": "represses"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Batf",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "trm_like_til_specific"
                    ],
                    "canonical": "trm_like_til_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "chromatin_over_rna"
                    ],
                    "canonical": "chromatin_over_rna"
                  }
                }
              },
              {
                "key": "Ahr",
                "fields": {
                  "program_specificity": {
                    "op": "in",
                    "values": [
                      "trm_specific",
                      "both"
                    ],
                    "canonical": "trm_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "rna_over_chromatin"
                    ],
                    "canonical": "rna_over_chromatin"
                  }
                }
              }
            ]
          },
          {
            "name": "regulator_set_no_extras_gate",
            "role": "gate",
            "field": "$.regulator_program_map[*]",
            "kind": "other",
            "expected": null
          },
          {
            "name": "program_identity_conclusion_gate",
            "role": "gate",
            "field": "program_identity_conclusion",
            "kind": "value",
            "expected": "distinct_state_shared_brake_divergent_activators"
          },
          {
            "name": "residency_install_logic_gate",
            "role": "gate",
            "field": "residency_install_logic",
            "kind": "value",
            "expected": "brake_release_permits_context_matched_activator"
          },
          {
            "name": "activator_context_dependence_gate",
            "role": "gate",
            "field": "activator_context_dependence",
            "kind": "value",
            "expected": "context_matched_crossover"
          },
          {
            "name": "evidence_ceiling_gate",
            "role": "gate",
            "field": "evidence_ceiling",
            "kind": "value",
            "expected": "specificity_dissociations_require_perturbation"
          },
          {
            "name": "cf_install_logic_flat_or_single_step",
            "role": "hard_fail",
            "field": "residency_install_logic",
            "kind": "value",
            "op": "in",
            "expected": [
              "flat_shared_residency_toolkit",
              "activator_induction_sufficient",
              "brake_release_alone_sufficient"
            ]
          },
          {
            "name": "cf_program_identity_lumped_or_wrong",
            "role": "hard_fail",
            "field": "program_identity_conclusion",
            "kind": "value",
            "op": "in",
            "expected": [
              "trm_like_til_is_tumor_trm",
              "trm_like_til_is_exhausted_subset",
              "trm_like_til_is_effector_state",
              "trm_like_til_is_transitional_intermediate",
              "trm_like_til_is_shared_program_distinct_niche"
            ]
          },
          {
            "name": "cf_ahr_assigned_to_til",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Ahr",
            "valueField": "program_specificity",
            "expected": [
              "trm_like_til_specific"
            ]
          },
          {
            "name": "cf_batf_assigned_to_trm_or_shared",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Batf",
            "valueField": "program_specificity",
            "expected": [
              "trm_specific",
              "both"
            ]
          },
          {
            "name": "cf_klf2_called_activator",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Klf2",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          },
          {
            "name": "cf_evidence_ceiling_overclaim",
            "role": "hard_fail",
            "field": "evidence_ceiling",
            "kind": "value",
            "op": "in",
            "expected": [
              "specificity_established_from_regulon_activity",
              "specificity_established_in_dataset",
              "causation_established_by_cross_sectional_concordance",
              "necessity_and_sufficiency_established"
            ]
          },
          {
            "name": "cf_activator_context_shared_or_single",
            "role": "hard_fail",
            "field": "activator_context_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "shared_activators_both_contexts",
              "single_global_activator",
              "redundant_activators"
            ]
          },
          {
            "name": "cf_single_modality_rule",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_hic1_motif_brake_mislabel",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_runx3_anticanonical_as_controller",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Runx3",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-20-beta-reasoning-pi",
      "modelName": "grok-4.20-beta-0309-reasoning",
      "harness": "pi",
      "task": "green_shared_vs_distinct_regulators_trm_vs_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 343,
      "cost": 0.054211850000000006,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_program_map",
            "kind": "list",
            "matchKey": "regulator",
            "k": 3,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "both"
                    ],
                    "canonical": "both"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "represses"
                    ],
                    "canonical": "represses"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Batf",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "trm_like_til_specific"
                    ],
                    "canonical": "trm_like_til_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "chromatin_over_rna"
                    ],
                    "canonical": "chromatin_over_rna"
                  }
                }
              },
              {
                "key": "Ahr",
                "fields": {
                  "program_specificity": {
                    "op": "in",
                    "values": [
                      "trm_specific",
                      "both"
                    ],
                    "canonical": "trm_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "rna_over_chromatin"
                    ],
                    "canonical": "rna_over_chromatin"
                  }
                }
              }
            ]
          },
          {
            "name": "regulator_set_no_extras_gate",
            "role": "gate",
            "field": "$.regulator_program_map[*]",
            "kind": "other",
            "expected": null
          },
          {
            "name": "program_identity_conclusion_gate",
            "role": "gate",
            "field": "program_identity_conclusion",
            "kind": "value",
            "expected": "distinct_state_shared_brake_divergent_activators"
          },
          {
            "name": "residency_install_logic_gate",
            "role": "gate",
            "field": "residency_install_logic",
            "kind": "value",
            "expected": "brake_release_permits_context_matched_activator"
          },
          {
            "name": "activator_context_dependence_gate",
            "role": "gate",
            "field": "activator_context_dependence",
            "kind": "value",
            "expected": "context_matched_crossover"
          },
          {
            "name": "evidence_ceiling_gate",
            "role": "gate",
            "field": "evidence_ceiling",
            "kind": "value",
            "expected": "specificity_dissociations_require_perturbation"
          },
          {
            "name": "cf_install_logic_flat_or_single_step",
            "role": "hard_fail",
            "field": "residency_install_logic",
            "kind": "value",
            "op": "in",
            "expected": [
              "flat_shared_residency_toolkit",
              "activator_induction_sufficient",
              "brake_release_alone_sufficient"
            ]
          },
          {
            "name": "cf_program_identity_lumped_or_wrong",
            "role": "hard_fail",
            "field": "program_identity_conclusion",
            "kind": "value",
            "op": "in",
            "expected": [
              "trm_like_til_is_tumor_trm",
              "trm_like_til_is_exhausted_subset",
              "trm_like_til_is_effector_state",
              "trm_like_til_is_transitional_intermediate",
              "trm_like_til_is_shared_program_distinct_niche"
            ]
          },
          {
            "name": "cf_ahr_assigned_to_til",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Ahr",
            "valueField": "program_specificity",
            "expected": [
              "trm_like_til_specific"
            ]
          },
          {
            "name": "cf_batf_assigned_to_trm_or_shared",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Batf",
            "valueField": "program_specificity",
            "expected": [
              "trm_specific",
              "both"
            ]
          },
          {
            "name": "cf_klf2_called_activator",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Klf2",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          },
          {
            "name": "cf_evidence_ceiling_overclaim",
            "role": "hard_fail",
            "field": "evidence_ceiling",
            "kind": "value",
            "op": "in",
            "expected": [
              "specificity_established_from_regulon_activity",
              "specificity_established_in_dataset",
              "causation_established_by_cross_sectional_concordance",
              "necessity_and_sufficiency_established"
            ]
          },
          {
            "name": "cf_activator_context_shared_or_single",
            "role": "hard_fail",
            "field": "activator_context_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "shared_activators_both_contexts",
              "single_global_activator",
              "redundant_activators"
            ]
          },
          {
            "name": "cf_single_modality_rule",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_hic1_motif_brake_mislabel",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_runx3_anticanonical_as_controller",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Runx3",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-3-pi",
      "modelName": "grok-4.3",
      "harness": "pi",
      "task": "green_shared_vs_distinct_regulators_trm_vs_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 188,
      "cost": 0.023381150000000003,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_program_map",
            "kind": "list",
            "matchKey": "regulator",
            "k": 3,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "both"
                    ],
                    "canonical": "both"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "represses"
                    ],
                    "canonical": "represses"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Batf",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "trm_like_til_specific"
                    ],
                    "canonical": "trm_like_til_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "chromatin_over_rna"
                    ],
                    "canonical": "chromatin_over_rna"
                  }
                }
              },
              {
                "key": "Ahr",
                "fields": {
                  "program_specificity": {
                    "op": "in",
                    "values": [
                      "trm_specific",
                      "both"
                    ],
                    "canonical": "trm_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "rna_over_chromatin"
                    ],
                    "canonical": "rna_over_chromatin"
                  }
                }
              }
            ]
          },
          {
            "name": "regulator_set_no_extras_gate",
            "role": "gate",
            "field": "$.regulator_program_map[*]",
            "kind": "other",
            "expected": null
          },
          {
            "name": "program_identity_conclusion_gate",
            "role": "gate",
            "field": "program_identity_conclusion",
            "kind": "value",
            "expected": "distinct_state_shared_brake_divergent_activators"
          },
          {
            "name": "residency_install_logic_gate",
            "role": "gate",
            "field": "residency_install_logic",
            "kind": "value",
            "expected": "brake_release_permits_context_matched_activator"
          },
          {
            "name": "activator_context_dependence_gate",
            "role": "gate",
            "field": "activator_context_dependence",
            "kind": "value",
            "expected": "context_matched_crossover"
          },
          {
            "name": "evidence_ceiling_gate",
            "role": "gate",
            "field": "evidence_ceiling",
            "kind": "value",
            "expected": "specificity_dissociations_require_perturbation"
          },
          {
            "name": "cf_install_logic_flat_or_single_step",
            "role": "hard_fail",
            "field": "residency_install_logic",
            "kind": "value",
            "op": "in",
            "expected": [
              "flat_shared_residency_toolkit",
              "activator_induction_sufficient",
              "brake_release_alone_sufficient"
            ]
          },
          {
            "name": "cf_program_identity_lumped_or_wrong",
            "role": "hard_fail",
            "field": "program_identity_conclusion",
            "kind": "value",
            "op": "in",
            "expected": [
              "trm_like_til_is_tumor_trm",
              "trm_like_til_is_exhausted_subset",
              "trm_like_til_is_effector_state",
              "trm_like_til_is_transitional_intermediate",
              "trm_like_til_is_shared_program_distinct_niche"
            ]
          },
          {
            "name": "cf_ahr_assigned_to_til",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Ahr",
            "valueField": "program_specificity",
            "expected": [
              "trm_like_til_specific"
            ]
          },
          {
            "name": "cf_batf_assigned_to_trm_or_shared",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Batf",
            "valueField": "program_specificity",
            "expected": [
              "trm_specific",
              "both"
            ]
          },
          {
            "name": "cf_klf2_called_activator",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Klf2",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          },
          {
            "name": "cf_evidence_ceiling_overclaim",
            "role": "hard_fail",
            "field": "evidence_ceiling",
            "kind": "value",
            "op": "in",
            "expected": [
              "specificity_established_from_regulon_activity",
              "specificity_established_in_dataset",
              "causation_established_by_cross_sectional_concordance",
              "necessity_and_sufficiency_established"
            ]
          },
          {
            "name": "cf_activator_context_shared_or_single",
            "role": "hard_fail",
            "field": "activator_context_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "shared_activators_both_contexts",
              "single_global_activator",
              "redundant_activators"
            ]
          },
          {
            "name": "cf_single_modality_rule",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_hic1_motif_brake_mislabel",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_runx3_anticanonical_as_controller",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Runx3",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-3-pi",
      "modelName": "grok-4.3",
      "harness": "pi",
      "task": "green_shared_vs_distinct_regulators_trm_vs_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 175,
      "cost": 0.0181889,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_program_map",
            "kind": "list",
            "matchKey": "regulator",
            "k": 3,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "both"
                    ],
                    "canonical": "both"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "represses"
                    ],
                    "canonical": "represses"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Batf",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "trm_like_til_specific"
                    ],
                    "canonical": "trm_like_til_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "chromatin_over_rna"
                    ],
                    "canonical": "chromatin_over_rna"
                  }
                }
              },
              {
                "key": "Ahr",
                "fields": {
                  "program_specificity": {
                    "op": "in",
                    "values": [
                      "trm_specific",
                      "both"
                    ],
                    "canonical": "trm_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "rna_over_chromatin"
                    ],
                    "canonical": "rna_over_chromatin"
                  }
                }
              }
            ]
          },
          {
            "name": "regulator_set_no_extras_gate",
            "role": "gate",
            "field": "$.regulator_program_map[*]",
            "kind": "other",
            "expected": null
          },
          {
            "name": "program_identity_conclusion_gate",
            "role": "gate",
            "field": "program_identity_conclusion",
            "kind": "value",
            "expected": "distinct_state_shared_brake_divergent_activators"
          },
          {
            "name": "residency_install_logic_gate",
            "role": "gate",
            "field": "residency_install_logic",
            "kind": "value",
            "expected": "brake_release_permits_context_matched_activator"
          },
          {
            "name": "activator_context_dependence_gate",
            "role": "gate",
            "field": "activator_context_dependence",
            "kind": "value",
            "expected": "context_matched_crossover"
          },
          {
            "name": "evidence_ceiling_gate",
            "role": "gate",
            "field": "evidence_ceiling",
            "kind": "value",
            "expected": "specificity_dissociations_require_perturbation"
          },
          {
            "name": "cf_install_logic_flat_or_single_step",
            "role": "hard_fail",
            "field": "residency_install_logic",
            "kind": "value",
            "op": "in",
            "expected": [
              "flat_shared_residency_toolkit",
              "activator_induction_sufficient",
              "brake_release_alone_sufficient"
            ]
          },
          {
            "name": "cf_program_identity_lumped_or_wrong",
            "role": "hard_fail",
            "field": "program_identity_conclusion",
            "kind": "value",
            "op": "in",
            "expected": [
              "trm_like_til_is_tumor_trm",
              "trm_like_til_is_exhausted_subset",
              "trm_like_til_is_effector_state",
              "trm_like_til_is_transitional_intermediate",
              "trm_like_til_is_shared_program_distinct_niche"
            ]
          },
          {
            "name": "cf_ahr_assigned_to_til",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Ahr",
            "valueField": "program_specificity",
            "expected": [
              "trm_like_til_specific"
            ]
          },
          {
            "name": "cf_batf_assigned_to_trm_or_shared",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Batf",
            "valueField": "program_specificity",
            "expected": [
              "trm_specific",
              "both"
            ]
          },
          {
            "name": "cf_klf2_called_activator",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Klf2",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          },
          {
            "name": "cf_evidence_ceiling_overclaim",
            "role": "hard_fail",
            "field": "evidence_ceiling",
            "kind": "value",
            "op": "in",
            "expected": [
              "specificity_established_from_regulon_activity",
              "specificity_established_in_dataset",
              "causation_established_by_cross_sectional_concordance",
              "necessity_and_sufficiency_established"
            ]
          },
          {
            "name": "cf_activator_context_shared_or_single",
            "role": "hard_fail",
            "field": "activator_context_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "shared_activators_both_contexts",
              "single_global_activator",
              "redundant_activators"
            ]
          },
          {
            "name": "cf_single_modality_rule",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_hic1_motif_brake_mislabel",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_runx3_anticanonical_as_controller",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Runx3",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-3-pi",
      "modelName": "grok-4.3",
      "harness": "pi",
      "task": "green_shared_vs_distinct_regulators_trm_vs_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 208,
      "cost": 0.022683750000000003,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_program_map",
            "kind": "list",
            "matchKey": "regulator",
            "k": 3,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "both"
                    ],
                    "canonical": "both"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "represses"
                    ],
                    "canonical": "represses"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Batf",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "trm_like_til_specific"
                    ],
                    "canonical": "trm_like_til_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "chromatin_over_rna"
                    ],
                    "canonical": "chromatin_over_rna"
                  }
                }
              },
              {
                "key": "Ahr",
                "fields": {
                  "program_specificity": {
                    "op": "in",
                    "values": [
                      "trm_specific",
                      "both"
                    ],
                    "canonical": "trm_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "rna_over_chromatin"
                    ],
                    "canonical": "rna_over_chromatin"
                  }
                }
              }
            ]
          },
          {
            "name": "regulator_set_no_extras_gate",
            "role": "gate",
            "field": "$.regulator_program_map[*]",
            "kind": "other",
            "expected": null
          },
          {
            "name": "program_identity_conclusion_gate",
            "role": "gate",
            "field": "program_identity_conclusion",
            "kind": "value",
            "expected": "distinct_state_shared_brake_divergent_activators"
          },
          {
            "name": "residency_install_logic_gate",
            "role": "gate",
            "field": "residency_install_logic",
            "kind": "value",
            "expected": "brake_release_permits_context_matched_activator"
          },
          {
            "name": "activator_context_dependence_gate",
            "role": "gate",
            "field": "activator_context_dependence",
            "kind": "value",
            "expected": "context_matched_crossover"
          },
          {
            "name": "evidence_ceiling_gate",
            "role": "gate",
            "field": "evidence_ceiling",
            "kind": "value",
            "expected": "specificity_dissociations_require_perturbation"
          },
          {
            "name": "cf_install_logic_flat_or_single_step",
            "role": "hard_fail",
            "field": "residency_install_logic",
            "kind": "value",
            "op": "in",
            "expected": [
              "flat_shared_residency_toolkit",
              "activator_induction_sufficient",
              "brake_release_alone_sufficient"
            ]
          },
          {
            "name": "cf_program_identity_lumped_or_wrong",
            "role": "hard_fail",
            "field": "program_identity_conclusion",
            "kind": "value",
            "op": "in",
            "expected": [
              "trm_like_til_is_tumor_trm",
              "trm_like_til_is_exhausted_subset",
              "trm_like_til_is_effector_state",
              "trm_like_til_is_transitional_intermediate",
              "trm_like_til_is_shared_program_distinct_niche"
            ]
          },
          {
            "name": "cf_ahr_assigned_to_til",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Ahr",
            "valueField": "program_specificity",
            "expected": [
              "trm_like_til_specific"
            ]
          },
          {
            "name": "cf_batf_assigned_to_trm_or_shared",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Batf",
            "valueField": "program_specificity",
            "expected": [
              "trm_specific",
              "both"
            ]
          },
          {
            "name": "cf_klf2_called_activator",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Klf2",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          },
          {
            "name": "cf_evidence_ceiling_overclaim",
            "role": "hard_fail",
            "field": "evidence_ceiling",
            "kind": "value",
            "op": "in",
            "expected": [
              "specificity_established_from_regulon_activity",
              "specificity_established_in_dataset",
              "causation_established_by_cross_sectional_concordance",
              "necessity_and_sufficiency_established"
            ]
          },
          {
            "name": "cf_activator_context_shared_or_single",
            "role": "hard_fail",
            "field": "activator_context_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "shared_activators_both_contexts",
              "single_global_activator",
              "redundant_activators"
            ]
          },
          {
            "name": "cf_single_modality_rule",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_hic1_motif_brake_mislabel",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_runx3_anticanonical_as_controller",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Runx3",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-kimi-k2p6-pi",
      "modelName": "kimi-k2p6",
      "harness": "pi",
      "task": "green_shared_vs_distinct_regulators_trm_vs_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 5359,
      "cost": 1.32628944,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_program_map",
            "kind": "list",
            "matchKey": "regulator",
            "k": 3,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "both"
                    ],
                    "canonical": "both"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "represses"
                    ],
                    "canonical": "represses"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Batf",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "trm_like_til_specific"
                    ],
                    "canonical": "trm_like_til_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "chromatin_over_rna"
                    ],
                    "canonical": "chromatin_over_rna"
                  }
                }
              },
              {
                "key": "Ahr",
                "fields": {
                  "program_specificity": {
                    "op": "in",
                    "values": [
                      "trm_specific",
                      "both"
                    ],
                    "canonical": "trm_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "rna_over_chromatin"
                    ],
                    "canonical": "rna_over_chromatin"
                  }
                }
              }
            ]
          },
          {
            "name": "regulator_set_no_extras_gate",
            "role": "gate",
            "field": "$.regulator_program_map[*]",
            "kind": "other",
            "expected": null
          },
          {
            "name": "program_identity_conclusion_gate",
            "role": "gate",
            "field": "program_identity_conclusion",
            "kind": "value",
            "expected": "distinct_state_shared_brake_divergent_activators"
          },
          {
            "name": "residency_install_logic_gate",
            "role": "gate",
            "field": "residency_install_logic",
            "kind": "value",
            "expected": "brake_release_permits_context_matched_activator"
          },
          {
            "name": "activator_context_dependence_gate",
            "role": "gate",
            "field": "activator_context_dependence",
            "kind": "value",
            "expected": "context_matched_crossover"
          },
          {
            "name": "evidence_ceiling_gate",
            "role": "gate",
            "field": "evidence_ceiling",
            "kind": "value",
            "expected": "specificity_dissociations_require_perturbation"
          },
          {
            "name": "cf_install_logic_flat_or_single_step",
            "role": "hard_fail",
            "field": "residency_install_logic",
            "kind": "value",
            "op": "in",
            "expected": [
              "flat_shared_residency_toolkit",
              "activator_induction_sufficient",
              "brake_release_alone_sufficient"
            ]
          },
          {
            "name": "cf_program_identity_lumped_or_wrong",
            "role": "hard_fail",
            "field": "program_identity_conclusion",
            "kind": "value",
            "op": "in",
            "expected": [
              "trm_like_til_is_tumor_trm",
              "trm_like_til_is_exhausted_subset",
              "trm_like_til_is_effector_state",
              "trm_like_til_is_transitional_intermediate",
              "trm_like_til_is_shared_program_distinct_niche"
            ]
          },
          {
            "name": "cf_ahr_assigned_to_til",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Ahr",
            "valueField": "program_specificity",
            "expected": [
              "trm_like_til_specific"
            ]
          },
          {
            "name": "cf_batf_assigned_to_trm_or_shared",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Batf",
            "valueField": "program_specificity",
            "expected": [
              "trm_specific",
              "both"
            ]
          },
          {
            "name": "cf_klf2_called_activator",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Klf2",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          },
          {
            "name": "cf_evidence_ceiling_overclaim",
            "role": "hard_fail",
            "field": "evidence_ceiling",
            "kind": "value",
            "op": "in",
            "expected": [
              "specificity_established_from_regulon_activity",
              "specificity_established_in_dataset",
              "causation_established_by_cross_sectional_concordance",
              "necessity_and_sufficiency_established"
            ]
          },
          {
            "name": "cf_activator_context_shared_or_single",
            "role": "hard_fail",
            "field": "activator_context_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "shared_activators_both_contexts",
              "single_global_activator",
              "redundant_activators"
            ]
          },
          {
            "name": "cf_single_modality_rule",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_hic1_motif_brake_mislabel",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_runx3_anticanonical_as_controller",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Runx3",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-kimi-k2p6-pi",
      "modelName": "kimi-k2p6",
      "harness": "pi",
      "task": "green_shared_vs_distinct_regulators_trm_vs_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 3015,
      "cost": 1.29066343,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_program_map",
            "kind": "list",
            "matchKey": "regulator",
            "k": 3,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "both"
                    ],
                    "canonical": "both"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "represses"
                    ],
                    "canonical": "represses"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Batf",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "trm_like_til_specific"
                    ],
                    "canonical": "trm_like_til_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "chromatin_over_rna"
                    ],
                    "canonical": "chromatin_over_rna"
                  }
                }
              },
              {
                "key": "Ahr",
                "fields": {
                  "program_specificity": {
                    "op": "in",
                    "values": [
                      "trm_specific",
                      "both"
                    ],
                    "canonical": "trm_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "rna_over_chromatin"
                    ],
                    "canonical": "rna_over_chromatin"
                  }
                }
              }
            ]
          },
          {
            "name": "regulator_set_no_extras_gate",
            "role": "gate",
            "field": "$.regulator_program_map[*]",
            "kind": "other",
            "expected": null
          },
          {
            "name": "program_identity_conclusion_gate",
            "role": "gate",
            "field": "program_identity_conclusion",
            "kind": "value",
            "expected": "distinct_state_shared_brake_divergent_activators"
          },
          {
            "name": "residency_install_logic_gate",
            "role": "gate",
            "field": "residency_install_logic",
            "kind": "value",
            "expected": "brake_release_permits_context_matched_activator"
          },
          {
            "name": "activator_context_dependence_gate",
            "role": "gate",
            "field": "activator_context_dependence",
            "kind": "value",
            "expected": "context_matched_crossover"
          },
          {
            "name": "evidence_ceiling_gate",
            "role": "gate",
            "field": "evidence_ceiling",
            "kind": "value",
            "expected": "specificity_dissociations_require_perturbation"
          },
          {
            "name": "cf_install_logic_flat_or_single_step",
            "role": "hard_fail",
            "field": "residency_install_logic",
            "kind": "value",
            "op": "in",
            "expected": [
              "flat_shared_residency_toolkit",
              "activator_induction_sufficient",
              "brake_release_alone_sufficient"
            ]
          },
          {
            "name": "cf_program_identity_lumped_or_wrong",
            "role": "hard_fail",
            "field": "program_identity_conclusion",
            "kind": "value",
            "op": "in",
            "expected": [
              "trm_like_til_is_tumor_trm",
              "trm_like_til_is_exhausted_subset",
              "trm_like_til_is_effector_state",
              "trm_like_til_is_transitional_intermediate",
              "trm_like_til_is_shared_program_distinct_niche"
            ]
          },
          {
            "name": "cf_ahr_assigned_to_til",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Ahr",
            "valueField": "program_specificity",
            "expected": [
              "trm_like_til_specific"
            ]
          },
          {
            "name": "cf_batf_assigned_to_trm_or_shared",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Batf",
            "valueField": "program_specificity",
            "expected": [
              "trm_specific",
              "both"
            ]
          },
          {
            "name": "cf_klf2_called_activator",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Klf2",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          },
          {
            "name": "cf_evidence_ceiling_overclaim",
            "role": "hard_fail",
            "field": "evidence_ceiling",
            "kind": "value",
            "op": "in",
            "expected": [
              "specificity_established_from_regulon_activity",
              "specificity_established_in_dataset",
              "causation_established_by_cross_sectional_concordance",
              "necessity_and_sufficiency_established"
            ]
          },
          {
            "name": "cf_activator_context_shared_or_single",
            "role": "hard_fail",
            "field": "activator_context_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "shared_activators_both_contexts",
              "single_global_activator",
              "redundant_activators"
            ]
          },
          {
            "name": "cf_single_modality_rule",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_hic1_motif_brake_mislabel",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_runx3_anticanonical_as_controller",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Runx3",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-kimi-k2p6-pi",
      "modelName": "kimi-k2p6",
      "harness": "pi",
      "task": "green_shared_vs_distinct_regulators_trm_vs_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 3389,
      "cost": 1.15798354,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_program_map",
            "kind": "list",
            "matchKey": "regulator",
            "k": 3,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "both"
                    ],
                    "canonical": "both"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "represses"
                    ],
                    "canonical": "represses"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Batf",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "trm_like_til_specific"
                    ],
                    "canonical": "trm_like_til_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "chromatin_over_rna"
                    ],
                    "canonical": "chromatin_over_rna"
                  }
                }
              },
              {
                "key": "Ahr",
                "fields": {
                  "program_specificity": {
                    "op": "in",
                    "values": [
                      "trm_specific",
                      "both"
                    ],
                    "canonical": "trm_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "rna_over_chromatin"
                    ],
                    "canonical": "rna_over_chromatin"
                  }
                }
              }
            ]
          },
          {
            "name": "regulator_set_no_extras_gate",
            "role": "gate",
            "field": "$.regulator_program_map[*]",
            "kind": "other",
            "expected": null
          },
          {
            "name": "program_identity_conclusion_gate",
            "role": "gate",
            "field": "program_identity_conclusion",
            "kind": "value",
            "expected": "distinct_state_shared_brake_divergent_activators"
          },
          {
            "name": "residency_install_logic_gate",
            "role": "gate",
            "field": "residency_install_logic",
            "kind": "value",
            "expected": "brake_release_permits_context_matched_activator"
          },
          {
            "name": "activator_context_dependence_gate",
            "role": "gate",
            "field": "activator_context_dependence",
            "kind": "value",
            "expected": "context_matched_crossover"
          },
          {
            "name": "evidence_ceiling_gate",
            "role": "gate",
            "field": "evidence_ceiling",
            "kind": "value",
            "expected": "specificity_dissociations_require_perturbation"
          },
          {
            "name": "cf_install_logic_flat_or_single_step",
            "role": "hard_fail",
            "field": "residency_install_logic",
            "kind": "value",
            "op": "in",
            "expected": [
              "flat_shared_residency_toolkit",
              "activator_induction_sufficient",
              "brake_release_alone_sufficient"
            ]
          },
          {
            "name": "cf_program_identity_lumped_or_wrong",
            "role": "hard_fail",
            "field": "program_identity_conclusion",
            "kind": "value",
            "op": "in",
            "expected": [
              "trm_like_til_is_tumor_trm",
              "trm_like_til_is_exhausted_subset",
              "trm_like_til_is_effector_state",
              "trm_like_til_is_transitional_intermediate",
              "trm_like_til_is_shared_program_distinct_niche"
            ]
          },
          {
            "name": "cf_ahr_assigned_to_til",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Ahr",
            "valueField": "program_specificity",
            "expected": [
              "trm_like_til_specific"
            ]
          },
          {
            "name": "cf_batf_assigned_to_trm_or_shared",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Batf",
            "valueField": "program_specificity",
            "expected": [
              "trm_specific",
              "both"
            ]
          },
          {
            "name": "cf_klf2_called_activator",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Klf2",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          },
          {
            "name": "cf_evidence_ceiling_overclaim",
            "role": "hard_fail",
            "field": "evidence_ceiling",
            "kind": "value",
            "op": "in",
            "expected": [
              "specificity_established_from_regulon_activity",
              "specificity_established_in_dataset",
              "causation_established_by_cross_sectional_concordance",
              "necessity_and_sufficiency_established"
            ]
          },
          {
            "name": "cf_activator_context_shared_or_single",
            "role": "hard_fail",
            "field": "activator_context_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "shared_activators_both_contexts",
              "single_global_activator",
              "redundant_activators"
            ]
          },
          {
            "name": "cf_single_modality_rule",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_hic1_motif_brake_mislabel",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_runx3_anticanonical_as_controller",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Runx3",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-6-claude-code",
      "modelName": "claude-opus-4-6",
      "harness": "claude-code",
      "task": "gse155381_q4_pseudotime_delay",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 1,
      "passed": true,
      "score": null,
      "durationS": 845,
      "cost": 1.6875134999999999,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "pseudotime_direction_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "E17_chronological_match_weighted",
            "role": "additive",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pluripotency_exit_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pseudotime_direction_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_synchronous_with_monkey_EPI",
              "chimera_human_EPI_developmentally_advanced",
              "chimera_human_EPI_delayed_at_pre_implantation_only_resolved_post_implantation"
            ]
          },
          {
            "name": "E17_chronological_match_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "resembles_monkey_E17_Gast_EPI",
              "resembles_chimera_monkey_E17_EPI_specifically",
              "chimera_human_E17_EPI_more_advanced_than_monkey_E17_Gast"
            ]
          },
          {
            "name": "pluripotency_exit_paper_inverted_or_prior_driven",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_advances_pluripotency_exit_prematurely",
              "naive_pluripotency_program_specifically_engaged",
              "pluripotency_core_retained_via_KLF17_naive_marker_specifically"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-6-claude-code",
      "modelName": "claude-opus-4-6",
      "harness": "claude-code",
      "task": "gse155381_q4_pseudotime_delay",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 1672,
      "cost": 3.347985,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "pseudotime_direction_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "E17_chronological_match_weighted",
            "role": "additive",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pluripotency_exit_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pseudotime_direction_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_synchronous_with_monkey_EPI",
              "chimera_human_EPI_developmentally_advanced",
              "chimera_human_EPI_delayed_at_pre_implantation_only_resolved_post_implantation"
            ]
          },
          {
            "name": "E17_chronological_match_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "resembles_monkey_E17_Gast_EPI",
              "resembles_chimera_monkey_E17_EPI_specifically",
              "chimera_human_E17_EPI_more_advanced_than_monkey_E17_Gast"
            ]
          },
          {
            "name": "pluripotency_exit_paper_inverted_or_prior_driven",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_advances_pluripotency_exit_prematurely",
              "naive_pluripotency_program_specifically_engaged",
              "pluripotency_core_retained_via_KLF17_naive_marker_specifically"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-6-claude-code",
      "modelName": "claude-opus-4-6",
      "harness": "claude-code",
      "task": "gse155381_q4_pseudotime_delay",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 3,
      "passed": true,
      "score": null,
      "durationS": 693,
      "cost": 1.6949665,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "pseudotime_direction_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "E17_chronological_match_weighted",
            "role": "additive",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pluripotency_exit_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pseudotime_direction_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_synchronous_with_monkey_EPI",
              "chimera_human_EPI_developmentally_advanced",
              "chimera_human_EPI_delayed_at_pre_implantation_only_resolved_post_implantation"
            ]
          },
          {
            "name": "E17_chronological_match_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "resembles_monkey_E17_Gast_EPI",
              "resembles_chimera_monkey_E17_EPI_specifically",
              "chimera_human_E17_EPI_more_advanced_than_monkey_E17_Gast"
            ]
          },
          {
            "name": "pluripotency_exit_paper_inverted_or_prior_driven",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_advances_pluripotency_exit_prematurely",
              "naive_pluripotency_program_specifically_engaged",
              "pluripotency_core_retained_via_KLF17_naive_marker_specifically"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-6-pi",
      "modelName": "claude-opus-4-6",
      "harness": "pi",
      "task": "gse155381_q4_pseudotime_delay",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 1,
      "passed": true,
      "score": null,
      "durationS": 1359,
      "cost": 2.85511825,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "pseudotime_direction_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "E17_chronological_match_weighted",
            "role": "additive",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pluripotency_exit_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pseudotime_direction_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_synchronous_with_monkey_EPI",
              "chimera_human_EPI_developmentally_advanced",
              "chimera_human_EPI_delayed_at_pre_implantation_only_resolved_post_implantation"
            ]
          },
          {
            "name": "E17_chronological_match_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "resembles_monkey_E17_Gast_EPI",
              "resembles_chimera_monkey_E17_EPI_specifically",
              "chimera_human_E17_EPI_more_advanced_than_monkey_E17_Gast"
            ]
          },
          {
            "name": "pluripotency_exit_paper_inverted_or_prior_driven",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_advances_pluripotency_exit_prematurely",
              "naive_pluripotency_program_specifically_engaged",
              "pluripotency_core_retained_via_KLF17_naive_marker_specifically"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-6-pi",
      "modelName": "claude-opus-4-6",
      "harness": "pi",
      "task": "gse155381_q4_pseudotime_delay",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 2,
      "passed": true,
      "score": null,
      "durationS": 1276,
      "cost": 2.7022422500000003,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "pseudotime_direction_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "E17_chronological_match_weighted",
            "role": "additive",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pluripotency_exit_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pseudotime_direction_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_synchronous_with_monkey_EPI",
              "chimera_human_EPI_developmentally_advanced",
              "chimera_human_EPI_delayed_at_pre_implantation_only_resolved_post_implantation"
            ]
          },
          {
            "name": "E17_chronological_match_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "resembles_monkey_E17_Gast_EPI",
              "resembles_chimera_monkey_E17_EPI_specifically",
              "chimera_human_E17_EPI_more_advanced_than_monkey_E17_Gast"
            ]
          },
          {
            "name": "pluripotency_exit_paper_inverted_or_prior_driven",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_advances_pluripotency_exit_prematurely",
              "naive_pluripotency_program_specifically_engaged",
              "pluripotency_core_retained_via_KLF17_naive_marker_specifically"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-6-pi",
      "modelName": "claude-opus-4-6",
      "harness": "pi",
      "task": "gse155381_q4_pseudotime_delay",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 3,
      "passed": true,
      "score": null,
      "durationS": 1909,
      "cost": 3.1195405,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "pseudotime_direction_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "E17_chronological_match_weighted",
            "role": "additive",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pluripotency_exit_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pseudotime_direction_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_synchronous_with_monkey_EPI",
              "chimera_human_EPI_developmentally_advanced",
              "chimera_human_EPI_delayed_at_pre_implantation_only_resolved_post_implantation"
            ]
          },
          {
            "name": "E17_chronological_match_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "resembles_monkey_E17_Gast_EPI",
              "resembles_chimera_monkey_E17_EPI_specifically",
              "chimera_human_E17_EPI_more_advanced_than_monkey_E17_Gast"
            ]
          },
          {
            "name": "pluripotency_exit_paper_inverted_or_prior_driven",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_advances_pluripotency_exit_prematurely",
              "naive_pluripotency_program_specifically_engaged",
              "pluripotency_core_retained_via_KLF17_naive_marker_specifically"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-7-claude-code",
      "modelName": "claude-opus-4-7",
      "harness": "claude-code",
      "task": "gse155381_q4_pseudotime_delay",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 1,
      "passed": true,
      "score": null,
      "durationS": 1442,
      "cost": 5.485195000000001,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "pseudotime_direction_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "E17_chronological_match_weighted",
            "role": "additive",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pluripotency_exit_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pseudotime_direction_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_synchronous_with_monkey_EPI",
              "chimera_human_EPI_developmentally_advanced",
              "chimera_human_EPI_delayed_at_pre_implantation_only_resolved_post_implantation"
            ]
          },
          {
            "name": "E17_chronological_match_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "resembles_monkey_E17_Gast_EPI",
              "resembles_chimera_monkey_E17_EPI_specifically",
              "chimera_human_E17_EPI_more_advanced_than_monkey_E17_Gast"
            ]
          },
          {
            "name": "pluripotency_exit_paper_inverted_or_prior_driven",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_advances_pluripotency_exit_prematurely",
              "naive_pluripotency_program_specifically_engaged",
              "pluripotency_core_retained_via_KLF17_naive_marker_specifically"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-7-claude-code",
      "modelName": "claude-opus-4-7",
      "harness": "claude-code",
      "task": "gse155381_q4_pseudotime_delay",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 2,
      "passed": true,
      "score": null,
      "durationS": 947,
      "cost": 3.4959502500000004,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "pseudotime_direction_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "E17_chronological_match_weighted",
            "role": "additive",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pluripotency_exit_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pseudotime_direction_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_synchronous_with_monkey_EPI",
              "chimera_human_EPI_developmentally_advanced",
              "chimera_human_EPI_delayed_at_pre_implantation_only_resolved_post_implantation"
            ]
          },
          {
            "name": "E17_chronological_match_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "resembles_monkey_E17_Gast_EPI",
              "resembles_chimera_monkey_E17_EPI_specifically",
              "chimera_human_E17_EPI_more_advanced_than_monkey_E17_Gast"
            ]
          },
          {
            "name": "pluripotency_exit_paper_inverted_or_prior_driven",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_advances_pluripotency_exit_prematurely",
              "naive_pluripotency_program_specifically_engaged",
              "pluripotency_core_retained_via_KLF17_naive_marker_specifically"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-7-claude-code",
      "modelName": "claude-opus-4-7",
      "harness": "claude-code",
      "task": "gse155381_q4_pseudotime_delay",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 3,
      "passed": true,
      "score": null,
      "durationS": 1641,
      "cost": 5.3087887500000015,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "pseudotime_direction_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "E17_chronological_match_weighted",
            "role": "additive",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pluripotency_exit_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pseudotime_direction_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_synchronous_with_monkey_EPI",
              "chimera_human_EPI_developmentally_advanced",
              "chimera_human_EPI_delayed_at_pre_implantation_only_resolved_post_implantation"
            ]
          },
          {
            "name": "E17_chronological_match_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "resembles_monkey_E17_Gast_EPI",
              "resembles_chimera_monkey_E17_EPI_specifically",
              "chimera_human_E17_EPI_more_advanced_than_monkey_E17_Gast"
            ]
          },
          {
            "name": "pluripotency_exit_paper_inverted_or_prior_driven",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_advances_pluripotency_exit_prematurely",
              "naive_pluripotency_program_specifically_engaged",
              "pluripotency_core_retained_via_KLF17_naive_marker_specifically"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-7-pi",
      "modelName": "claude-opus-4-7",
      "harness": "pi",
      "task": "gse155381_q4_pseudotime_delay",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 1,
      "passed": true,
      "score": null,
      "durationS": 573,
      "cost": 1.4063807499999996,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "pseudotime_direction_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "E17_chronological_match_weighted",
            "role": "additive",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pluripotency_exit_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pseudotime_direction_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_synchronous_with_monkey_EPI",
              "chimera_human_EPI_developmentally_advanced",
              "chimera_human_EPI_delayed_at_pre_implantation_only_resolved_post_implantation"
            ]
          },
          {
            "name": "E17_chronological_match_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "resembles_monkey_E17_Gast_EPI",
              "resembles_chimera_monkey_E17_EPI_specifically",
              "chimera_human_E17_EPI_more_advanced_than_monkey_E17_Gast"
            ]
          },
          {
            "name": "pluripotency_exit_paper_inverted_or_prior_driven",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_advances_pluripotency_exit_prematurely",
              "naive_pluripotency_program_specifically_engaged",
              "pluripotency_core_retained_via_KLF17_naive_marker_specifically"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-7-pi",
      "modelName": "claude-opus-4-7",
      "harness": "pi",
      "task": "gse155381_q4_pseudotime_delay",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 2,
      "passed": true,
      "score": null,
      "durationS": 1179,
      "cost": 2.28120325,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "pseudotime_direction_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "E17_chronological_match_weighted",
            "role": "additive",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pluripotency_exit_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pseudotime_direction_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_synchronous_with_monkey_EPI",
              "chimera_human_EPI_developmentally_advanced",
              "chimera_human_EPI_delayed_at_pre_implantation_only_resolved_post_implantation"
            ]
          },
          {
            "name": "E17_chronological_match_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "resembles_monkey_E17_Gast_EPI",
              "resembles_chimera_monkey_E17_EPI_specifically",
              "chimera_human_E17_EPI_more_advanced_than_monkey_E17_Gast"
            ]
          },
          {
            "name": "pluripotency_exit_paper_inverted_or_prior_driven",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_advances_pluripotency_exit_prematurely",
              "naive_pluripotency_program_specifically_engaged",
              "pluripotency_core_retained_via_KLF17_naive_marker_specifically"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-7-pi",
      "modelName": "claude-opus-4-7",
      "harness": "pi",
      "task": "gse155381_q4_pseudotime_delay",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 3,
      "passed": true,
      "score": null,
      "durationS": 910,
      "cost": 2.5057342500000006,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "pseudotime_direction_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "E17_chronological_match_weighted",
            "role": "additive",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pluripotency_exit_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pseudotime_direction_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_synchronous_with_monkey_EPI",
              "chimera_human_EPI_developmentally_advanced",
              "chimera_human_EPI_delayed_at_pre_implantation_only_resolved_post_implantation"
            ]
          },
          {
            "name": "E17_chronological_match_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "resembles_monkey_E17_Gast_EPI",
              "resembles_chimera_monkey_E17_EPI_specifically",
              "chimera_human_E17_EPI_more_advanced_than_monkey_E17_Gast"
            ]
          },
          {
            "name": "pluripotency_exit_paper_inverted_or_prior_driven",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_advances_pluripotency_exit_prematurely",
              "naive_pluripotency_program_specifically_engaged",
              "pluripotency_core_retained_via_KLF17_naive_marker_specifically"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-8-claude-code",
      "modelName": "claude-opus-4-8",
      "harness": "claude-code",
      "task": "gse155381_q4_pseudotime_delay",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 1,
      "passed": true,
      "score": null,
      "durationS": 1886,
      "cost": 5.430708499999999,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "pseudotime_direction_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "E17_chronological_match_weighted",
            "role": "additive",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pluripotency_exit_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pseudotime_direction_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_synchronous_with_monkey_EPI",
              "chimera_human_EPI_developmentally_advanced",
              "chimera_human_EPI_delayed_at_pre_implantation_only_resolved_post_implantation"
            ]
          },
          {
            "name": "E17_chronological_match_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "resembles_monkey_E17_Gast_EPI",
              "resembles_chimera_monkey_E17_EPI_specifically",
              "chimera_human_E17_EPI_more_advanced_than_monkey_E17_Gast"
            ]
          },
          {
            "name": "pluripotency_exit_paper_inverted_or_prior_driven",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_advances_pluripotency_exit_prematurely",
              "naive_pluripotency_program_specifically_engaged",
              "pluripotency_core_retained_via_KLF17_naive_marker_specifically"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-8-claude-code",
      "modelName": "claude-opus-4-8",
      "harness": "claude-code",
      "task": "gse155381_q4_pseudotime_delay",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 2,
      "passed": true,
      "score": null,
      "durationS": 1980,
      "cost": 6.108470500000001,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "pseudotime_direction_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "E17_chronological_match_weighted",
            "role": "additive",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pluripotency_exit_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pseudotime_direction_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_synchronous_with_monkey_EPI",
              "chimera_human_EPI_developmentally_advanced",
              "chimera_human_EPI_delayed_at_pre_implantation_only_resolved_post_implantation"
            ]
          },
          {
            "name": "E17_chronological_match_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "resembles_monkey_E17_Gast_EPI",
              "resembles_chimera_monkey_E17_EPI_specifically",
              "chimera_human_E17_EPI_more_advanced_than_monkey_E17_Gast"
            ]
          },
          {
            "name": "pluripotency_exit_paper_inverted_or_prior_driven",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_advances_pluripotency_exit_prematurely",
              "naive_pluripotency_program_specifically_engaged",
              "pluripotency_core_retained_via_KLF17_naive_marker_specifically"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-8-claude-code",
      "modelName": "claude-opus-4-8",
      "harness": "claude-code",
      "task": "gse155381_q4_pseudotime_delay",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 3,
      "passed": true,
      "score": null,
      "durationS": 2240,
      "cost": 7.293773500000001,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "pseudotime_direction_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "E17_chronological_match_weighted",
            "role": "additive",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pluripotency_exit_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pseudotime_direction_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_synchronous_with_monkey_EPI",
              "chimera_human_EPI_developmentally_advanced",
              "chimera_human_EPI_delayed_at_pre_implantation_only_resolved_post_implantation"
            ]
          },
          {
            "name": "E17_chronological_match_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "resembles_monkey_E17_Gast_EPI",
              "resembles_chimera_monkey_E17_EPI_specifically",
              "chimera_human_E17_EPI_more_advanced_than_monkey_E17_Gast"
            ]
          },
          {
            "name": "pluripotency_exit_paper_inverted_or_prior_driven",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_advances_pluripotency_exit_prematurely",
              "naive_pluripotency_program_specifically_engaged",
              "pluripotency_core_retained_via_KLF17_naive_marker_specifically"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-8-pi",
      "modelName": "claude-opus-4-8",
      "harness": "pi",
      "task": "gse155381_q4_pseudotime_delay",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 1,
      "passed": true,
      "score": null,
      "durationS": 1008,
      "cost": 2.4191320000000003,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "pseudotime_direction_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "E17_chronological_match_weighted",
            "role": "additive",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pluripotency_exit_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pseudotime_direction_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_synchronous_with_monkey_EPI",
              "chimera_human_EPI_developmentally_advanced",
              "chimera_human_EPI_delayed_at_pre_implantation_only_resolved_post_implantation"
            ]
          },
          {
            "name": "E17_chronological_match_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "resembles_monkey_E17_Gast_EPI",
              "resembles_chimera_monkey_E17_EPI_specifically",
              "chimera_human_E17_EPI_more_advanced_than_monkey_E17_Gast"
            ]
          },
          {
            "name": "pluripotency_exit_paper_inverted_or_prior_driven",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_advances_pluripotency_exit_prematurely",
              "naive_pluripotency_program_specifically_engaged",
              "pluripotency_core_retained_via_KLF17_naive_marker_specifically"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-8-pi",
      "modelName": "claude-opus-4-8",
      "harness": "pi",
      "task": "gse155381_q4_pseudotime_delay",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 2,
      "passed": true,
      "score": null,
      "durationS": 1137,
      "cost": 2.9610880000000006,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "pseudotime_direction_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "E17_chronological_match_weighted",
            "role": "additive",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pluripotency_exit_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pseudotime_direction_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_synchronous_with_monkey_EPI",
              "chimera_human_EPI_developmentally_advanced",
              "chimera_human_EPI_delayed_at_pre_implantation_only_resolved_post_implantation"
            ]
          },
          {
            "name": "E17_chronological_match_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "resembles_monkey_E17_Gast_EPI",
              "resembles_chimera_monkey_E17_EPI_specifically",
              "chimera_human_E17_EPI_more_advanced_than_monkey_E17_Gast"
            ]
          },
          {
            "name": "pluripotency_exit_paper_inverted_or_prior_driven",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_advances_pluripotency_exit_prematurely",
              "naive_pluripotency_program_specifically_engaged",
              "pluripotency_core_retained_via_KLF17_naive_marker_specifically"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-8-pi",
      "modelName": "claude-opus-4-8",
      "harness": "pi",
      "task": "gse155381_q4_pseudotime_delay",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 3,
      "passed": true,
      "score": null,
      "durationS": 1598,
      "cost": 3.189386,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "pseudotime_direction_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "E17_chronological_match_weighted",
            "role": "additive",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pluripotency_exit_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pseudotime_direction_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_synchronous_with_monkey_EPI",
              "chimera_human_EPI_developmentally_advanced",
              "chimera_human_EPI_delayed_at_pre_implantation_only_resolved_post_implantation"
            ]
          },
          {
            "name": "E17_chronological_match_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "resembles_monkey_E17_Gast_EPI",
              "resembles_chimera_monkey_E17_EPI_specifically",
              "chimera_human_E17_EPI_more_advanced_than_monkey_E17_Gast"
            ]
          },
          {
            "name": "pluripotency_exit_paper_inverted_or_prior_driven",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_advances_pluripotency_exit_prematurely",
              "naive_pluripotency_program_specifically_engaged",
              "pluripotency_core_retained_via_KLF17_naive_marker_specifically"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-4-6-pi",
      "modelName": "claude-sonnet-4-6",
      "harness": "pi",
      "task": "gse155381_q4_pseudotime_delay",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 1031,
      "cost": 1.3784331,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "pseudotime_direction_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "E17_chronological_match_weighted",
            "role": "additive",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pluripotency_exit_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pseudotime_direction_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_synchronous_with_monkey_EPI",
              "chimera_human_EPI_developmentally_advanced",
              "chimera_human_EPI_delayed_at_pre_implantation_only_resolved_post_implantation"
            ]
          },
          {
            "name": "E17_chronological_match_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "resembles_monkey_E17_Gast_EPI",
              "resembles_chimera_monkey_E17_EPI_specifically",
              "chimera_human_E17_EPI_more_advanced_than_monkey_E17_Gast"
            ]
          },
          {
            "name": "pluripotency_exit_paper_inverted_or_prior_driven",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_advances_pluripotency_exit_prematurely",
              "naive_pluripotency_program_specifically_engaged",
              "pluripotency_core_retained_via_KLF17_naive_marker_specifically"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-4-6-pi",
      "modelName": "claude-sonnet-4-6",
      "harness": "pi",
      "task": "gse155381_q4_pseudotime_delay",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 2,
      "passed": true,
      "score": null,
      "durationS": 1092,
      "cost": 1.3346277,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "pseudotime_direction_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "E17_chronological_match_weighted",
            "role": "additive",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pluripotency_exit_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pseudotime_direction_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_synchronous_with_monkey_EPI",
              "chimera_human_EPI_developmentally_advanced",
              "chimera_human_EPI_delayed_at_pre_implantation_only_resolved_post_implantation"
            ]
          },
          {
            "name": "E17_chronological_match_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "resembles_monkey_E17_Gast_EPI",
              "resembles_chimera_monkey_E17_EPI_specifically",
              "chimera_human_E17_EPI_more_advanced_than_monkey_E17_Gast"
            ]
          },
          {
            "name": "pluripotency_exit_paper_inverted_or_prior_driven",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_advances_pluripotency_exit_prematurely",
              "naive_pluripotency_program_specifically_engaged",
              "pluripotency_core_retained_via_KLF17_naive_marker_specifically"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-4-6-pi",
      "modelName": "claude-sonnet-4-6",
      "harness": "pi",
      "task": "gse155381_q4_pseudotime_delay",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1575,
      "cost": 2.1084216,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "pseudotime_direction_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "E17_chronological_match_weighted",
            "role": "additive",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pluripotency_exit_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pseudotime_direction_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_synchronous_with_monkey_EPI",
              "chimera_human_EPI_developmentally_advanced",
              "chimera_human_EPI_delayed_at_pre_implantation_only_resolved_post_implantation"
            ]
          },
          {
            "name": "E17_chronological_match_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "resembles_monkey_E17_Gast_EPI",
              "resembles_chimera_monkey_E17_EPI_specifically",
              "chimera_human_E17_EPI_more_advanced_than_monkey_E17_Gast"
            ]
          },
          {
            "name": "pluripotency_exit_paper_inverted_or_prior_driven",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_advances_pluripotency_exit_prematurely",
              "naive_pluripotency_program_specifically_engaged",
              "pluripotency_core_retained_via_KLF17_naive_marker_specifically"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gemini-3-1-pro-preview-pi",
      "modelName": "gemini-3.1-pro-preview",
      "harness": "pi",
      "task": "gse155381_q4_pseudotime_delay",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 1,
      "passed": true,
      "score": null,
      "durationS": 485,
      "cost": 0.8113128000000001,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "pseudotime_direction_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "E17_chronological_match_weighted",
            "role": "additive",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pluripotency_exit_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pseudotime_direction_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_synchronous_with_monkey_EPI",
              "chimera_human_EPI_developmentally_advanced",
              "chimera_human_EPI_delayed_at_pre_implantation_only_resolved_post_implantation"
            ]
          },
          {
            "name": "E17_chronological_match_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "resembles_monkey_E17_Gast_EPI",
              "resembles_chimera_monkey_E17_EPI_specifically",
              "chimera_human_E17_EPI_more_advanced_than_monkey_E17_Gast"
            ]
          },
          {
            "name": "pluripotency_exit_paper_inverted_or_prior_driven",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_advances_pluripotency_exit_prematurely",
              "naive_pluripotency_program_specifically_engaged",
              "pluripotency_core_retained_via_KLF17_naive_marker_specifically"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gemini-3-1-pro-preview-pi",
      "modelName": "gemini-3.1-pro-preview",
      "harness": "pi",
      "task": "gse155381_q4_pseudotime_delay",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 1257,
      "cost": 1.0325818,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "pseudotime_direction_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "E17_chronological_match_weighted",
            "role": "additive",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pluripotency_exit_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pseudotime_direction_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_synchronous_with_monkey_EPI",
              "chimera_human_EPI_developmentally_advanced",
              "chimera_human_EPI_delayed_at_pre_implantation_only_resolved_post_implantation"
            ]
          },
          {
            "name": "E17_chronological_match_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "resembles_monkey_E17_Gast_EPI",
              "resembles_chimera_monkey_E17_EPI_specifically",
              "chimera_human_E17_EPI_more_advanced_than_monkey_E17_Gast"
            ]
          },
          {
            "name": "pluripotency_exit_paper_inverted_or_prior_driven",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_advances_pluripotency_exit_prematurely",
              "naive_pluripotency_program_specifically_engaged",
              "pluripotency_core_retained_via_KLF17_naive_marker_specifically"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gemini-3-1-pro-preview-pi",
      "modelName": "gemini-3.1-pro-preview",
      "harness": "pi",
      "task": "gse155381_q4_pseudotime_delay",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 3,
      "passed": true,
      "score": null,
      "durationS": 432,
      "cost": 1.0884683999999998,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "pseudotime_direction_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "E17_chronological_match_weighted",
            "role": "additive",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pluripotency_exit_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pseudotime_direction_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_synchronous_with_monkey_EPI",
              "chimera_human_EPI_developmentally_advanced",
              "chimera_human_EPI_delayed_at_pre_implantation_only_resolved_post_implantation"
            ]
          },
          {
            "name": "E17_chronological_match_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "resembles_monkey_E17_Gast_EPI",
              "resembles_chimera_monkey_E17_EPI_specifically",
              "chimera_human_E17_EPI_more_advanced_than_monkey_E17_Gast"
            ]
          },
          {
            "name": "pluripotency_exit_paper_inverted_or_prior_driven",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_advances_pluripotency_exit_prematurely",
              "naive_pluripotency_program_specifically_engaged",
              "pluripotency_core_retained_via_KLF17_naive_marker_specifically"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gemini-3-5-flash-pi",
      "modelName": "gemini-3.5-flash",
      "harness": "pi",
      "task": "gse155381_q4_pseudotime_delay",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 1033,
      "cost": 3.7781314500000014,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "pseudotime_direction_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "E17_chronological_match_weighted",
            "role": "additive",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pluripotency_exit_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pseudotime_direction_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_synchronous_with_monkey_EPI",
              "chimera_human_EPI_developmentally_advanced",
              "chimera_human_EPI_delayed_at_pre_implantation_only_resolved_post_implantation"
            ]
          },
          {
            "name": "E17_chronological_match_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "resembles_monkey_E17_Gast_EPI",
              "resembles_chimera_monkey_E17_EPI_specifically",
              "chimera_human_E17_EPI_more_advanced_than_monkey_E17_Gast"
            ]
          },
          {
            "name": "pluripotency_exit_paper_inverted_or_prior_driven",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_advances_pluripotency_exit_prematurely",
              "naive_pluripotency_program_specifically_engaged",
              "pluripotency_core_retained_via_KLF17_naive_marker_specifically"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gemini-3-5-flash-pi",
      "modelName": "gemini-3.5-flash",
      "harness": "pi",
      "task": "gse155381_q4_pseudotime_delay",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 2,
      "passed": true,
      "score": null,
      "durationS": 1057,
      "cost": 1.6443913499999998,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "pseudotime_direction_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "E17_chronological_match_weighted",
            "role": "additive",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pluripotency_exit_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pseudotime_direction_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_synchronous_with_monkey_EPI",
              "chimera_human_EPI_developmentally_advanced",
              "chimera_human_EPI_delayed_at_pre_implantation_only_resolved_post_implantation"
            ]
          },
          {
            "name": "E17_chronological_match_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "resembles_monkey_E17_Gast_EPI",
              "resembles_chimera_monkey_E17_EPI_specifically",
              "chimera_human_E17_EPI_more_advanced_than_monkey_E17_Gast"
            ]
          },
          {
            "name": "pluripotency_exit_paper_inverted_or_prior_driven",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_advances_pluripotency_exit_prematurely",
              "naive_pluripotency_program_specifically_engaged",
              "pluripotency_core_retained_via_KLF17_naive_marker_specifically"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gemini-3-5-flash-pi",
      "modelName": "gemini-3.5-flash",
      "harness": "pi",
      "task": "gse155381_q4_pseudotime_delay",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1087,
      "cost": 5.247100350000003,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "pseudotime_direction_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "E17_chronological_match_weighted",
            "role": "additive",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pluripotency_exit_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pseudotime_direction_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_synchronous_with_monkey_EPI",
              "chimera_human_EPI_developmentally_advanced",
              "chimera_human_EPI_delayed_at_pre_implantation_only_resolved_post_implantation"
            ]
          },
          {
            "name": "E17_chronological_match_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "resembles_monkey_E17_Gast_EPI",
              "resembles_chimera_monkey_E17_EPI_specifically",
              "chimera_human_E17_EPI_more_advanced_than_monkey_E17_Gast"
            ]
          },
          {
            "name": "pluripotency_exit_paper_inverted_or_prior_driven",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_advances_pluripotency_exit_prematurely",
              "naive_pluripotency_program_specifically_engaged",
              "pluripotency_core_retained_via_KLF17_naive_marker_specifically"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-glm-5p2-pi",
      "modelName": "glm-5p2",
      "harness": "pi",
      "task": "gse155381_q4_pseudotime_delay",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 2580,
      "cost": 3.4160433,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "pseudotime_direction_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "E17_chronological_match_weighted",
            "role": "additive",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pluripotency_exit_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pseudotime_direction_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_synchronous_with_monkey_EPI",
              "chimera_human_EPI_developmentally_advanced",
              "chimera_human_EPI_delayed_at_pre_implantation_only_resolved_post_implantation"
            ]
          },
          {
            "name": "E17_chronological_match_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "resembles_monkey_E17_Gast_EPI",
              "resembles_chimera_monkey_E17_EPI_specifically",
              "chimera_human_E17_EPI_more_advanced_than_monkey_E17_Gast"
            ]
          },
          {
            "name": "pluripotency_exit_paper_inverted_or_prior_driven",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_advances_pluripotency_exit_prematurely",
              "naive_pluripotency_program_specifically_engaged",
              "pluripotency_core_retained_via_KLF17_naive_marker_specifically"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-glm-5p2-pi",
      "modelName": "glm-5p2",
      "harness": "pi",
      "task": "gse155381_q4_pseudotime_delay",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 2911,
      "cost": 2.52375602,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "pseudotime_direction_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "E17_chronological_match_weighted",
            "role": "additive",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pluripotency_exit_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pseudotime_direction_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_synchronous_with_monkey_EPI",
              "chimera_human_EPI_developmentally_advanced",
              "chimera_human_EPI_delayed_at_pre_implantation_only_resolved_post_implantation"
            ]
          },
          {
            "name": "E17_chronological_match_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "resembles_monkey_E17_Gast_EPI",
              "resembles_chimera_monkey_E17_EPI_specifically",
              "chimera_human_E17_EPI_more_advanced_than_monkey_E17_Gast"
            ]
          },
          {
            "name": "pluripotency_exit_paper_inverted_or_prior_driven",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_advances_pluripotency_exit_prematurely",
              "naive_pluripotency_program_specifically_engaged",
              "pluripotency_core_retained_via_KLF17_naive_marker_specifically"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-glm-5p2-pi",
      "modelName": "glm-5p2",
      "harness": "pi",
      "task": "gse155381_q4_pseudotime_delay",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 2255,
      "cost": 2.16688324,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "pseudotime_direction_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "E17_chronological_match_weighted",
            "role": "additive",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pluripotency_exit_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pseudotime_direction_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_synchronous_with_monkey_EPI",
              "chimera_human_EPI_developmentally_advanced",
              "chimera_human_EPI_delayed_at_pre_implantation_only_resolved_post_implantation"
            ]
          },
          {
            "name": "E17_chronological_match_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "resembles_monkey_E17_Gast_EPI",
              "resembles_chimera_monkey_E17_EPI_specifically",
              "chimera_human_E17_EPI_more_advanced_than_monkey_E17_Gast"
            ]
          },
          {
            "name": "pluripotency_exit_paper_inverted_or_prior_driven",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_advances_pluripotency_exit_prematurely",
              "naive_pluripotency_program_specifically_engaged",
              "pluripotency_core_retained_via_KLF17_naive_marker_specifically"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-4-codex",
      "modelName": "gpt-5.4",
      "harness": "openai-codex",
      "task": "gse155381_q4_pseudotime_delay",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 1256,
      "cost": 2.019304,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "pseudotime_direction_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "E17_chronological_match_weighted",
            "role": "additive",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pluripotency_exit_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pseudotime_direction_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_synchronous_with_monkey_EPI",
              "chimera_human_EPI_developmentally_advanced",
              "chimera_human_EPI_delayed_at_pre_implantation_only_resolved_post_implantation"
            ]
          },
          {
            "name": "E17_chronological_match_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "resembles_monkey_E17_Gast_EPI",
              "resembles_chimera_monkey_E17_EPI_specifically",
              "chimera_human_E17_EPI_more_advanced_than_monkey_E17_Gast"
            ]
          },
          {
            "name": "pluripotency_exit_paper_inverted_or_prior_driven",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_advances_pluripotency_exit_prematurely",
              "naive_pluripotency_program_specifically_engaged",
              "pluripotency_core_retained_via_KLF17_naive_marker_specifically"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-4-codex",
      "modelName": "gpt-5.4",
      "harness": "openai-codex",
      "task": "gse155381_q4_pseudotime_delay",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 1153,
      "cost": 2.7916429999999997,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "pseudotime_direction_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "E17_chronological_match_weighted",
            "role": "additive",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pluripotency_exit_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pseudotime_direction_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_synchronous_with_monkey_EPI",
              "chimera_human_EPI_developmentally_advanced",
              "chimera_human_EPI_delayed_at_pre_implantation_only_resolved_post_implantation"
            ]
          },
          {
            "name": "E17_chronological_match_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "resembles_monkey_E17_Gast_EPI",
              "resembles_chimera_monkey_E17_EPI_specifically",
              "chimera_human_E17_EPI_more_advanced_than_monkey_E17_Gast"
            ]
          },
          {
            "name": "pluripotency_exit_paper_inverted_or_prior_driven",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_advances_pluripotency_exit_prematurely",
              "naive_pluripotency_program_specifically_engaged",
              "pluripotency_core_retained_via_KLF17_naive_marker_specifically"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-4-codex",
      "modelName": "gpt-5.4",
      "harness": "openai-codex",
      "task": "gse155381_q4_pseudotime_delay",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 793,
      "cost": 1.6706775,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "pseudotime_direction_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "E17_chronological_match_weighted",
            "role": "additive",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pluripotency_exit_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pseudotime_direction_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_synchronous_with_monkey_EPI",
              "chimera_human_EPI_developmentally_advanced",
              "chimera_human_EPI_delayed_at_pre_implantation_only_resolved_post_implantation"
            ]
          },
          {
            "name": "E17_chronological_match_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "resembles_monkey_E17_Gast_EPI",
              "resembles_chimera_monkey_E17_EPI_specifically",
              "chimera_human_E17_EPI_more_advanced_than_monkey_E17_Gast"
            ]
          },
          {
            "name": "pluripotency_exit_paper_inverted_or_prior_driven",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_advances_pluripotency_exit_prematurely",
              "naive_pluripotency_program_specifically_engaged",
              "pluripotency_core_retained_via_KLF17_naive_marker_specifically"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-4-pi",
      "modelName": "gpt-5.4",
      "harness": "pi",
      "task": "gse155381_q4_pseudotime_delay",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 1562,
      "cost": 2.844252000000001,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "pseudotime_direction_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "E17_chronological_match_weighted",
            "role": "additive",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pluripotency_exit_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pseudotime_direction_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_synchronous_with_monkey_EPI",
              "chimera_human_EPI_developmentally_advanced",
              "chimera_human_EPI_delayed_at_pre_implantation_only_resolved_post_implantation"
            ]
          },
          {
            "name": "E17_chronological_match_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "resembles_monkey_E17_Gast_EPI",
              "resembles_chimera_monkey_E17_EPI_specifically",
              "chimera_human_E17_EPI_more_advanced_than_monkey_E17_Gast"
            ]
          },
          {
            "name": "pluripotency_exit_paper_inverted_or_prior_driven",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_advances_pluripotency_exit_prematurely",
              "naive_pluripotency_program_specifically_engaged",
              "pluripotency_core_retained_via_KLF17_naive_marker_specifically"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-4-pi",
      "modelName": "gpt-5.4",
      "harness": "pi",
      "task": "gse155381_q4_pseudotime_delay",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 1695,
      "cost": 3.056128,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "pseudotime_direction_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "E17_chronological_match_weighted",
            "role": "additive",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pluripotency_exit_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pseudotime_direction_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_synchronous_with_monkey_EPI",
              "chimera_human_EPI_developmentally_advanced",
              "chimera_human_EPI_delayed_at_pre_implantation_only_resolved_post_implantation"
            ]
          },
          {
            "name": "E17_chronological_match_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "resembles_monkey_E17_Gast_EPI",
              "resembles_chimera_monkey_E17_EPI_specifically",
              "chimera_human_E17_EPI_more_advanced_than_monkey_E17_Gast"
            ]
          },
          {
            "name": "pluripotency_exit_paper_inverted_or_prior_driven",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_advances_pluripotency_exit_prematurely",
              "naive_pluripotency_program_specifically_engaged",
              "pluripotency_core_retained_via_KLF17_naive_marker_specifically"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-4-pi",
      "modelName": "gpt-5.4",
      "harness": "pi",
      "task": "gse155381_q4_pseudotime_delay",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1041,
      "cost": 1.5634040000000002,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "pseudotime_direction_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "E17_chronological_match_weighted",
            "role": "additive",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pluripotency_exit_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pseudotime_direction_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_synchronous_with_monkey_EPI",
              "chimera_human_EPI_developmentally_advanced",
              "chimera_human_EPI_delayed_at_pre_implantation_only_resolved_post_implantation"
            ]
          },
          {
            "name": "E17_chronological_match_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "resembles_monkey_E17_Gast_EPI",
              "resembles_chimera_monkey_E17_EPI_specifically",
              "chimera_human_E17_EPI_more_advanced_than_monkey_E17_Gast"
            ]
          },
          {
            "name": "pluripotency_exit_paper_inverted_or_prior_driven",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_advances_pluripotency_exit_prematurely",
              "naive_pluripotency_program_specifically_engaged",
              "pluripotency_core_retained_via_KLF17_naive_marker_specifically"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-5-codex",
      "modelName": "gpt-5.5",
      "harness": "openai-codex",
      "task": "gse155381_q4_pseudotime_delay",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 782,
      "cost": 2.7704250000000004,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "pseudotime_direction_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "E17_chronological_match_weighted",
            "role": "additive",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pluripotency_exit_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pseudotime_direction_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_synchronous_with_monkey_EPI",
              "chimera_human_EPI_developmentally_advanced",
              "chimera_human_EPI_delayed_at_pre_implantation_only_resolved_post_implantation"
            ]
          },
          {
            "name": "E17_chronological_match_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "resembles_monkey_E17_Gast_EPI",
              "resembles_chimera_monkey_E17_EPI_specifically",
              "chimera_human_E17_EPI_more_advanced_than_monkey_E17_Gast"
            ]
          },
          {
            "name": "pluripotency_exit_paper_inverted_or_prior_driven",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_advances_pluripotency_exit_prematurely",
              "naive_pluripotency_program_specifically_engaged",
              "pluripotency_core_retained_via_KLF17_naive_marker_specifically"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-5-codex",
      "modelName": "gpt-5.5",
      "harness": "openai-codex",
      "task": "gse155381_q4_pseudotime_delay",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 505,
      "cost": 2.456512,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "pseudotime_direction_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "E17_chronological_match_weighted",
            "role": "additive",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pluripotency_exit_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pseudotime_direction_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_synchronous_with_monkey_EPI",
              "chimera_human_EPI_developmentally_advanced",
              "chimera_human_EPI_delayed_at_pre_implantation_only_resolved_post_implantation"
            ]
          },
          {
            "name": "E17_chronological_match_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "resembles_monkey_E17_Gast_EPI",
              "resembles_chimera_monkey_E17_EPI_specifically",
              "chimera_human_E17_EPI_more_advanced_than_monkey_E17_Gast"
            ]
          },
          {
            "name": "pluripotency_exit_paper_inverted_or_prior_driven",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_advances_pluripotency_exit_prematurely",
              "naive_pluripotency_program_specifically_engaged",
              "pluripotency_core_retained_via_KLF17_naive_marker_specifically"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-5-codex",
      "modelName": "gpt-5.5",
      "harness": "openai-codex",
      "task": "gse155381_q4_pseudotime_delay",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 3,
      "passed": true,
      "score": null,
      "durationS": 824,
      "cost": 3.156002,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "pseudotime_direction_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "E17_chronological_match_weighted",
            "role": "additive",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pluripotency_exit_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pseudotime_direction_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_synchronous_with_monkey_EPI",
              "chimera_human_EPI_developmentally_advanced",
              "chimera_human_EPI_delayed_at_pre_implantation_only_resolved_post_implantation"
            ]
          },
          {
            "name": "E17_chronological_match_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "resembles_monkey_E17_Gast_EPI",
              "resembles_chimera_monkey_E17_EPI_specifically",
              "chimera_human_E17_EPI_more_advanced_than_monkey_E17_Gast"
            ]
          },
          {
            "name": "pluripotency_exit_paper_inverted_or_prior_driven",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_advances_pluripotency_exit_prematurely",
              "naive_pluripotency_program_specifically_engaged",
              "pluripotency_core_retained_via_KLF17_naive_marker_specifically"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-5-pi",
      "modelName": "gpt-5.5",
      "harness": "pi",
      "task": "gse155381_q4_pseudotime_delay",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 2418,
      "cost": 4.594713999999999,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "pseudotime_direction_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "E17_chronological_match_weighted",
            "role": "additive",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pluripotency_exit_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pseudotime_direction_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_synchronous_with_monkey_EPI",
              "chimera_human_EPI_developmentally_advanced",
              "chimera_human_EPI_delayed_at_pre_implantation_only_resolved_post_implantation"
            ]
          },
          {
            "name": "E17_chronological_match_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "resembles_monkey_E17_Gast_EPI",
              "resembles_chimera_monkey_E17_EPI_specifically",
              "chimera_human_E17_EPI_more_advanced_than_monkey_E17_Gast"
            ]
          },
          {
            "name": "pluripotency_exit_paper_inverted_or_prior_driven",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_advances_pluripotency_exit_prematurely",
              "naive_pluripotency_program_specifically_engaged",
              "pluripotency_core_retained_via_KLF17_naive_marker_specifically"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-5-pi",
      "modelName": "gpt-5.5",
      "harness": "pi",
      "task": "gse155381_q4_pseudotime_delay",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 2,
      "passed": true,
      "score": null,
      "durationS": 1487,
      "cost": 3.9494420000000003,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "pseudotime_direction_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "E17_chronological_match_weighted",
            "role": "additive",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pluripotency_exit_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pseudotime_direction_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_synchronous_with_monkey_EPI",
              "chimera_human_EPI_developmentally_advanced",
              "chimera_human_EPI_delayed_at_pre_implantation_only_resolved_post_implantation"
            ]
          },
          {
            "name": "E17_chronological_match_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "resembles_monkey_E17_Gast_EPI",
              "resembles_chimera_monkey_E17_EPI_specifically",
              "chimera_human_E17_EPI_more_advanced_than_monkey_E17_Gast"
            ]
          },
          {
            "name": "pluripotency_exit_paper_inverted_or_prior_driven",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_advances_pluripotency_exit_prematurely",
              "naive_pluripotency_program_specifically_engaged",
              "pluripotency_core_retained_via_KLF17_naive_marker_specifically"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-5-pi",
      "modelName": "gpt-5.5",
      "harness": "pi",
      "task": "gse155381_q4_pseudotime_delay",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 3,
      "passed": true,
      "score": null,
      "durationS": 1123,
      "cost": 2.9847009999999994,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "pseudotime_direction_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "E17_chronological_match_weighted",
            "role": "additive",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pluripotency_exit_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pseudotime_direction_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_synchronous_with_monkey_EPI",
              "chimera_human_EPI_developmentally_advanced",
              "chimera_human_EPI_delayed_at_pre_implantation_only_resolved_post_implantation"
            ]
          },
          {
            "name": "E17_chronological_match_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "resembles_monkey_E17_Gast_EPI",
              "resembles_chimera_monkey_E17_EPI_specifically",
              "chimera_human_E17_EPI_more_advanced_than_monkey_E17_Gast"
            ]
          },
          {
            "name": "pluripotency_exit_paper_inverted_or_prior_driven",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_advances_pluripotency_exit_prematurely",
              "naive_pluripotency_program_specifically_engaged",
              "pluripotency_core_retained_via_KLF17_naive_marker_specifically"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-20-beta-reasoning-pi",
      "modelName": "grok-4.20-beta-0309-reasoning",
      "harness": "pi",
      "task": "gse155381_q4_pseudotime_delay",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 47,
      "cost": 0.020998400000000004,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "pseudotime_direction_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "E17_chronological_match_weighted",
            "role": "additive",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pluripotency_exit_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pseudotime_direction_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_synchronous_with_monkey_EPI",
              "chimera_human_EPI_developmentally_advanced",
              "chimera_human_EPI_delayed_at_pre_implantation_only_resolved_post_implantation"
            ]
          },
          {
            "name": "E17_chronological_match_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "resembles_monkey_E17_Gast_EPI",
              "resembles_chimera_monkey_E17_EPI_specifically",
              "chimera_human_E17_EPI_more_advanced_than_monkey_E17_Gast"
            ]
          },
          {
            "name": "pluripotency_exit_paper_inverted_or_prior_driven",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_advances_pluripotency_exit_prematurely",
              "naive_pluripotency_program_specifically_engaged",
              "pluripotency_core_retained_via_KLF17_naive_marker_specifically"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-20-beta-reasoning-pi",
      "modelName": "grok-4.20-beta-0309-reasoning",
      "harness": "pi",
      "task": "gse155381_q4_pseudotime_delay",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 2,
      "passed": true,
      "score": null,
      "durationS": 293,
      "cost": 0.1056597,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "pseudotime_direction_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "E17_chronological_match_weighted",
            "role": "additive",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pluripotency_exit_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pseudotime_direction_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_synchronous_with_monkey_EPI",
              "chimera_human_EPI_developmentally_advanced",
              "chimera_human_EPI_delayed_at_pre_implantation_only_resolved_post_implantation"
            ]
          },
          {
            "name": "E17_chronological_match_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "resembles_monkey_E17_Gast_EPI",
              "resembles_chimera_monkey_E17_EPI_specifically",
              "chimera_human_E17_EPI_more_advanced_than_monkey_E17_Gast"
            ]
          },
          {
            "name": "pluripotency_exit_paper_inverted_or_prior_driven",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_advances_pluripotency_exit_prematurely",
              "naive_pluripotency_program_specifically_engaged",
              "pluripotency_core_retained_via_KLF17_naive_marker_specifically"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-20-beta-reasoning-pi",
      "modelName": "grok-4.20-beta-0309-reasoning",
      "harness": "pi",
      "task": "gse155381_q4_pseudotime_delay",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 143,
      "cost": 0.05460965,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "pseudotime_direction_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "E17_chronological_match_weighted",
            "role": "additive",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pluripotency_exit_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pseudotime_direction_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_synchronous_with_monkey_EPI",
              "chimera_human_EPI_developmentally_advanced",
              "chimera_human_EPI_delayed_at_pre_implantation_only_resolved_post_implantation"
            ]
          },
          {
            "name": "E17_chronological_match_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "resembles_monkey_E17_Gast_EPI",
              "resembles_chimera_monkey_E17_EPI_specifically",
              "chimera_human_E17_EPI_more_advanced_than_monkey_E17_Gast"
            ]
          },
          {
            "name": "pluripotency_exit_paper_inverted_or_prior_driven",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_advances_pluripotency_exit_prematurely",
              "naive_pluripotency_program_specifically_engaged",
              "pluripotency_core_retained_via_KLF17_naive_marker_specifically"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-3-pi",
      "modelName": "grok-4.3",
      "harness": "pi",
      "task": "gse155381_q4_pseudotime_delay",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 44,
      "cost": 0.009022100000000002,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "pseudotime_direction_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "E17_chronological_match_weighted",
            "role": "additive",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pluripotency_exit_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pseudotime_direction_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_synchronous_with_monkey_EPI",
              "chimera_human_EPI_developmentally_advanced",
              "chimera_human_EPI_delayed_at_pre_implantation_only_resolved_post_implantation"
            ]
          },
          {
            "name": "E17_chronological_match_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "resembles_monkey_E17_Gast_EPI",
              "resembles_chimera_monkey_E17_EPI_specifically",
              "chimera_human_E17_EPI_more_advanced_than_monkey_E17_Gast"
            ]
          },
          {
            "name": "pluripotency_exit_paper_inverted_or_prior_driven",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_advances_pluripotency_exit_prematurely",
              "naive_pluripotency_program_specifically_engaged",
              "pluripotency_core_retained_via_KLF17_naive_marker_specifically"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-3-pi",
      "modelName": "grok-4.3",
      "harness": "pi",
      "task": "gse155381_q4_pseudotime_delay",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 59,
      "cost": 0.0111912,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "pseudotime_direction_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "E17_chronological_match_weighted",
            "role": "additive",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pluripotency_exit_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pseudotime_direction_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_synchronous_with_monkey_EPI",
              "chimera_human_EPI_developmentally_advanced",
              "chimera_human_EPI_delayed_at_pre_implantation_only_resolved_post_implantation"
            ]
          },
          {
            "name": "E17_chronological_match_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "resembles_monkey_E17_Gast_EPI",
              "resembles_chimera_monkey_E17_EPI_specifically",
              "chimera_human_E17_EPI_more_advanced_than_monkey_E17_Gast"
            ]
          },
          {
            "name": "pluripotency_exit_paper_inverted_or_prior_driven",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_advances_pluripotency_exit_prematurely",
              "naive_pluripotency_program_specifically_engaged",
              "pluripotency_core_retained_via_KLF17_naive_marker_specifically"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-3-pi",
      "modelName": "grok-4.3",
      "harness": "pi",
      "task": "gse155381_q4_pseudotime_delay",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 54,
      "cost": 0.009868200000000002,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "pseudotime_direction_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "E17_chronological_match_weighted",
            "role": "additive",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pluripotency_exit_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pseudotime_direction_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_synchronous_with_monkey_EPI",
              "chimera_human_EPI_developmentally_advanced",
              "chimera_human_EPI_delayed_at_pre_implantation_only_resolved_post_implantation"
            ]
          },
          {
            "name": "E17_chronological_match_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "resembles_monkey_E17_Gast_EPI",
              "resembles_chimera_monkey_E17_EPI_specifically",
              "chimera_human_E17_EPI_more_advanced_than_monkey_E17_Gast"
            ]
          },
          {
            "name": "pluripotency_exit_paper_inverted_or_prior_driven",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_advances_pluripotency_exit_prematurely",
              "naive_pluripotency_program_specifically_engaged",
              "pluripotency_core_retained_via_KLF17_naive_marker_specifically"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-kimi-k2p6-pi",
      "modelName": "kimi-k2p6",
      "harness": "pi",
      "task": "gse155381_q4_pseudotime_delay",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 1,
      "passed": true,
      "score": null,
      "durationS": 1525,
      "cost": 0.7684748499999998,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "pseudotime_direction_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "E17_chronological_match_weighted",
            "role": "additive",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pluripotency_exit_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pseudotime_direction_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_synchronous_with_monkey_EPI",
              "chimera_human_EPI_developmentally_advanced",
              "chimera_human_EPI_delayed_at_pre_implantation_only_resolved_post_implantation"
            ]
          },
          {
            "name": "E17_chronological_match_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "resembles_monkey_E17_Gast_EPI",
              "resembles_chimera_monkey_E17_EPI_specifically",
              "chimera_human_E17_EPI_more_advanced_than_monkey_E17_Gast"
            ]
          },
          {
            "name": "pluripotency_exit_paper_inverted_or_prior_driven",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_advances_pluripotency_exit_prematurely",
              "naive_pluripotency_program_specifically_engaged",
              "pluripotency_core_retained_via_KLF17_naive_marker_specifically"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-kimi-k2p6-pi",
      "modelName": "kimi-k2p6",
      "harness": "pi",
      "task": "gse155381_q4_pseudotime_delay",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 3295,
      "cost": 1.19356501,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "pseudotime_direction_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "E17_chronological_match_weighted",
            "role": "additive",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pluripotency_exit_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pseudotime_direction_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_synchronous_with_monkey_EPI",
              "chimera_human_EPI_developmentally_advanced",
              "chimera_human_EPI_delayed_at_pre_implantation_only_resolved_post_implantation"
            ]
          },
          {
            "name": "E17_chronological_match_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "resembles_monkey_E17_Gast_EPI",
              "resembles_chimera_monkey_E17_EPI_specifically",
              "chimera_human_E17_EPI_more_advanced_than_monkey_E17_Gast"
            ]
          },
          {
            "name": "pluripotency_exit_paper_inverted_or_prior_driven",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_advances_pluripotency_exit_prematurely",
              "naive_pluripotency_program_specifically_engaged",
              "pluripotency_core_retained_via_KLF17_naive_marker_specifically"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-kimi-k2p6-pi",
      "modelName": "kimi-k2p6",
      "harness": "pi",
      "task": "gse155381_q4_pseudotime_delay",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 2869,
      "cost": 1.32593356,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "pseudotime_direction_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "E17_chronological_match_weighted",
            "role": "additive",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pluripotency_exit_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pseudotime_direction_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_synchronous_with_monkey_EPI",
              "chimera_human_EPI_developmentally_advanced",
              "chimera_human_EPI_delayed_at_pre_implantation_only_resolved_post_implantation"
            ]
          },
          {
            "name": "E17_chronological_match_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "resembles_monkey_E17_Gast_EPI",
              "resembles_chimera_monkey_E17_EPI_specifically",
              "chimera_human_E17_EPI_more_advanced_than_monkey_E17_Gast"
            ]
          },
          {
            "name": "pluripotency_exit_paper_inverted_or_prior_driven",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_advances_pluripotency_exit_prematurely",
              "naive_pluripotency_program_specifically_engaged",
              "pluripotency_core_retained_via_KLF17_naive_marker_specifically"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-6-claude-code",
      "modelName": "claude-opus-4-6",
      "harness": "claude-code",
      "task": "gse155381_q5_chimera_monkey_perturbation",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 477,
      "cost": 1.09097225,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "deg_direction_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "comparison_subset_choice_weighted",
            "role": "additive",
            "field": "$.comparison_subset_choice",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "panel_resolved_strong_target_ccn1_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN1",
              "CYR61"
            ]
          },
          {
            "name": "panel_resolved_strong_target_ccn2_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN2",
              "CTGF"
            ]
          },
          {
            "name": "hippo_pathway_weighted",
            "role": "additive",
            "field": "$.hippo_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "tgf_beta_pathway_weighted",
            "role": "additive",
            "field": "$.tgf_beta_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "deg_direction_batch_effect_uncorrected",
            "role": "hard_fail",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_dominated_global",
              "up_dominated_in_chosen_subset"
            ]
          },
          {
            "name": "silent_panel_dropout_paired_with_directional_pathway_claim",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "hippo_pathway_direction": {
                "op": "in",
                "values": [
                  "hippo_pathway_upregulated",
                  "hippo_pathway_unchanged",
                  "wnt_pathway_specifically_perturbed",
                  "stress_response_pathway_dominantly_engaged",
                  "no_pathway_signal_above_confound",
                  "yap_tead_direct_target_panel_downregulated",
                  "yap_tead_panel_dependent_direction",
                  "hippo_pathway_downregulated"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-6-claude-code",
      "modelName": "claude-opus-4-6",
      "harness": "claude-code",
      "task": "gse155381_q5_chimera_monkey_perturbation",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 585,
      "cost": 1.1800415,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "deg_direction_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "comparison_subset_choice_weighted",
            "role": "additive",
            "field": "$.comparison_subset_choice",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "panel_resolved_strong_target_ccn1_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN1",
              "CYR61"
            ]
          },
          {
            "name": "panel_resolved_strong_target_ccn2_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN2",
              "CTGF"
            ]
          },
          {
            "name": "hippo_pathway_weighted",
            "role": "additive",
            "field": "$.hippo_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "tgf_beta_pathway_weighted",
            "role": "additive",
            "field": "$.tgf_beta_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "deg_direction_batch_effect_uncorrected",
            "role": "hard_fail",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_dominated_global",
              "up_dominated_in_chosen_subset"
            ]
          },
          {
            "name": "silent_panel_dropout_paired_with_directional_pathway_claim",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "hippo_pathway_direction": {
                "op": "in",
                "values": [
                  "hippo_pathway_upregulated",
                  "hippo_pathway_unchanged",
                  "wnt_pathway_specifically_perturbed",
                  "stress_response_pathway_dominantly_engaged",
                  "no_pathway_signal_above_confound",
                  "yap_tead_direct_target_panel_downregulated",
                  "yap_tead_panel_dependent_direction",
                  "hippo_pathway_downregulated"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-6-claude-code",
      "modelName": "claude-opus-4-6",
      "harness": "claude-code",
      "task": "gse155381_q5_chimera_monkey_perturbation",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 797,
      "cost": 1.5906697499999998,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "deg_direction_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "comparison_subset_choice_weighted",
            "role": "additive",
            "field": "$.comparison_subset_choice",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "panel_resolved_strong_target_ccn1_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN1",
              "CYR61"
            ]
          },
          {
            "name": "panel_resolved_strong_target_ccn2_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN2",
              "CTGF"
            ]
          },
          {
            "name": "hippo_pathway_weighted",
            "role": "additive",
            "field": "$.hippo_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "tgf_beta_pathway_weighted",
            "role": "additive",
            "field": "$.tgf_beta_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "deg_direction_batch_effect_uncorrected",
            "role": "hard_fail",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_dominated_global",
              "up_dominated_in_chosen_subset"
            ]
          },
          {
            "name": "silent_panel_dropout_paired_with_directional_pathway_claim",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "hippo_pathway_direction": {
                "op": "in",
                "values": [
                  "hippo_pathway_upregulated",
                  "hippo_pathway_unchanged",
                  "wnt_pathway_specifically_perturbed",
                  "stress_response_pathway_dominantly_engaged",
                  "no_pathway_signal_above_confound",
                  "yap_tead_direct_target_panel_downregulated",
                  "yap_tead_panel_dependent_direction",
                  "hippo_pathway_downregulated"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-6-pi",
      "modelName": "claude-opus-4-6",
      "harness": "pi",
      "task": "gse155381_q5_chimera_monkey_perturbation",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 555,
      "cost": 0.9361125,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "deg_direction_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "comparison_subset_choice_weighted",
            "role": "additive",
            "field": "$.comparison_subset_choice",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "panel_resolved_strong_target_ccn1_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN1",
              "CYR61"
            ]
          },
          {
            "name": "panel_resolved_strong_target_ccn2_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN2",
              "CTGF"
            ]
          },
          {
            "name": "hippo_pathway_weighted",
            "role": "additive",
            "field": "$.hippo_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "tgf_beta_pathway_weighted",
            "role": "additive",
            "field": "$.tgf_beta_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "deg_direction_batch_effect_uncorrected",
            "role": "hard_fail",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_dominated_global",
              "up_dominated_in_chosen_subset"
            ]
          },
          {
            "name": "silent_panel_dropout_paired_with_directional_pathway_claim",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "hippo_pathway_direction": {
                "op": "in",
                "values": [
                  "hippo_pathway_upregulated",
                  "hippo_pathway_unchanged",
                  "wnt_pathway_specifically_perturbed",
                  "stress_response_pathway_dominantly_engaged",
                  "no_pathway_signal_above_confound",
                  "yap_tead_direct_target_panel_downregulated",
                  "yap_tead_panel_dependent_direction",
                  "hippo_pathway_downregulated"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-6-pi",
      "modelName": "claude-opus-4-6",
      "harness": "pi",
      "task": "gse155381_q5_chimera_monkey_perturbation",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 1567,
      "cost": 2.01955975,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "deg_direction_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "comparison_subset_choice_weighted",
            "role": "additive",
            "field": "$.comparison_subset_choice",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "panel_resolved_strong_target_ccn1_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN1",
              "CYR61"
            ]
          },
          {
            "name": "panel_resolved_strong_target_ccn2_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN2",
              "CTGF"
            ]
          },
          {
            "name": "hippo_pathway_weighted",
            "role": "additive",
            "field": "$.hippo_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "tgf_beta_pathway_weighted",
            "role": "additive",
            "field": "$.tgf_beta_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "deg_direction_batch_effect_uncorrected",
            "role": "hard_fail",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_dominated_global",
              "up_dominated_in_chosen_subset"
            ]
          },
          {
            "name": "silent_panel_dropout_paired_with_directional_pathway_claim",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "hippo_pathway_direction": {
                "op": "in",
                "values": [
                  "hippo_pathway_upregulated",
                  "hippo_pathway_unchanged",
                  "wnt_pathway_specifically_perturbed",
                  "stress_response_pathway_dominantly_engaged",
                  "no_pathway_signal_above_confound",
                  "yap_tead_direct_target_panel_downregulated",
                  "yap_tead_panel_dependent_direction",
                  "hippo_pathway_downregulated"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-6-pi",
      "modelName": "claude-opus-4-6",
      "harness": "pi",
      "task": "gse155381_q5_chimera_monkey_perturbation",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 538,
      "cost": 0.960346,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "deg_direction_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "comparison_subset_choice_weighted",
            "role": "additive",
            "field": "$.comparison_subset_choice",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "panel_resolved_strong_target_ccn1_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN1",
              "CYR61"
            ]
          },
          {
            "name": "panel_resolved_strong_target_ccn2_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN2",
              "CTGF"
            ]
          },
          {
            "name": "hippo_pathway_weighted",
            "role": "additive",
            "field": "$.hippo_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "tgf_beta_pathway_weighted",
            "role": "additive",
            "field": "$.tgf_beta_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "deg_direction_batch_effect_uncorrected",
            "role": "hard_fail",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_dominated_global",
              "up_dominated_in_chosen_subset"
            ]
          },
          {
            "name": "silent_panel_dropout_paired_with_directional_pathway_claim",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "hippo_pathway_direction": {
                "op": "in",
                "values": [
                  "hippo_pathway_upregulated",
                  "hippo_pathway_unchanged",
                  "wnt_pathway_specifically_perturbed",
                  "stress_response_pathway_dominantly_engaged",
                  "no_pathway_signal_above_confound",
                  "yap_tead_direct_target_panel_downregulated",
                  "yap_tead_panel_dependent_direction",
                  "hippo_pathway_downregulated"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-7-claude-code",
      "modelName": "claude-opus-4-7",
      "harness": "claude-code",
      "task": "gse155381_q5_chimera_monkey_perturbation",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 1292,
      "cost": 4.45934575,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "deg_direction_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "comparison_subset_choice_weighted",
            "role": "additive",
            "field": "$.comparison_subset_choice",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "panel_resolved_strong_target_ccn1_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN1",
              "CYR61"
            ]
          },
          {
            "name": "panel_resolved_strong_target_ccn2_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN2",
              "CTGF"
            ]
          },
          {
            "name": "hippo_pathway_weighted",
            "role": "additive",
            "field": "$.hippo_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "tgf_beta_pathway_weighted",
            "role": "additive",
            "field": "$.tgf_beta_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "deg_direction_batch_effect_uncorrected",
            "role": "hard_fail",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_dominated_global",
              "up_dominated_in_chosen_subset"
            ]
          },
          {
            "name": "silent_panel_dropout_paired_with_directional_pathway_claim",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "hippo_pathway_direction": {
                "op": "in",
                "values": [
                  "hippo_pathway_upregulated",
                  "hippo_pathway_unchanged",
                  "wnt_pathway_specifically_perturbed",
                  "stress_response_pathway_dominantly_engaged",
                  "no_pathway_signal_above_confound",
                  "yap_tead_direct_target_panel_downregulated",
                  "yap_tead_panel_dependent_direction",
                  "hippo_pathway_downregulated"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-7-claude-code",
      "modelName": "claude-opus-4-7",
      "harness": "claude-code",
      "task": "gse155381_q5_chimera_monkey_perturbation",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 1110,
      "cost": 4.372108750000001,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "deg_direction_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "comparison_subset_choice_weighted",
            "role": "additive",
            "field": "$.comparison_subset_choice",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "panel_resolved_strong_target_ccn1_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN1",
              "CYR61"
            ]
          },
          {
            "name": "panel_resolved_strong_target_ccn2_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN2",
              "CTGF"
            ]
          },
          {
            "name": "hippo_pathway_weighted",
            "role": "additive",
            "field": "$.hippo_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "tgf_beta_pathway_weighted",
            "role": "additive",
            "field": "$.tgf_beta_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "deg_direction_batch_effect_uncorrected",
            "role": "hard_fail",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_dominated_global",
              "up_dominated_in_chosen_subset"
            ]
          },
          {
            "name": "silent_panel_dropout_paired_with_directional_pathway_claim",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "hippo_pathway_direction": {
                "op": "in",
                "values": [
                  "hippo_pathway_upregulated",
                  "hippo_pathway_unchanged",
                  "wnt_pathway_specifically_perturbed",
                  "stress_response_pathway_dominantly_engaged",
                  "no_pathway_signal_above_confound",
                  "yap_tead_direct_target_panel_downregulated",
                  "yap_tead_panel_dependent_direction",
                  "hippo_pathway_downregulated"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-7-claude-code",
      "modelName": "claude-opus-4-7",
      "harness": "claude-code",
      "task": "gse155381_q5_chimera_monkey_perturbation",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1102,
      "cost": 4.213935,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "deg_direction_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "comparison_subset_choice_weighted",
            "role": "additive",
            "field": "$.comparison_subset_choice",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "panel_resolved_strong_target_ccn1_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN1",
              "CYR61"
            ]
          },
          {
            "name": "panel_resolved_strong_target_ccn2_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN2",
              "CTGF"
            ]
          },
          {
            "name": "hippo_pathway_weighted",
            "role": "additive",
            "field": "$.hippo_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "tgf_beta_pathway_weighted",
            "role": "additive",
            "field": "$.tgf_beta_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "deg_direction_batch_effect_uncorrected",
            "role": "hard_fail",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_dominated_global",
              "up_dominated_in_chosen_subset"
            ]
          },
          {
            "name": "silent_panel_dropout_paired_with_directional_pathway_claim",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "hippo_pathway_direction": {
                "op": "in",
                "values": [
                  "hippo_pathway_upregulated",
                  "hippo_pathway_unchanged",
                  "wnt_pathway_specifically_perturbed",
                  "stress_response_pathway_dominantly_engaged",
                  "no_pathway_signal_above_confound",
                  "yap_tead_direct_target_panel_downregulated",
                  "yap_tead_panel_dependent_direction",
                  "hippo_pathway_downregulated"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-7-pi",
      "modelName": "claude-opus-4-7",
      "harness": "pi",
      "task": "gse155381_q5_chimera_monkey_perturbation",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 458,
      "cost": 0.7343085,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "deg_direction_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "comparison_subset_choice_weighted",
            "role": "additive",
            "field": "$.comparison_subset_choice",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "panel_resolved_strong_target_ccn1_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN1",
              "CYR61"
            ]
          },
          {
            "name": "panel_resolved_strong_target_ccn2_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN2",
              "CTGF"
            ]
          },
          {
            "name": "hippo_pathway_weighted",
            "role": "additive",
            "field": "$.hippo_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "tgf_beta_pathway_weighted",
            "role": "additive",
            "field": "$.tgf_beta_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "deg_direction_batch_effect_uncorrected",
            "role": "hard_fail",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_dominated_global",
              "up_dominated_in_chosen_subset"
            ]
          },
          {
            "name": "silent_panel_dropout_paired_with_directional_pathway_claim",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "hippo_pathway_direction": {
                "op": "in",
                "values": [
                  "hippo_pathway_upregulated",
                  "hippo_pathway_unchanged",
                  "wnt_pathway_specifically_perturbed",
                  "stress_response_pathway_dominantly_engaged",
                  "no_pathway_signal_above_confound",
                  "yap_tead_direct_target_panel_downregulated",
                  "yap_tead_panel_dependent_direction",
                  "hippo_pathway_downregulated"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-7-pi",
      "modelName": "claude-opus-4-7",
      "harness": "pi",
      "task": "gse155381_q5_chimera_monkey_perturbation",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 802,
      "cost": 1.133102,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "deg_direction_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "comparison_subset_choice_weighted",
            "role": "additive",
            "field": "$.comparison_subset_choice",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "panel_resolved_strong_target_ccn1_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN1",
              "CYR61"
            ]
          },
          {
            "name": "panel_resolved_strong_target_ccn2_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN2",
              "CTGF"
            ]
          },
          {
            "name": "hippo_pathway_weighted",
            "role": "additive",
            "field": "$.hippo_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "tgf_beta_pathway_weighted",
            "role": "additive",
            "field": "$.tgf_beta_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "deg_direction_batch_effect_uncorrected",
            "role": "hard_fail",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_dominated_global",
              "up_dominated_in_chosen_subset"
            ]
          },
          {
            "name": "silent_panel_dropout_paired_with_directional_pathway_claim",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "hippo_pathway_direction": {
                "op": "in",
                "values": [
                  "hippo_pathway_upregulated",
                  "hippo_pathway_unchanged",
                  "wnt_pathway_specifically_perturbed",
                  "stress_response_pathway_dominantly_engaged",
                  "no_pathway_signal_above_confound",
                  "yap_tead_direct_target_panel_downregulated",
                  "yap_tead_panel_dependent_direction",
                  "hippo_pathway_downregulated"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-7-pi",
      "modelName": "claude-opus-4-7",
      "harness": "pi",
      "task": "gse155381_q5_chimera_monkey_perturbation",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 726,
      "cost": 1.3004765000000003,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "deg_direction_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "comparison_subset_choice_weighted",
            "role": "additive",
            "field": "$.comparison_subset_choice",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "panel_resolved_strong_target_ccn1_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN1",
              "CYR61"
            ]
          },
          {
            "name": "panel_resolved_strong_target_ccn2_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN2",
              "CTGF"
            ]
          },
          {
            "name": "hippo_pathway_weighted",
            "role": "additive",
            "field": "$.hippo_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "tgf_beta_pathway_weighted",
            "role": "additive",
            "field": "$.tgf_beta_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "deg_direction_batch_effect_uncorrected",
            "role": "hard_fail",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_dominated_global",
              "up_dominated_in_chosen_subset"
            ]
          },
          {
            "name": "silent_panel_dropout_paired_with_directional_pathway_claim",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "hippo_pathway_direction": {
                "op": "in",
                "values": [
                  "hippo_pathway_upregulated",
                  "hippo_pathway_unchanged",
                  "wnt_pathway_specifically_perturbed",
                  "stress_response_pathway_dominantly_engaged",
                  "no_pathway_signal_above_confound",
                  "yap_tead_direct_target_panel_downregulated",
                  "yap_tead_panel_dependent_direction",
                  "hippo_pathway_downregulated"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-8-claude-code",
      "modelName": "claude-opus-4-8",
      "harness": "claude-code",
      "task": "gse155381_q5_chimera_monkey_perturbation",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 1196,
      "cost": 3.7344765000000004,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "deg_direction_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "comparison_subset_choice_weighted",
            "role": "additive",
            "field": "$.comparison_subset_choice",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "panel_resolved_strong_target_ccn1_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN1",
              "CYR61"
            ]
          },
          {
            "name": "panel_resolved_strong_target_ccn2_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN2",
              "CTGF"
            ]
          },
          {
            "name": "hippo_pathway_weighted",
            "role": "additive",
            "field": "$.hippo_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "tgf_beta_pathway_weighted",
            "role": "additive",
            "field": "$.tgf_beta_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "deg_direction_batch_effect_uncorrected",
            "role": "hard_fail",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_dominated_global",
              "up_dominated_in_chosen_subset"
            ]
          },
          {
            "name": "silent_panel_dropout_paired_with_directional_pathway_claim",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "hippo_pathway_direction": {
                "op": "in",
                "values": [
                  "hippo_pathway_upregulated",
                  "hippo_pathway_unchanged",
                  "wnt_pathway_specifically_perturbed",
                  "stress_response_pathway_dominantly_engaged",
                  "no_pathway_signal_above_confound",
                  "yap_tead_direct_target_panel_downregulated",
                  "yap_tead_panel_dependent_direction",
                  "hippo_pathway_downregulated"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-8-claude-code",
      "modelName": "claude-opus-4-8",
      "harness": "claude-code",
      "task": "gse155381_q5_chimera_monkey_perturbation",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 1911,
      "cost": 7.2448265,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "deg_direction_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "comparison_subset_choice_weighted",
            "role": "additive",
            "field": "$.comparison_subset_choice",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "panel_resolved_strong_target_ccn1_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN1",
              "CYR61"
            ]
          },
          {
            "name": "panel_resolved_strong_target_ccn2_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN2",
              "CTGF"
            ]
          },
          {
            "name": "hippo_pathway_weighted",
            "role": "additive",
            "field": "$.hippo_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "tgf_beta_pathway_weighted",
            "role": "additive",
            "field": "$.tgf_beta_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "deg_direction_batch_effect_uncorrected",
            "role": "hard_fail",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_dominated_global",
              "up_dominated_in_chosen_subset"
            ]
          },
          {
            "name": "silent_panel_dropout_paired_with_directional_pathway_claim",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "hippo_pathway_direction": {
                "op": "in",
                "values": [
                  "hippo_pathway_upregulated",
                  "hippo_pathway_unchanged",
                  "wnt_pathway_specifically_perturbed",
                  "stress_response_pathway_dominantly_engaged",
                  "no_pathway_signal_above_confound",
                  "yap_tead_direct_target_panel_downregulated",
                  "yap_tead_panel_dependent_direction",
                  "hippo_pathway_downregulated"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-8-claude-code",
      "modelName": "claude-opus-4-8",
      "harness": "claude-code",
      "task": "gse155381_q5_chimera_monkey_perturbation",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1378,
      "cost": 3.9660172499999997,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "deg_direction_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "comparison_subset_choice_weighted",
            "role": "additive",
            "field": "$.comparison_subset_choice",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "panel_resolved_strong_target_ccn1_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN1",
              "CYR61"
            ]
          },
          {
            "name": "panel_resolved_strong_target_ccn2_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN2",
              "CTGF"
            ]
          },
          {
            "name": "hippo_pathway_weighted",
            "role": "additive",
            "field": "$.hippo_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "tgf_beta_pathway_weighted",
            "role": "additive",
            "field": "$.tgf_beta_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "deg_direction_batch_effect_uncorrected",
            "role": "hard_fail",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_dominated_global",
              "up_dominated_in_chosen_subset"
            ]
          },
          {
            "name": "silent_panel_dropout_paired_with_directional_pathway_claim",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "hippo_pathway_direction": {
                "op": "in",
                "values": [
                  "hippo_pathway_upregulated",
                  "hippo_pathway_unchanged",
                  "wnt_pathway_specifically_perturbed",
                  "stress_response_pathway_dominantly_engaged",
                  "no_pathway_signal_above_confound",
                  "yap_tead_direct_target_panel_downregulated",
                  "yap_tead_panel_dependent_direction",
                  "hippo_pathway_downregulated"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-8-pi",
      "modelName": "claude-opus-4-8",
      "harness": "pi",
      "task": "gse155381_q5_chimera_monkey_perturbation",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 2172,
      "cost": 3.687858750000001,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "deg_direction_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "comparison_subset_choice_weighted",
            "role": "additive",
            "field": "$.comparison_subset_choice",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "panel_resolved_strong_target_ccn1_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN1",
              "CYR61"
            ]
          },
          {
            "name": "panel_resolved_strong_target_ccn2_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN2",
              "CTGF"
            ]
          },
          {
            "name": "hippo_pathway_weighted",
            "role": "additive",
            "field": "$.hippo_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "tgf_beta_pathway_weighted",
            "role": "additive",
            "field": "$.tgf_beta_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "deg_direction_batch_effect_uncorrected",
            "role": "hard_fail",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_dominated_global",
              "up_dominated_in_chosen_subset"
            ]
          },
          {
            "name": "silent_panel_dropout_paired_with_directional_pathway_claim",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "hippo_pathway_direction": {
                "op": "in",
                "values": [
                  "hippo_pathway_upregulated",
                  "hippo_pathway_unchanged",
                  "wnt_pathway_specifically_perturbed",
                  "stress_response_pathway_dominantly_engaged",
                  "no_pathway_signal_above_confound",
                  "yap_tead_direct_target_panel_downregulated",
                  "yap_tead_panel_dependent_direction",
                  "hippo_pathway_downregulated"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-8-pi",
      "modelName": "claude-opus-4-8",
      "harness": "pi",
      "task": "gse155381_q5_chimera_monkey_perturbation",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 1323,
      "cost": 2.7129969999999997,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "deg_direction_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "comparison_subset_choice_weighted",
            "role": "additive",
            "field": "$.comparison_subset_choice",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "panel_resolved_strong_target_ccn1_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN1",
              "CYR61"
            ]
          },
          {
            "name": "panel_resolved_strong_target_ccn2_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN2",
              "CTGF"
            ]
          },
          {
            "name": "hippo_pathway_weighted",
            "role": "additive",
            "field": "$.hippo_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "tgf_beta_pathway_weighted",
            "role": "additive",
            "field": "$.tgf_beta_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "deg_direction_batch_effect_uncorrected",
            "role": "hard_fail",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_dominated_global",
              "up_dominated_in_chosen_subset"
            ]
          },
          {
            "name": "silent_panel_dropout_paired_with_directional_pathway_claim",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "hippo_pathway_direction": {
                "op": "in",
                "values": [
                  "hippo_pathway_upregulated",
                  "hippo_pathway_unchanged",
                  "wnt_pathway_specifically_perturbed",
                  "stress_response_pathway_dominantly_engaged",
                  "no_pathway_signal_above_confound",
                  "yap_tead_direct_target_panel_downregulated",
                  "yap_tead_panel_dependent_direction",
                  "hippo_pathway_downregulated"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-8-pi",
      "modelName": "claude-opus-4-8",
      "harness": "pi",
      "task": "gse155381_q5_chimera_monkey_perturbation",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 2119,
      "cost": 2.8716030000000003,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "deg_direction_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "comparison_subset_choice_weighted",
            "role": "additive",
            "field": "$.comparison_subset_choice",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "panel_resolved_strong_target_ccn1_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN1",
              "CYR61"
            ]
          },
          {
            "name": "panel_resolved_strong_target_ccn2_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN2",
              "CTGF"
            ]
          },
          {
            "name": "hippo_pathway_weighted",
            "role": "additive",
            "field": "$.hippo_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "tgf_beta_pathway_weighted",
            "role": "additive",
            "field": "$.tgf_beta_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "deg_direction_batch_effect_uncorrected",
            "role": "hard_fail",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_dominated_global",
              "up_dominated_in_chosen_subset"
            ]
          },
          {
            "name": "silent_panel_dropout_paired_with_directional_pathway_claim",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "hippo_pathway_direction": {
                "op": "in",
                "values": [
                  "hippo_pathway_upregulated",
                  "hippo_pathway_unchanged",
                  "wnt_pathway_specifically_perturbed",
                  "stress_response_pathway_dominantly_engaged",
                  "no_pathway_signal_above_confound",
                  "yap_tead_direct_target_panel_downregulated",
                  "yap_tead_panel_dependent_direction",
                  "hippo_pathway_downregulated"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-4-6-pi",
      "modelName": "claude-sonnet-4-6",
      "harness": "pi",
      "task": "gse155381_q5_chimera_monkey_perturbation",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 991,
      "cost": 1.2497917499999998,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "deg_direction_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "comparison_subset_choice_weighted",
            "role": "additive",
            "field": "$.comparison_subset_choice",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "panel_resolved_strong_target_ccn1_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN1",
              "CYR61"
            ]
          },
          {
            "name": "panel_resolved_strong_target_ccn2_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN2",
              "CTGF"
            ]
          },
          {
            "name": "hippo_pathway_weighted",
            "role": "additive",
            "field": "$.hippo_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "tgf_beta_pathway_weighted",
            "role": "additive",
            "field": "$.tgf_beta_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "deg_direction_batch_effect_uncorrected",
            "role": "hard_fail",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_dominated_global",
              "up_dominated_in_chosen_subset"
            ]
          },
          {
            "name": "silent_panel_dropout_paired_with_directional_pathway_claim",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "hippo_pathway_direction": {
                "op": "in",
                "values": [
                  "hippo_pathway_upregulated",
                  "hippo_pathway_unchanged",
                  "wnt_pathway_specifically_perturbed",
                  "stress_response_pathway_dominantly_engaged",
                  "no_pathway_signal_above_confound",
                  "yap_tead_direct_target_panel_downregulated",
                  "yap_tead_panel_dependent_direction",
                  "hippo_pathway_downregulated"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-4-6-pi",
      "modelName": "claude-sonnet-4-6",
      "harness": "pi",
      "task": "gse155381_q5_chimera_monkey_perturbation",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 1044,
      "cost": 0.7539804,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "deg_direction_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "comparison_subset_choice_weighted",
            "role": "additive",
            "field": "$.comparison_subset_choice",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "panel_resolved_strong_target_ccn1_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN1",
              "CYR61"
            ]
          },
          {
            "name": "panel_resolved_strong_target_ccn2_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN2",
              "CTGF"
            ]
          },
          {
            "name": "hippo_pathway_weighted",
            "role": "additive",
            "field": "$.hippo_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "tgf_beta_pathway_weighted",
            "role": "additive",
            "field": "$.tgf_beta_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "deg_direction_batch_effect_uncorrected",
            "role": "hard_fail",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_dominated_global",
              "up_dominated_in_chosen_subset"
            ]
          },
          {
            "name": "silent_panel_dropout_paired_with_directional_pathway_claim",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "hippo_pathway_direction": {
                "op": "in",
                "values": [
                  "hippo_pathway_upregulated",
                  "hippo_pathway_unchanged",
                  "wnt_pathway_specifically_perturbed",
                  "stress_response_pathway_dominantly_engaged",
                  "no_pathway_signal_above_confound",
                  "yap_tead_direct_target_panel_downregulated",
                  "yap_tead_panel_dependent_direction",
                  "hippo_pathway_downregulated"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-4-6-pi",
      "modelName": "claude-sonnet-4-6",
      "harness": "pi",
      "task": "gse155381_q5_chimera_monkey_perturbation",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1075,
      "cost": 0.7856299500000001,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "deg_direction_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "comparison_subset_choice_weighted",
            "role": "additive",
            "field": "$.comparison_subset_choice",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "panel_resolved_strong_target_ccn1_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN1",
              "CYR61"
            ]
          },
          {
            "name": "panel_resolved_strong_target_ccn2_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN2",
              "CTGF"
            ]
          },
          {
            "name": "hippo_pathway_weighted",
            "role": "additive",
            "field": "$.hippo_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "tgf_beta_pathway_weighted",
            "role": "additive",
            "field": "$.tgf_beta_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "deg_direction_batch_effect_uncorrected",
            "role": "hard_fail",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_dominated_global",
              "up_dominated_in_chosen_subset"
            ]
          },
          {
            "name": "silent_panel_dropout_paired_with_directional_pathway_claim",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "hippo_pathway_direction": {
                "op": "in",
                "values": [
                  "hippo_pathway_upregulated",
                  "hippo_pathway_unchanged",
                  "wnt_pathway_specifically_perturbed",
                  "stress_response_pathway_dominantly_engaged",
                  "no_pathway_signal_above_confound",
                  "yap_tead_direct_target_panel_downregulated",
                  "yap_tead_panel_dependent_direction",
                  "hippo_pathway_downregulated"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-gemini-3-1-pro-preview-pi",
      "modelName": "gemini-3.1-pro-preview",
      "harness": "pi",
      "task": "gse155381_q5_chimera_monkey_perturbation",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 814,
      "cost": 0.6160513999999998,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "deg_direction_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "comparison_subset_choice_weighted",
            "role": "additive",
            "field": "$.comparison_subset_choice",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "panel_resolved_strong_target_ccn1_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN1",
              "CYR61"
            ]
          },
          {
            "name": "panel_resolved_strong_target_ccn2_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN2",
              "CTGF"
            ]
          },
          {
            "name": "hippo_pathway_weighted",
            "role": "additive",
            "field": "$.hippo_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "tgf_beta_pathway_weighted",
            "role": "additive",
            "field": "$.tgf_beta_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "deg_direction_batch_effect_uncorrected",
            "role": "hard_fail",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_dominated_global",
              "up_dominated_in_chosen_subset"
            ]
          },
          {
            "name": "silent_panel_dropout_paired_with_directional_pathway_claim",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "hippo_pathway_direction": {
                "op": "in",
                "values": [
                  "hippo_pathway_upregulated",
                  "hippo_pathway_unchanged",
                  "wnt_pathway_specifically_perturbed",
                  "stress_response_pathway_dominantly_engaged",
                  "no_pathway_signal_above_confound",
                  "yap_tead_direct_target_panel_downregulated",
                  "yap_tead_panel_dependent_direction",
                  "hippo_pathway_downregulated"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-gemini-3-1-pro-preview-pi",
      "modelName": "gemini-3.1-pro-preview",
      "harness": "pi",
      "task": "gse155381_q5_chimera_monkey_perturbation",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 873,
      "cost": 0.46039459999999993,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "deg_direction_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "comparison_subset_choice_weighted",
            "role": "additive",
            "field": "$.comparison_subset_choice",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "panel_resolved_strong_target_ccn1_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN1",
              "CYR61"
            ]
          },
          {
            "name": "panel_resolved_strong_target_ccn2_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN2",
              "CTGF"
            ]
          },
          {
            "name": "hippo_pathway_weighted",
            "role": "additive",
            "field": "$.hippo_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "tgf_beta_pathway_weighted",
            "role": "additive",
            "field": "$.tgf_beta_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "deg_direction_batch_effect_uncorrected",
            "role": "hard_fail",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_dominated_global",
              "up_dominated_in_chosen_subset"
            ]
          },
          {
            "name": "silent_panel_dropout_paired_with_directional_pathway_claim",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "hippo_pathway_direction": {
                "op": "in",
                "values": [
                  "hippo_pathway_upregulated",
                  "hippo_pathway_unchanged",
                  "wnt_pathway_specifically_perturbed",
                  "stress_response_pathway_dominantly_engaged",
                  "no_pathway_signal_above_confound",
                  "yap_tead_direct_target_panel_downregulated",
                  "yap_tead_panel_dependent_direction",
                  "hippo_pathway_downregulated"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-gemini-3-1-pro-preview-pi",
      "modelName": "gemini-3.1-pro-preview",
      "harness": "pi",
      "task": "gse155381_q5_chimera_monkey_perturbation",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 852,
      "cost": 1.1495818000000007,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "deg_direction_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "comparison_subset_choice_weighted",
            "role": "additive",
            "field": "$.comparison_subset_choice",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "panel_resolved_strong_target_ccn1_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN1",
              "CYR61"
            ]
          },
          {
            "name": "panel_resolved_strong_target_ccn2_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN2",
              "CTGF"
            ]
          },
          {
            "name": "hippo_pathway_weighted",
            "role": "additive",
            "field": "$.hippo_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "tgf_beta_pathway_weighted",
            "role": "additive",
            "field": "$.tgf_beta_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "deg_direction_batch_effect_uncorrected",
            "role": "hard_fail",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_dominated_global",
              "up_dominated_in_chosen_subset"
            ]
          },
          {
            "name": "silent_panel_dropout_paired_with_directional_pathway_claim",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "hippo_pathway_direction": {
                "op": "in",
                "values": [
                  "hippo_pathway_upregulated",
                  "hippo_pathway_unchanged",
                  "wnt_pathway_specifically_perturbed",
                  "stress_response_pathway_dominantly_engaged",
                  "no_pathway_signal_above_confound",
                  "yap_tead_direct_target_panel_downregulated",
                  "yap_tead_panel_dependent_direction",
                  "hippo_pathway_downregulated"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-gemini-3-5-flash-pi",
      "modelName": "gemini-3.5-flash",
      "harness": "pi",
      "task": "gse155381_q5_chimera_monkey_perturbation",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 556,
      "cost": 0.7957417499999999,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "deg_direction_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "comparison_subset_choice_weighted",
            "role": "additive",
            "field": "$.comparison_subset_choice",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "panel_resolved_strong_target_ccn1_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN1",
              "CYR61"
            ]
          },
          {
            "name": "panel_resolved_strong_target_ccn2_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN2",
              "CTGF"
            ]
          },
          {
            "name": "hippo_pathway_weighted",
            "role": "additive",
            "field": "$.hippo_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "tgf_beta_pathway_weighted",
            "role": "additive",
            "field": "$.tgf_beta_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "deg_direction_batch_effect_uncorrected",
            "role": "hard_fail",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_dominated_global",
              "up_dominated_in_chosen_subset"
            ]
          },
          {
            "name": "silent_panel_dropout_paired_with_directional_pathway_claim",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "hippo_pathway_direction": {
                "op": "in",
                "values": [
                  "hippo_pathway_upregulated",
                  "hippo_pathway_unchanged",
                  "wnt_pathway_specifically_perturbed",
                  "stress_response_pathway_dominantly_engaged",
                  "no_pathway_signal_above_confound",
                  "yap_tead_direct_target_panel_downregulated",
                  "yap_tead_panel_dependent_direction",
                  "hippo_pathway_downregulated"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-gemini-3-5-flash-pi",
      "modelName": "gemini-3.5-flash",
      "harness": "pi",
      "task": "gse155381_q5_chimera_monkey_perturbation",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 1025,
      "cost": 1.7390361000000005,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "deg_direction_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "comparison_subset_choice_weighted",
            "role": "additive",
            "field": "$.comparison_subset_choice",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "panel_resolved_strong_target_ccn1_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN1",
              "CYR61"
            ]
          },
          {
            "name": "panel_resolved_strong_target_ccn2_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN2",
              "CTGF"
            ]
          },
          {
            "name": "hippo_pathway_weighted",
            "role": "additive",
            "field": "$.hippo_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "tgf_beta_pathway_weighted",
            "role": "additive",
            "field": "$.tgf_beta_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "deg_direction_batch_effect_uncorrected",
            "role": "hard_fail",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_dominated_global",
              "up_dominated_in_chosen_subset"
            ]
          },
          {
            "name": "silent_panel_dropout_paired_with_directional_pathway_claim",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "hippo_pathway_direction": {
                "op": "in",
                "values": [
                  "hippo_pathway_upregulated",
                  "hippo_pathway_unchanged",
                  "wnt_pathway_specifically_perturbed",
                  "stress_response_pathway_dominantly_engaged",
                  "no_pathway_signal_above_confound",
                  "yap_tead_direct_target_panel_downregulated",
                  "yap_tead_panel_dependent_direction",
                  "hippo_pathway_downregulated"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-gemini-3-5-flash-pi",
      "modelName": "gemini-3.5-flash",
      "harness": "pi",
      "task": "gse155381_q5_chimera_monkey_perturbation",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1119,
      "cost": 2.2456732500000007,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "deg_direction_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "comparison_subset_choice_weighted",
            "role": "additive",
            "field": "$.comparison_subset_choice",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "panel_resolved_strong_target_ccn1_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN1",
              "CYR61"
            ]
          },
          {
            "name": "panel_resolved_strong_target_ccn2_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN2",
              "CTGF"
            ]
          },
          {
            "name": "hippo_pathway_weighted",
            "role": "additive",
            "field": "$.hippo_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "tgf_beta_pathway_weighted",
            "role": "additive",
            "field": "$.tgf_beta_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "deg_direction_batch_effect_uncorrected",
            "role": "hard_fail",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_dominated_global",
              "up_dominated_in_chosen_subset"
            ]
          },
          {
            "name": "silent_panel_dropout_paired_with_directional_pathway_claim",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "hippo_pathway_direction": {
                "op": "in",
                "values": [
                  "hippo_pathway_upregulated",
                  "hippo_pathway_unchanged",
                  "wnt_pathway_specifically_perturbed",
                  "stress_response_pathway_dominantly_engaged",
                  "no_pathway_signal_above_confound",
                  "yap_tead_direct_target_panel_downregulated",
                  "yap_tead_panel_dependent_direction",
                  "hippo_pathway_downregulated"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-glm-5p2-pi",
      "modelName": "glm-5p2",
      "harness": "pi",
      "task": "gse155381_q5_chimera_monkey_perturbation",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 3525,
      "cost": 4.06840508,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "deg_direction_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "comparison_subset_choice_weighted",
            "role": "additive",
            "field": "$.comparison_subset_choice",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "panel_resolved_strong_target_ccn1_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN1",
              "CYR61"
            ]
          },
          {
            "name": "panel_resolved_strong_target_ccn2_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN2",
              "CTGF"
            ]
          },
          {
            "name": "hippo_pathway_weighted",
            "role": "additive",
            "field": "$.hippo_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "tgf_beta_pathway_weighted",
            "role": "additive",
            "field": "$.tgf_beta_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "deg_direction_batch_effect_uncorrected",
            "role": "hard_fail",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_dominated_global",
              "up_dominated_in_chosen_subset"
            ]
          },
          {
            "name": "silent_panel_dropout_paired_with_directional_pathway_claim",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "hippo_pathway_direction": {
                "op": "in",
                "values": [
                  "hippo_pathway_upregulated",
                  "hippo_pathway_unchanged",
                  "wnt_pathway_specifically_perturbed",
                  "stress_response_pathway_dominantly_engaged",
                  "no_pathway_signal_above_confound",
                  "yap_tead_direct_target_panel_downregulated",
                  "yap_tead_panel_dependent_direction",
                  "hippo_pathway_downregulated"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-glm-5p2-pi",
      "modelName": "glm-5p2",
      "harness": "pi",
      "task": "gse155381_q5_chimera_monkey_perturbation",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 1256,
      "cost": 0.9618263399999999,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "deg_direction_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "comparison_subset_choice_weighted",
            "role": "additive",
            "field": "$.comparison_subset_choice",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "panel_resolved_strong_target_ccn1_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN1",
              "CYR61"
            ]
          },
          {
            "name": "panel_resolved_strong_target_ccn2_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN2",
              "CTGF"
            ]
          },
          {
            "name": "hippo_pathway_weighted",
            "role": "additive",
            "field": "$.hippo_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "tgf_beta_pathway_weighted",
            "role": "additive",
            "field": "$.tgf_beta_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "deg_direction_batch_effect_uncorrected",
            "role": "hard_fail",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_dominated_global",
              "up_dominated_in_chosen_subset"
            ]
          },
          {
            "name": "silent_panel_dropout_paired_with_directional_pathway_claim",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "hippo_pathway_direction": {
                "op": "in",
                "values": [
                  "hippo_pathway_upregulated",
                  "hippo_pathway_unchanged",
                  "wnt_pathway_specifically_perturbed",
                  "stress_response_pathway_dominantly_engaged",
                  "no_pathway_signal_above_confound",
                  "yap_tead_direct_target_panel_downregulated",
                  "yap_tead_panel_dependent_direction",
                  "hippo_pathway_downregulated"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-glm-5p2-pi",
      "modelName": "glm-5p2",
      "harness": "pi",
      "task": "gse155381_q5_chimera_monkey_perturbation",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1852,
      "cost": 2.87670602,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "deg_direction_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "comparison_subset_choice_weighted",
            "role": "additive",
            "field": "$.comparison_subset_choice",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "panel_resolved_strong_target_ccn1_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN1",
              "CYR61"
            ]
          },
          {
            "name": "panel_resolved_strong_target_ccn2_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN2",
              "CTGF"
            ]
          },
          {
            "name": "hippo_pathway_weighted",
            "role": "additive",
            "field": "$.hippo_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "tgf_beta_pathway_weighted",
            "role": "additive",
            "field": "$.tgf_beta_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "deg_direction_batch_effect_uncorrected",
            "role": "hard_fail",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_dominated_global",
              "up_dominated_in_chosen_subset"
            ]
          },
          {
            "name": "silent_panel_dropout_paired_with_directional_pathway_claim",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "hippo_pathway_direction": {
                "op": "in",
                "values": [
                  "hippo_pathway_upregulated",
                  "hippo_pathway_unchanged",
                  "wnt_pathway_specifically_perturbed",
                  "stress_response_pathway_dominantly_engaged",
                  "no_pathway_signal_above_confound",
                  "yap_tead_direct_target_panel_downregulated",
                  "yap_tead_panel_dependent_direction",
                  "hippo_pathway_downregulated"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-4-codex",
      "modelName": "gpt-5.4",
      "harness": "openai-codex",
      "task": "gse155381_q5_chimera_monkey_perturbation",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 841,
      "cost": 1.6327555,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "deg_direction_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "comparison_subset_choice_weighted",
            "role": "additive",
            "field": "$.comparison_subset_choice",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "panel_resolved_strong_target_ccn1_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN1",
              "CYR61"
            ]
          },
          {
            "name": "panel_resolved_strong_target_ccn2_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN2",
              "CTGF"
            ]
          },
          {
            "name": "hippo_pathway_weighted",
            "role": "additive",
            "field": "$.hippo_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "tgf_beta_pathway_weighted",
            "role": "additive",
            "field": "$.tgf_beta_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "deg_direction_batch_effect_uncorrected",
            "role": "hard_fail",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_dominated_global",
              "up_dominated_in_chosen_subset"
            ]
          },
          {
            "name": "silent_panel_dropout_paired_with_directional_pathway_claim",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "hippo_pathway_direction": {
                "op": "in",
                "values": [
                  "hippo_pathway_upregulated",
                  "hippo_pathway_unchanged",
                  "wnt_pathway_specifically_perturbed",
                  "stress_response_pathway_dominantly_engaged",
                  "no_pathway_signal_above_confound",
                  "yap_tead_direct_target_panel_downregulated",
                  "yap_tead_panel_dependent_direction",
                  "hippo_pathway_downregulated"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-4-codex",
      "modelName": "gpt-5.4",
      "harness": "openai-codex",
      "task": "gse155381_q5_chimera_monkey_perturbation",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 524,
      "cost": 0.713469,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "deg_direction_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "comparison_subset_choice_weighted",
            "role": "additive",
            "field": "$.comparison_subset_choice",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "panel_resolved_strong_target_ccn1_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN1",
              "CYR61"
            ]
          },
          {
            "name": "panel_resolved_strong_target_ccn2_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN2",
              "CTGF"
            ]
          },
          {
            "name": "hippo_pathway_weighted",
            "role": "additive",
            "field": "$.hippo_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "tgf_beta_pathway_weighted",
            "role": "additive",
            "field": "$.tgf_beta_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "deg_direction_batch_effect_uncorrected",
            "role": "hard_fail",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_dominated_global",
              "up_dominated_in_chosen_subset"
            ]
          },
          {
            "name": "silent_panel_dropout_paired_with_directional_pathway_claim",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "hippo_pathway_direction": {
                "op": "in",
                "values": [
                  "hippo_pathway_upregulated",
                  "hippo_pathway_unchanged",
                  "wnt_pathway_specifically_perturbed",
                  "stress_response_pathway_dominantly_engaged",
                  "no_pathway_signal_above_confound",
                  "yap_tead_direct_target_panel_downregulated",
                  "yap_tead_panel_dependent_direction",
                  "hippo_pathway_downregulated"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-4-codex",
      "modelName": "gpt-5.4",
      "harness": "openai-codex",
      "task": "gse155381_q5_chimera_monkey_perturbation",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 587,
      "cost": 2.18696,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "deg_direction_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "comparison_subset_choice_weighted",
            "role": "additive",
            "field": "$.comparison_subset_choice",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "panel_resolved_strong_target_ccn1_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN1",
              "CYR61"
            ]
          },
          {
            "name": "panel_resolved_strong_target_ccn2_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN2",
              "CTGF"
            ]
          },
          {
            "name": "hippo_pathway_weighted",
            "role": "additive",
            "field": "$.hippo_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "tgf_beta_pathway_weighted",
            "role": "additive",
            "field": "$.tgf_beta_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "deg_direction_batch_effect_uncorrected",
            "role": "hard_fail",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_dominated_global",
              "up_dominated_in_chosen_subset"
            ]
          },
          {
            "name": "silent_panel_dropout_paired_with_directional_pathway_claim",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "hippo_pathway_direction": {
                "op": "in",
                "values": [
                  "hippo_pathway_upregulated",
                  "hippo_pathway_unchanged",
                  "wnt_pathway_specifically_perturbed",
                  "stress_response_pathway_dominantly_engaged",
                  "no_pathway_signal_above_confound",
                  "yap_tead_direct_target_panel_downregulated",
                  "yap_tead_panel_dependent_direction",
                  "hippo_pathway_downregulated"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-4-pi",
      "modelName": "gpt-5.4",
      "harness": "pi",
      "task": "gse155381_q5_chimera_monkey_perturbation",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 1553,
      "cost": 1.0325685000000002,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "deg_direction_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "comparison_subset_choice_weighted",
            "role": "additive",
            "field": "$.comparison_subset_choice",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "panel_resolved_strong_target_ccn1_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN1",
              "CYR61"
            ]
          },
          {
            "name": "panel_resolved_strong_target_ccn2_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN2",
              "CTGF"
            ]
          },
          {
            "name": "hippo_pathway_weighted",
            "role": "additive",
            "field": "$.hippo_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "tgf_beta_pathway_weighted",
            "role": "additive",
            "field": "$.tgf_beta_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "deg_direction_batch_effect_uncorrected",
            "role": "hard_fail",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_dominated_global",
              "up_dominated_in_chosen_subset"
            ]
          },
          {
            "name": "silent_panel_dropout_paired_with_directional_pathway_claim",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "hippo_pathway_direction": {
                "op": "in",
                "values": [
                  "hippo_pathway_upregulated",
                  "hippo_pathway_unchanged",
                  "wnt_pathway_specifically_perturbed",
                  "stress_response_pathway_dominantly_engaged",
                  "no_pathway_signal_above_confound",
                  "yap_tead_direct_target_panel_downregulated",
                  "yap_tead_panel_dependent_direction",
                  "hippo_pathway_downregulated"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-4-pi",
      "modelName": "gpt-5.4",
      "harness": "pi",
      "task": "gse155381_q5_chimera_monkey_perturbation",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 1141,
      "cost": 0.8002695000000003,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "deg_direction_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "comparison_subset_choice_weighted",
            "role": "additive",
            "field": "$.comparison_subset_choice",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "panel_resolved_strong_target_ccn1_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN1",
              "CYR61"
            ]
          },
          {
            "name": "panel_resolved_strong_target_ccn2_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN2",
              "CTGF"
            ]
          },
          {
            "name": "hippo_pathway_weighted",
            "role": "additive",
            "field": "$.hippo_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "tgf_beta_pathway_weighted",
            "role": "additive",
            "field": "$.tgf_beta_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "deg_direction_batch_effect_uncorrected",
            "role": "hard_fail",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_dominated_global",
              "up_dominated_in_chosen_subset"
            ]
          },
          {
            "name": "silent_panel_dropout_paired_with_directional_pathway_claim",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "hippo_pathway_direction": {
                "op": "in",
                "values": [
                  "hippo_pathway_upregulated",
                  "hippo_pathway_unchanged",
                  "wnt_pathway_specifically_perturbed",
                  "stress_response_pathway_dominantly_engaged",
                  "no_pathway_signal_above_confound",
                  "yap_tead_direct_target_panel_downregulated",
                  "yap_tead_panel_dependent_direction",
                  "hippo_pathway_downregulated"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-4-pi",
      "modelName": "gpt-5.4",
      "harness": "pi",
      "task": "gse155381_q5_chimera_monkey_perturbation",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1255,
      "cost": 1.2539569999999998,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "deg_direction_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "comparison_subset_choice_weighted",
            "role": "additive",
            "field": "$.comparison_subset_choice",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "panel_resolved_strong_target_ccn1_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN1",
              "CYR61"
            ]
          },
          {
            "name": "panel_resolved_strong_target_ccn2_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN2",
              "CTGF"
            ]
          },
          {
            "name": "hippo_pathway_weighted",
            "role": "additive",
            "field": "$.hippo_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "tgf_beta_pathway_weighted",
            "role": "additive",
            "field": "$.tgf_beta_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "deg_direction_batch_effect_uncorrected",
            "role": "hard_fail",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_dominated_global",
              "up_dominated_in_chosen_subset"
            ]
          },
          {
            "name": "silent_panel_dropout_paired_with_directional_pathway_claim",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "hippo_pathway_direction": {
                "op": "in",
                "values": [
                  "hippo_pathway_upregulated",
                  "hippo_pathway_unchanged",
                  "wnt_pathway_specifically_perturbed",
                  "stress_response_pathway_dominantly_engaged",
                  "no_pathway_signal_above_confound",
                  "yap_tead_direct_target_panel_downregulated",
                  "yap_tead_panel_dependent_direction",
                  "hippo_pathway_downregulated"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-5-codex",
      "modelName": "gpt-5.5",
      "harness": "openai-codex",
      "task": "gse155381_q5_chimera_monkey_perturbation",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 624,
      "cost": 1.5475720000000002,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "deg_direction_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "comparison_subset_choice_weighted",
            "role": "additive",
            "field": "$.comparison_subset_choice",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "panel_resolved_strong_target_ccn1_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN1",
              "CYR61"
            ]
          },
          {
            "name": "panel_resolved_strong_target_ccn2_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN2",
              "CTGF"
            ]
          },
          {
            "name": "hippo_pathway_weighted",
            "role": "additive",
            "field": "$.hippo_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "tgf_beta_pathway_weighted",
            "role": "additive",
            "field": "$.tgf_beta_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "deg_direction_batch_effect_uncorrected",
            "role": "hard_fail",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_dominated_global",
              "up_dominated_in_chosen_subset"
            ]
          },
          {
            "name": "silent_panel_dropout_paired_with_directional_pathway_claim",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "hippo_pathway_direction": {
                "op": "in",
                "values": [
                  "hippo_pathway_upregulated",
                  "hippo_pathway_unchanged",
                  "wnt_pathway_specifically_perturbed",
                  "stress_response_pathway_dominantly_engaged",
                  "no_pathway_signal_above_confound",
                  "yap_tead_direct_target_panel_downregulated",
                  "yap_tead_panel_dependent_direction",
                  "hippo_pathway_downregulated"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-5-codex",
      "modelName": "gpt-5.5",
      "harness": "openai-codex",
      "task": "gse155381_q5_chimera_monkey_perturbation",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 1140,
      "cost": 3.0247399999999995,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "deg_direction_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "comparison_subset_choice_weighted",
            "role": "additive",
            "field": "$.comparison_subset_choice",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "panel_resolved_strong_target_ccn1_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN1",
              "CYR61"
            ]
          },
          {
            "name": "panel_resolved_strong_target_ccn2_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN2",
              "CTGF"
            ]
          },
          {
            "name": "hippo_pathway_weighted",
            "role": "additive",
            "field": "$.hippo_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "tgf_beta_pathway_weighted",
            "role": "additive",
            "field": "$.tgf_beta_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "deg_direction_batch_effect_uncorrected",
            "role": "hard_fail",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_dominated_global",
              "up_dominated_in_chosen_subset"
            ]
          },
          {
            "name": "silent_panel_dropout_paired_with_directional_pathway_claim",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "hippo_pathway_direction": {
                "op": "in",
                "values": [
                  "hippo_pathway_upregulated",
                  "hippo_pathway_unchanged",
                  "wnt_pathway_specifically_perturbed",
                  "stress_response_pathway_dominantly_engaged",
                  "no_pathway_signal_above_confound",
                  "yap_tead_direct_target_panel_downregulated",
                  "yap_tead_panel_dependent_direction",
                  "hippo_pathway_downregulated"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-5-codex",
      "modelName": "gpt-5.5",
      "harness": "openai-codex",
      "task": "gse155381_q5_chimera_monkey_perturbation",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 604,
      "cost": 2.10576,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "deg_direction_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "comparison_subset_choice_weighted",
            "role": "additive",
            "field": "$.comparison_subset_choice",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "panel_resolved_strong_target_ccn1_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN1",
              "CYR61"
            ]
          },
          {
            "name": "panel_resolved_strong_target_ccn2_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN2",
              "CTGF"
            ]
          },
          {
            "name": "hippo_pathway_weighted",
            "role": "additive",
            "field": "$.hippo_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "tgf_beta_pathway_weighted",
            "role": "additive",
            "field": "$.tgf_beta_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "deg_direction_batch_effect_uncorrected",
            "role": "hard_fail",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_dominated_global",
              "up_dominated_in_chosen_subset"
            ]
          },
          {
            "name": "silent_panel_dropout_paired_with_directional_pathway_claim",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "hippo_pathway_direction": {
                "op": "in",
                "values": [
                  "hippo_pathway_upregulated",
                  "hippo_pathway_unchanged",
                  "wnt_pathway_specifically_perturbed",
                  "stress_response_pathway_dominantly_engaged",
                  "no_pathway_signal_above_confound",
                  "yap_tead_direct_target_panel_downregulated",
                  "yap_tead_panel_dependent_direction",
                  "hippo_pathway_downregulated"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-5-pi",
      "modelName": "gpt-5.5",
      "harness": "pi",
      "task": "gse155381_q5_chimera_monkey_perturbation",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 726,
      "cost": 1.3201770000000002,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "deg_direction_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "comparison_subset_choice_weighted",
            "role": "additive",
            "field": "$.comparison_subset_choice",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "panel_resolved_strong_target_ccn1_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN1",
              "CYR61"
            ]
          },
          {
            "name": "panel_resolved_strong_target_ccn2_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN2",
              "CTGF"
            ]
          },
          {
            "name": "hippo_pathway_weighted",
            "role": "additive",
            "field": "$.hippo_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "tgf_beta_pathway_weighted",
            "role": "additive",
            "field": "$.tgf_beta_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "deg_direction_batch_effect_uncorrected",
            "role": "hard_fail",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_dominated_global",
              "up_dominated_in_chosen_subset"
            ]
          },
          {
            "name": "silent_panel_dropout_paired_with_directional_pathway_claim",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "hippo_pathway_direction": {
                "op": "in",
                "values": [
                  "hippo_pathway_upregulated",
                  "hippo_pathway_unchanged",
                  "wnt_pathway_specifically_perturbed",
                  "stress_response_pathway_dominantly_engaged",
                  "no_pathway_signal_above_confound",
                  "yap_tead_direct_target_panel_downregulated",
                  "yap_tead_panel_dependent_direction",
                  "hippo_pathway_downregulated"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-5-pi",
      "modelName": "gpt-5.5",
      "harness": "pi",
      "task": "gse155381_q5_chimera_monkey_perturbation",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 1225,
      "cost": 3.1772650000000002,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "deg_direction_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "comparison_subset_choice_weighted",
            "role": "additive",
            "field": "$.comparison_subset_choice",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "panel_resolved_strong_target_ccn1_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN1",
              "CYR61"
            ]
          },
          {
            "name": "panel_resolved_strong_target_ccn2_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN2",
              "CTGF"
            ]
          },
          {
            "name": "hippo_pathway_weighted",
            "role": "additive",
            "field": "$.hippo_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "tgf_beta_pathway_weighted",
            "role": "additive",
            "field": "$.tgf_beta_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "deg_direction_batch_effect_uncorrected",
            "role": "hard_fail",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_dominated_global",
              "up_dominated_in_chosen_subset"
            ]
          },
          {
            "name": "silent_panel_dropout_paired_with_directional_pathway_claim",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "hippo_pathway_direction": {
                "op": "in",
                "values": [
                  "hippo_pathway_upregulated",
                  "hippo_pathway_unchanged",
                  "wnt_pathway_specifically_perturbed",
                  "stress_response_pathway_dominantly_engaged",
                  "no_pathway_signal_above_confound",
                  "yap_tead_direct_target_panel_downregulated",
                  "yap_tead_panel_dependent_direction",
                  "hippo_pathway_downregulated"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-5-pi",
      "modelName": "gpt-5.5",
      "harness": "pi",
      "task": "gse155381_q5_chimera_monkey_perturbation",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 847,
      "cost": 1.2355410000000002,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "deg_direction_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "comparison_subset_choice_weighted",
            "role": "additive",
            "field": "$.comparison_subset_choice",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "panel_resolved_strong_target_ccn1_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN1",
              "CYR61"
            ]
          },
          {
            "name": "panel_resolved_strong_target_ccn2_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN2",
              "CTGF"
            ]
          },
          {
            "name": "hippo_pathway_weighted",
            "role": "additive",
            "field": "$.hippo_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "tgf_beta_pathway_weighted",
            "role": "additive",
            "field": "$.tgf_beta_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "deg_direction_batch_effect_uncorrected",
            "role": "hard_fail",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_dominated_global",
              "up_dominated_in_chosen_subset"
            ]
          },
          {
            "name": "silent_panel_dropout_paired_with_directional_pathway_claim",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "hippo_pathway_direction": {
                "op": "in",
                "values": [
                  "hippo_pathway_upregulated",
                  "hippo_pathway_unchanged",
                  "wnt_pathway_specifically_perturbed",
                  "stress_response_pathway_dominantly_engaged",
                  "no_pathway_signal_above_confound",
                  "yap_tead_direct_target_panel_downregulated",
                  "yap_tead_panel_dependent_direction",
                  "hippo_pathway_downregulated"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-20-beta-reasoning-pi",
      "modelName": "grok-4.20-beta-0309-reasoning",
      "harness": "pi",
      "task": "gse155381_q5_chimera_monkey_perturbation",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 105,
      "cost": 0.029117350000000004,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "deg_direction_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "comparison_subset_choice_weighted",
            "role": "additive",
            "field": "$.comparison_subset_choice",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "panel_resolved_strong_target_ccn1_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN1",
              "CYR61"
            ]
          },
          {
            "name": "panel_resolved_strong_target_ccn2_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN2",
              "CTGF"
            ]
          },
          {
            "name": "hippo_pathway_weighted",
            "role": "additive",
            "field": "$.hippo_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "tgf_beta_pathway_weighted",
            "role": "additive",
            "field": "$.tgf_beta_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "deg_direction_batch_effect_uncorrected",
            "role": "hard_fail",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_dominated_global",
              "up_dominated_in_chosen_subset"
            ]
          },
          {
            "name": "silent_panel_dropout_paired_with_directional_pathway_claim",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "hippo_pathway_direction": {
                "op": "in",
                "values": [
                  "hippo_pathway_upregulated",
                  "hippo_pathway_unchanged",
                  "wnt_pathway_specifically_perturbed",
                  "stress_response_pathway_dominantly_engaged",
                  "no_pathway_signal_above_confound",
                  "yap_tead_direct_target_panel_downregulated",
                  "yap_tead_panel_dependent_direction",
                  "hippo_pathway_downregulated"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-20-beta-reasoning-pi",
      "modelName": "grok-4.20-beta-0309-reasoning",
      "harness": "pi",
      "task": "gse155381_q5_chimera_monkey_perturbation",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 128,
      "cost": 0.08489105000000001,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "deg_direction_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "comparison_subset_choice_weighted",
            "role": "additive",
            "field": "$.comparison_subset_choice",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "panel_resolved_strong_target_ccn1_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN1",
              "CYR61"
            ]
          },
          {
            "name": "panel_resolved_strong_target_ccn2_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN2",
              "CTGF"
            ]
          },
          {
            "name": "hippo_pathway_weighted",
            "role": "additive",
            "field": "$.hippo_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "tgf_beta_pathway_weighted",
            "role": "additive",
            "field": "$.tgf_beta_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "deg_direction_batch_effect_uncorrected",
            "role": "hard_fail",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_dominated_global",
              "up_dominated_in_chosen_subset"
            ]
          },
          {
            "name": "silent_panel_dropout_paired_with_directional_pathway_claim",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "hippo_pathway_direction": {
                "op": "in",
                "values": [
                  "hippo_pathway_upregulated",
                  "hippo_pathway_unchanged",
                  "wnt_pathway_specifically_perturbed",
                  "stress_response_pathway_dominantly_engaged",
                  "no_pathway_signal_above_confound",
                  "yap_tead_direct_target_panel_downregulated",
                  "yap_tead_panel_dependent_direction",
                  "hippo_pathway_downregulated"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-20-beta-reasoning-pi",
      "modelName": "grok-4.20-beta-0309-reasoning",
      "harness": "pi",
      "task": "gse155381_q5_chimera_monkey_perturbation",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 111,
      "cost": 0.09708420000000001,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "deg_direction_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "comparison_subset_choice_weighted",
            "role": "additive",
            "field": "$.comparison_subset_choice",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "panel_resolved_strong_target_ccn1_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN1",
              "CYR61"
            ]
          },
          {
            "name": "panel_resolved_strong_target_ccn2_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN2",
              "CTGF"
            ]
          },
          {
            "name": "hippo_pathway_weighted",
            "role": "additive",
            "field": "$.hippo_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "tgf_beta_pathway_weighted",
            "role": "additive",
            "field": "$.tgf_beta_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "deg_direction_batch_effect_uncorrected",
            "role": "hard_fail",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_dominated_global",
              "up_dominated_in_chosen_subset"
            ]
          },
          {
            "name": "silent_panel_dropout_paired_with_directional_pathway_claim",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "hippo_pathway_direction": {
                "op": "in",
                "values": [
                  "hippo_pathway_upregulated",
                  "hippo_pathway_unchanged",
                  "wnt_pathway_specifically_perturbed",
                  "stress_response_pathway_dominantly_engaged",
                  "no_pathway_signal_above_confound",
                  "yap_tead_direct_target_panel_downregulated",
                  "yap_tead_panel_dependent_direction",
                  "hippo_pathway_downregulated"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-3-pi",
      "modelName": "grok-4.3",
      "harness": "pi",
      "task": "gse155381_q5_chimera_monkey_perturbation",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 60,
      "cost": 0.020878050000000002,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "deg_direction_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "comparison_subset_choice_weighted",
            "role": "additive",
            "field": "$.comparison_subset_choice",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "panel_resolved_strong_target_ccn1_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN1",
              "CYR61"
            ]
          },
          {
            "name": "panel_resolved_strong_target_ccn2_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN2",
              "CTGF"
            ]
          },
          {
            "name": "hippo_pathway_weighted",
            "role": "additive",
            "field": "$.hippo_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "tgf_beta_pathway_weighted",
            "role": "additive",
            "field": "$.tgf_beta_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "deg_direction_batch_effect_uncorrected",
            "role": "hard_fail",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_dominated_global",
              "up_dominated_in_chosen_subset"
            ]
          },
          {
            "name": "silent_panel_dropout_paired_with_directional_pathway_claim",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "hippo_pathway_direction": {
                "op": "in",
                "values": [
                  "hippo_pathway_upregulated",
                  "hippo_pathway_unchanged",
                  "wnt_pathway_specifically_perturbed",
                  "stress_response_pathway_dominantly_engaged",
                  "no_pathway_signal_above_confound",
                  "yap_tead_direct_target_panel_downregulated",
                  "yap_tead_panel_dependent_direction",
                  "hippo_pathway_downregulated"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-3-pi",
      "modelName": "grok-4.3",
      "harness": "pi",
      "task": "gse155381_q5_chimera_monkey_perturbation",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 209,
      "cost": 0.022364050000000003,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "deg_direction_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "comparison_subset_choice_weighted",
            "role": "additive",
            "field": "$.comparison_subset_choice",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "panel_resolved_strong_target_ccn1_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN1",
              "CYR61"
            ]
          },
          {
            "name": "panel_resolved_strong_target_ccn2_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN2",
              "CTGF"
            ]
          },
          {
            "name": "hippo_pathway_weighted",
            "role": "additive",
            "field": "$.hippo_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "tgf_beta_pathway_weighted",
            "role": "additive",
            "field": "$.tgf_beta_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "deg_direction_batch_effect_uncorrected",
            "role": "hard_fail",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_dominated_global",
              "up_dominated_in_chosen_subset"
            ]
          },
          {
            "name": "silent_panel_dropout_paired_with_directional_pathway_claim",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "hippo_pathway_direction": {
                "op": "in",
                "values": [
                  "hippo_pathway_upregulated",
                  "hippo_pathway_unchanged",
                  "wnt_pathway_specifically_perturbed",
                  "stress_response_pathway_dominantly_engaged",
                  "no_pathway_signal_above_confound",
                  "yap_tead_direct_target_panel_downregulated",
                  "yap_tead_panel_dependent_direction",
                  "hippo_pathway_downregulated"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-3-pi",
      "modelName": "grok-4.3",
      "harness": "pi",
      "task": "gse155381_q5_chimera_monkey_perturbation",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 93,
      "cost": 0.024933350000000003,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "deg_direction_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "comparison_subset_choice_weighted",
            "role": "additive",
            "field": "$.comparison_subset_choice",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "panel_resolved_strong_target_ccn1_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN1",
              "CYR61"
            ]
          },
          {
            "name": "panel_resolved_strong_target_ccn2_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN2",
              "CTGF"
            ]
          },
          {
            "name": "hippo_pathway_weighted",
            "role": "additive",
            "field": "$.hippo_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "tgf_beta_pathway_weighted",
            "role": "additive",
            "field": "$.tgf_beta_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "deg_direction_batch_effect_uncorrected",
            "role": "hard_fail",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_dominated_global",
              "up_dominated_in_chosen_subset"
            ]
          },
          {
            "name": "silent_panel_dropout_paired_with_directional_pathway_claim",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "hippo_pathway_direction": {
                "op": "in",
                "values": [
                  "hippo_pathway_upregulated",
                  "hippo_pathway_unchanged",
                  "wnt_pathway_specifically_perturbed",
                  "stress_response_pathway_dominantly_engaged",
                  "no_pathway_signal_above_confound",
                  "yap_tead_direct_target_panel_downregulated",
                  "yap_tead_panel_dependent_direction",
                  "hippo_pathway_downregulated"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-kimi-k2p6-pi",
      "modelName": "kimi-k2p6",
      "harness": "pi",
      "task": "gse155381_q5_chimera_monkey_perturbation",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 1102,
      "cost": 0.37494353,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "deg_direction_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "comparison_subset_choice_weighted",
            "role": "additive",
            "field": "$.comparison_subset_choice",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "panel_resolved_strong_target_ccn1_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN1",
              "CYR61"
            ]
          },
          {
            "name": "panel_resolved_strong_target_ccn2_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN2",
              "CTGF"
            ]
          },
          {
            "name": "hippo_pathway_weighted",
            "role": "additive",
            "field": "$.hippo_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "tgf_beta_pathway_weighted",
            "role": "additive",
            "field": "$.tgf_beta_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "deg_direction_batch_effect_uncorrected",
            "role": "hard_fail",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_dominated_global",
              "up_dominated_in_chosen_subset"
            ]
          },
          {
            "name": "silent_panel_dropout_paired_with_directional_pathway_claim",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "hippo_pathway_direction": {
                "op": "in",
                "values": [
                  "hippo_pathway_upregulated",
                  "hippo_pathway_unchanged",
                  "wnt_pathway_specifically_perturbed",
                  "stress_response_pathway_dominantly_engaged",
                  "no_pathway_signal_above_confound",
                  "yap_tead_direct_target_panel_downregulated",
                  "yap_tead_panel_dependent_direction",
                  "hippo_pathway_downregulated"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-kimi-k2p6-pi",
      "modelName": "kimi-k2p6",
      "harness": "pi",
      "task": "gse155381_q5_chimera_monkey_perturbation",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 1177,
      "cost": 0.43445237,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "deg_direction_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "comparison_subset_choice_weighted",
            "role": "additive",
            "field": "$.comparison_subset_choice",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "panel_resolved_strong_target_ccn1_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN1",
              "CYR61"
            ]
          },
          {
            "name": "panel_resolved_strong_target_ccn2_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN2",
              "CTGF"
            ]
          },
          {
            "name": "hippo_pathway_weighted",
            "role": "additive",
            "field": "$.hippo_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "tgf_beta_pathway_weighted",
            "role": "additive",
            "field": "$.tgf_beta_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "deg_direction_batch_effect_uncorrected",
            "role": "hard_fail",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_dominated_global",
              "up_dominated_in_chosen_subset"
            ]
          },
          {
            "name": "silent_panel_dropout_paired_with_directional_pathway_claim",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "hippo_pathway_direction": {
                "op": "in",
                "values": [
                  "hippo_pathway_upregulated",
                  "hippo_pathway_unchanged",
                  "wnt_pathway_specifically_perturbed",
                  "stress_response_pathway_dominantly_engaged",
                  "no_pathway_signal_above_confound",
                  "yap_tead_direct_target_panel_downregulated",
                  "yap_tead_panel_dependent_direction",
                  "hippo_pathway_downregulated"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-kimi-k2p6-pi",
      "modelName": "kimi-k2p6",
      "harness": "pi",
      "task": "gse155381_q5_chimera_monkey_perturbation",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 2350,
      "cost": 0.20110382999999998,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "deg_direction_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "comparison_subset_choice_weighted",
            "role": "additive",
            "field": "$.comparison_subset_choice",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "panel_resolved_strong_target_ccn1_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN1",
              "CYR61"
            ]
          },
          {
            "name": "panel_resolved_strong_target_ccn2_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN2",
              "CTGF"
            ]
          },
          {
            "name": "hippo_pathway_weighted",
            "role": "additive",
            "field": "$.hippo_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "tgf_beta_pathway_weighted",
            "role": "additive",
            "field": "$.tgf_beta_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "deg_direction_batch_effect_uncorrected",
            "role": "hard_fail",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_dominated_global",
              "up_dominated_in_chosen_subset"
            ]
          },
          {
            "name": "silent_panel_dropout_paired_with_directional_pathway_claim",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "hippo_pathway_direction": {
                "op": "in",
                "values": [
                  "hippo_pathway_upregulated",
                  "hippo_pathway_unchanged",
                  "wnt_pathway_specifically_perturbed",
                  "stress_response_pathway_dominantly_engaged",
                  "no_pathway_signal_above_confound",
                  "yap_tead_direct_target_panel_downregulated",
                  "yap_tead_panel_dependent_direction",
                  "hippo_pathway_downregulated"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-6-claude-code",
      "modelName": "claude-opus-4-6",
      "harness": "claude-code",
      "task": "gse155381_q7_xspecies_epi_communication",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 1,
      "passed": false,
      "score": 3.75,
      "durationS": 1018.31,
      "cost": 2.1987712499999996,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "richness_weighted",
            "role": "additive",
            "field": "$.xspecies_signaling_richness",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_weighted",
            "role": "additive",
            "field": "$.dominant_signaling_axis",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_wnt_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "WNT_FZD_LRP_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_fgf_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "FGF_FGFR_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_notch_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "NOTCH_DLL_JAG_family"
                ]
              }
            }
          },
          {
            "name": "signaling_directionality_weighted",
            "role": "additive",
            "field": "$.signaling_directionality",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "top_signal_interaction_class_weighted",
            "role": "additive",
            "field": "$.top_signal_interaction_class",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          }
        ]
      },
      "reasoning": "all_of [pass_rule=score_threshold]: FAIL\n  + richness_weighted: passed=True, score=1.5\n  + dominant_axis_weighted: passed=True, score=0.25\n  + signaling_directionality_weighted: passed=True, score=1.0\n  + top_signal_interaction_class_weighted: passed=True, score=1.0\n  + hard_fail 'dominant_axis_wnt_unsupported_by_own_pairs': not triggered\n  + hard_fail 'dominant_axis_fgf_unsupported_by_own_pairs': not triggered\n  + hard_fail 'dominant_axis_notch_unsupported_by_own_pairs': not triggered",
      "metrics": {
        "pass_rule": "score_threshold",
        "scoring_count": 4,
        "scoring_passed": 4,
        "scoring_total_score": 3.75,
        "hard_fail_triggered": []
      },
      "fieldScores": {
        "richness_weighted": 1.5,
        "dominant_axis_weighted": 0.25,
        "signaling_directionality_weighted": 1,
        "top_signal_interaction_class_weighted": 1
      },
      "graderChecks": [
        {
          "name": "richness_weighted",
          "kind": "gate",
          "passed": true,
          "score": 1.5
        },
        {
          "name": "dominant_axis_weighted",
          "kind": "gate",
          "passed": true,
          "score": 0.25
        },
        {
          "name": "signaling_directionality_weighted",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "top_signal_interaction_class_weighted",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "dominant_axis_wnt_unsupported_by_own_pairs",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "dominant_axis_fgf_unsupported_by_own_pairs",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "dominant_axis_notch_unsupported_by_own_pairs",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "dominant_axis_support": [
          "FBLN1",
          "ITGB1",
          "LAMA1",
          "LAMA2",
          "RPSA",
          "VCAN",
          "COL4A1",
          "CD47",
          "LUM",
          "LAMB2",
          "FN1",
          "ITGA5"
        ],
        "dominant_signaling_axis": "ECM_collagen_integrin_family",
        "signaling_directionality": "substantial_signal_in_both_directions",
        "top_signal_interaction_class": "protein_ligand_receptor_dominant",
        "xspecies_signaling_richness": "many_significant_pairs_across_multiple_signalling_families"
      },
      "nTurns": 27,
      "checkpointSignals": {
        "clustering": true,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": true,
        "nerveJointAnchor": false,
        "multiomeAssembly": false,
        "depthNormalizedAtac": false,
        "regulonInference": false,
        "cd8SubsetIntegration": false,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": true,
        "orthologProjection": true,
        "epiSubsetSelection": true,
        "bidirectionalLrTest": true,
        "enrichmentFamilyRanking": true,
        "mousePseudobulkDesign": false,
        "cancerEpithelialDenominator": false,
        "geneLevelDe": false,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "trajectoryFile": "scbench-long/trajectories/gse155381_q7_xspecies_epi_communication__claude-opus-4-6__claude-code__r1.json",
      "runId": "claude-opus-4-6__claude-code__gse155381_q7_xspecies_epi_communication__r1",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-opus-4-6-claude-code",
      "modelName": "claude-opus-4-6",
      "harness": "claude-code",
      "task": "gse155381_q7_xspecies_epi_communication",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 2,
      "passed": false,
      "score": 3.75,
      "durationS": 620.27,
      "cost": 1.447886,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "richness_weighted",
            "role": "additive",
            "field": "$.xspecies_signaling_richness",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_weighted",
            "role": "additive",
            "field": "$.dominant_signaling_axis",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_wnt_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "WNT_FZD_LRP_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_fgf_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "FGF_FGFR_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_notch_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "NOTCH_DLL_JAG_family"
                ]
              }
            }
          },
          {
            "name": "signaling_directionality_weighted",
            "role": "additive",
            "field": "$.signaling_directionality",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "top_signal_interaction_class_weighted",
            "role": "additive",
            "field": "$.top_signal_interaction_class",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          }
        ]
      },
      "reasoning": "all_of [pass_rule=score_threshold]: FAIL\n  + richness_weighted: passed=True, score=1.5\n  + dominant_axis_weighted: passed=True, score=0.25\n  + signaling_directionality_weighted: passed=True, score=1.0\n  + top_signal_interaction_class_weighted: passed=True, score=1.0\n  + hard_fail 'dominant_axis_wnt_unsupported_by_own_pairs': not triggered\n  + hard_fail 'dominant_axis_fgf_unsupported_by_own_pairs': not triggered\n  + hard_fail 'dominant_axis_notch_unsupported_by_own_pairs': not triggered",
      "metrics": {
        "pass_rule": "score_threshold",
        "scoring_count": 4,
        "scoring_passed": 4,
        "scoring_total_score": 3.75,
        "hard_fail_triggered": []
      },
      "fieldScores": {
        "richness_weighted": 1.5,
        "dominant_axis_weighted": 0.25,
        "signaling_directionality_weighted": 1,
        "top_signal_interaction_class_weighted": 1
      },
      "graderChecks": [
        {
          "name": "richness_weighted",
          "kind": "gate",
          "passed": true,
          "score": 1.5
        },
        {
          "name": "dominant_axis_weighted",
          "kind": "gate",
          "passed": true,
          "score": 0.25
        },
        {
          "name": "signaling_directionality_weighted",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "top_signal_interaction_class_weighted",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "dominant_axis_wnt_unsupported_by_own_pairs",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "dominant_axis_fgf_unsupported_by_own_pairs",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "dominant_axis_notch_unsupported_by_own_pairs",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "dominant_axis_support": [
          "FN1",
          "COL3A1",
          "COL11A1",
          "LAMA4",
          "LAMB1",
          "LAMC1",
          "THBS2",
          "MFGE8",
          "SPP1",
          "COL4A1",
          "DDR1",
          "ITGA6",
          "ITGB1",
          "SDC2",
          "ITGAV",
          "CD47",
          "ITGA5"
        ],
        "dominant_signaling_axis": "ECM_collagen_integrin_family",
        "signaling_directionality": "substantial_signal_in_both_directions",
        "top_signal_interaction_class": "protein_ligand_receptor_dominant",
        "xspecies_signaling_richness": "many_significant_pairs_across_multiple_signalling_families"
      },
      "nTurns": 32,
      "checkpointSignals": {
        "clustering": true,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": true,
        "nerveJointAnchor": false,
        "multiomeAssembly": false,
        "depthNormalizedAtac": false,
        "regulonInference": false,
        "cd8SubsetIntegration": false,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": true,
        "orthologProjection": true,
        "epiSubsetSelection": true,
        "bidirectionalLrTest": true,
        "enrichmentFamilyRanking": true,
        "mousePseudobulkDesign": false,
        "cancerEpithelialDenominator": true,
        "geneLevelDe": false,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "trajectoryFile": "scbench-long/trajectories/gse155381_q7_xspecies_epi_communication__claude-opus-4-6__claude-code__r2.json",
      "runId": "claude-opus-4-6__claude-code__gse155381_q7_xspecies_epi_communication__r2",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-opus-4-6-claude-code",
      "modelName": "claude-opus-4-6",
      "harness": "claude-code",
      "task": "gse155381_q7_xspecies_epi_communication",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 3,
      "passed": false,
      "score": 4.25,
      "durationS": 1076.39,
      "cost": 2.12488625,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "richness_weighted",
            "role": "additive",
            "field": "$.xspecies_signaling_richness",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_weighted",
            "role": "additive",
            "field": "$.dominant_signaling_axis",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_wnt_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "WNT_FZD_LRP_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_fgf_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "FGF_FGFR_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_notch_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "NOTCH_DLL_JAG_family"
                ]
              }
            }
          },
          {
            "name": "signaling_directionality_weighted",
            "role": "additive",
            "field": "$.signaling_directionality",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "top_signal_interaction_class_weighted",
            "role": "additive",
            "field": "$.top_signal_interaction_class",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          }
        ]
      },
      "reasoning": "all_of [pass_rule=score_threshold]: FAIL\n  + richness_weighted: passed=True, score=1.5\n  + dominant_axis_weighted: passed=True, score=0.75\n  + signaling_directionality_weighted: passed=True, score=1.0\n  + top_signal_interaction_class_weighted: passed=True, score=1.0\n  + hard_fail 'dominant_axis_wnt_unsupported_by_own_pairs': not triggered\n  + hard_fail 'dominant_axis_fgf_unsupported_by_own_pairs': not triggered\n  + hard_fail 'dominant_axis_notch_unsupported_by_own_pairs': not triggered",
      "metrics": {
        "pass_rule": "score_threshold",
        "scoring_count": 4,
        "scoring_passed": 4,
        "scoring_total_score": 4.25,
        "hard_fail_triggered": []
      },
      "fieldScores": {
        "richness_weighted": 1.5,
        "dominant_axis_weighted": 0.75,
        "signaling_directionality_weighted": 1,
        "top_signal_interaction_class_weighted": 1
      },
      "graderChecks": [
        {
          "name": "richness_weighted",
          "kind": "gate",
          "passed": true,
          "score": 1.5
        },
        {
          "name": "dominant_axis_weighted",
          "kind": "gate",
          "passed": true,
          "score": 0.75
        },
        {
          "name": "signaling_directionality_weighted",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "top_signal_interaction_class_weighted",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "dominant_axis_wnt_unsupported_by_own_pairs",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "dominant_axis_fgf_unsupported_by_own_pairs",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "dominant_axis_notch_unsupported_by_own_pairs",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "dominant_axis_support": [
          "MDK",
          "NCL",
          "SDC2",
          "PTPRZ1",
          "LRP1",
          "SORL1",
          "ITGA6",
          "ITGB1"
        ],
        "dominant_signaling_axis": "secreted_growth_factor_RTK_family",
        "signaling_directionality": "substantial_signal_in_both_directions",
        "top_signal_interaction_class": "protein_ligand_receptor_dominant",
        "xspecies_signaling_richness": "many_significant_pairs_across_multiple_signalling_families"
      },
      "nTurns": 40,
      "checkpointSignals": {
        "clustering": true,
        "markerAnnotation": true,
        "scoreBasedAnnotation": true,
        "normalization": true,
        "nerveJointAnchor": false,
        "multiomeAssembly": false,
        "depthNormalizedAtac": false,
        "regulonInference": false,
        "cd8SubsetIntegration": false,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": true,
        "orthologProjection": true,
        "epiSubsetSelection": true,
        "bidirectionalLrTest": true,
        "enrichmentFamilyRanking": true,
        "mousePseudobulkDesign": false,
        "cancerEpithelialDenominator": true,
        "geneLevelDe": false,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "trajectoryFile": "scbench-long/trajectories/gse155381_q7_xspecies_epi_communication__claude-opus-4-6__claude-code__r3.json",
      "runId": "claude-opus-4-6__claude-code__gse155381_q7_xspecies_epi_communication__r3",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-opus-4-6-pi",
      "modelName": "claude-opus-4-6",
      "harness": "pi",
      "task": "gse155381_q7_xspecies_epi_communication",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 1,
      "passed": false,
      "score": 3.25,
      "durationS": 1137.25,
      "cost": 1.56317425,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "richness_weighted",
            "role": "additive",
            "field": "$.xspecies_signaling_richness",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_weighted",
            "role": "additive",
            "field": "$.dominant_signaling_axis",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_wnt_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "WNT_FZD_LRP_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_fgf_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "FGF_FGFR_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_notch_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "NOTCH_DLL_JAG_family"
                ]
              }
            }
          },
          {
            "name": "signaling_directionality_weighted",
            "role": "additive",
            "field": "$.signaling_directionality",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "top_signal_interaction_class_weighted",
            "role": "additive",
            "field": "$.top_signal_interaction_class",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          }
        ]
      },
      "reasoning": "all_of [pass_rule=score_threshold]: FAIL\n  + richness_weighted: passed=True, score=1.5\n  + dominant_axis_weighted: passed=True, score=0.25\n  + signaling_directionality_weighted: passed=True, score=0.5\n  + top_signal_interaction_class_weighted: passed=True, score=1.0\n  + hard_fail 'dominant_axis_wnt_unsupported_by_own_pairs': not triggered\n  + hard_fail 'dominant_axis_fgf_unsupported_by_own_pairs': not triggered\n  + hard_fail 'dominant_axis_notch_unsupported_by_own_pairs': not triggered",
      "metrics": {
        "pass_rule": "score_threshold",
        "scoring_count": 4,
        "scoring_passed": 4,
        "scoring_total_score": 3.25,
        "hard_fail_triggered": []
      },
      "fieldScores": {
        "richness_weighted": 1.5,
        "dominant_axis_weighted": 0.25,
        "signaling_directionality_weighted": 0.5,
        "top_signal_interaction_class_weighted": 1
      },
      "graderChecks": [
        {
          "name": "richness_weighted",
          "kind": "gate",
          "passed": true,
          "score": 1.5
        },
        {
          "name": "dominant_axis_weighted",
          "kind": "gate",
          "passed": true,
          "score": 0.25
        },
        {
          "name": "signaling_directionality_weighted",
          "kind": "gate",
          "passed": true,
          "score": 0.5
        },
        {
          "name": "top_signal_interaction_class_weighted",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "dominant_axis_wnt_unsupported_by_own_pairs",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "dominant_axis_fgf_unsupported_by_own_pairs",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "dominant_axis_notch_unsupported_by_own_pairs",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "dominant_axis_support": [
          "FN1",
          "ITGA6",
          "SDC2",
          "VTN",
          "CD47",
          "COL4A5",
          "SPP1",
          "CD44",
          "COMP",
          "COL9A2",
          "APOE",
          "F13A1",
          "ITGA4"
        ],
        "dominant_signaling_axis": "ECM_collagen_integrin_family",
        "signaling_directionality": "predominantly_chimera_monkey_to_chimera_human",
        "top_signal_interaction_class": "protein_ligand_receptor_dominant",
        "xspecies_signaling_richness": "many_significant_pairs_across_multiple_signalling_families"
      },
      "nTurns": 37,
      "checkpointSignals": {
        "clustering": true,
        "markerAnnotation": true,
        "scoreBasedAnnotation": true,
        "normalization": true,
        "nerveJointAnchor": false,
        "multiomeAssembly": false,
        "depthNormalizedAtac": false,
        "regulonInference": false,
        "cd8SubsetIntegration": false,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": true,
        "orthologProjection": true,
        "epiSubsetSelection": true,
        "bidirectionalLrTest": true,
        "enrichmentFamilyRanking": true,
        "mousePseudobulkDesign": false,
        "cancerEpithelialDenominator": false,
        "geneLevelDe": false,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "trajectoryFile": "scbench-long/trajectories/gse155381_q7_xspecies_epi_communication__claude-opus-4-6__pi__r1.json",
      "runId": "claude-opus-4-6__pi__gse155381_q7_xspecies_epi_communication__r1",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-opus-4-6-pi",
      "modelName": "claude-opus-4-6",
      "harness": "pi",
      "task": "gse155381_q7_xspecies_epi_communication",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 2,
      "passed": false,
      "score": 3.25,
      "durationS": 1137.25,
      "cost": 1.56317425,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "richness_weighted",
            "role": "additive",
            "field": "$.xspecies_signaling_richness",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_weighted",
            "role": "additive",
            "field": "$.dominant_signaling_axis",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_wnt_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "WNT_FZD_LRP_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_fgf_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "FGF_FGFR_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_notch_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "NOTCH_DLL_JAG_family"
                ]
              }
            }
          },
          {
            "name": "signaling_directionality_weighted",
            "role": "additive",
            "field": "$.signaling_directionality",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "top_signal_interaction_class_weighted",
            "role": "additive",
            "field": "$.top_signal_interaction_class",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          }
        ]
      },
      "reasoning": "all_of [pass_rule=score_threshold]: FAIL\n  + richness_weighted: passed=True, score=1.5\n  + dominant_axis_weighted: passed=True, score=0.25\n  + signaling_directionality_weighted: passed=True, score=0.5\n  + top_signal_interaction_class_weighted: passed=True, score=1.0\n  + hard_fail 'dominant_axis_wnt_unsupported_by_own_pairs': not triggered\n  + hard_fail 'dominant_axis_fgf_unsupported_by_own_pairs': not triggered\n  + hard_fail 'dominant_axis_notch_unsupported_by_own_pairs': not triggered",
      "metrics": {
        "pass_rule": "score_threshold",
        "scoring_count": 4,
        "scoring_passed": 4,
        "scoring_total_score": 3.25,
        "hard_fail_triggered": []
      },
      "fieldScores": {
        "richness_weighted": 1.5,
        "dominant_axis_weighted": 0.25,
        "signaling_directionality_weighted": 0.5,
        "top_signal_interaction_class_weighted": 1
      },
      "graderChecks": [
        {
          "name": "richness_weighted",
          "kind": "gate",
          "passed": true,
          "score": 1.5
        },
        {
          "name": "dominant_axis_weighted",
          "kind": "gate",
          "passed": true,
          "score": 0.25
        },
        {
          "name": "signaling_directionality_weighted",
          "kind": "gate",
          "passed": true,
          "score": 0.5
        },
        {
          "name": "top_signal_interaction_class_weighted",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "dominant_axis_wnt_unsupported_by_own_pairs",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "dominant_axis_fgf_unsupported_by_own_pairs",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "dominant_axis_notch_unsupported_by_own_pairs",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "dominant_axis_support": [
          "FN1",
          "ITGA6",
          "SDC2",
          "VTN",
          "CD47",
          "COL4A5",
          "SPP1",
          "CD44",
          "COMP",
          "COL9A2",
          "APOE",
          "F13A1",
          "ITGA4"
        ],
        "dominant_signaling_axis": "ECM_collagen_integrin_family",
        "signaling_directionality": "predominantly_chimera_monkey_to_chimera_human",
        "top_signal_interaction_class": "protein_ligand_receptor_dominant",
        "xspecies_signaling_richness": "many_significant_pairs_across_multiple_signalling_families"
      },
      "nTurns": 37,
      "checkpointSignals": {
        "clustering": true,
        "markerAnnotation": true,
        "scoreBasedAnnotation": true,
        "normalization": true,
        "nerveJointAnchor": false,
        "multiomeAssembly": false,
        "depthNormalizedAtac": false,
        "regulonInference": false,
        "cd8SubsetIntegration": false,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": true,
        "orthologProjection": true,
        "epiSubsetSelection": true,
        "bidirectionalLrTest": true,
        "enrichmentFamilyRanking": true,
        "mousePseudobulkDesign": false,
        "cancerEpithelialDenominator": false,
        "geneLevelDe": false,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "trajectoryFile": "scbench-long/trajectories/gse155381_q7_xspecies_epi_communication__claude-opus-4-6__pi__r2.json",
      "runId": "claude-opus-4-6__pi__gse155381_q7_xspecies_epi_communication__r2",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-opus-4-6-pi",
      "modelName": "claude-opus-4-6",
      "harness": "pi",
      "task": "gse155381_q7_xspecies_epi_communication",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 3,
      "passed": false,
      "score": 4.25,
      "durationS": 1767.58,
      "cost": 2.40655225,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "richness_weighted",
            "role": "additive",
            "field": "$.xspecies_signaling_richness",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_weighted",
            "role": "additive",
            "field": "$.dominant_signaling_axis",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_wnt_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "WNT_FZD_LRP_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_fgf_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "FGF_FGFR_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_notch_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "NOTCH_DLL_JAG_family"
                ]
              }
            }
          },
          {
            "name": "signaling_directionality_weighted",
            "role": "additive",
            "field": "$.signaling_directionality",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "top_signal_interaction_class_weighted",
            "role": "additive",
            "field": "$.top_signal_interaction_class",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          }
        ]
      },
      "reasoning": "all_of [pass_rule=score_threshold]: FAIL\n  + richness_weighted: passed=True, score=1.5\n  + dominant_axis_weighted: passed=True, score=0.75\n  + signaling_directionality_weighted: passed=True, score=1.0\n  + top_signal_interaction_class_weighted: passed=True, score=1.0\n  + hard_fail 'dominant_axis_wnt_unsupported_by_own_pairs': not triggered\n  + hard_fail 'dominant_axis_fgf_unsupported_by_own_pairs': not triggered\n  + hard_fail 'dominant_axis_notch_unsupported_by_own_pairs': not triggered",
      "metrics": {
        "pass_rule": "score_threshold",
        "scoring_count": 4,
        "scoring_passed": 4,
        "scoring_total_score": 4.25,
        "hard_fail_triggered": []
      },
      "fieldScores": {
        "richness_weighted": 1.5,
        "dominant_axis_weighted": 0.75,
        "signaling_directionality_weighted": 1,
        "top_signal_interaction_class_weighted": 1
      },
      "graderChecks": [
        {
          "name": "richness_weighted",
          "kind": "gate",
          "passed": true,
          "score": 1.5
        },
        {
          "name": "dominant_axis_weighted",
          "kind": "gate",
          "passed": true,
          "score": 0.75
        },
        {
          "name": "signaling_directionality_weighted",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "top_signal_interaction_class_weighted",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "dominant_axis_wnt_unsupported_by_own_pairs",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "dominant_axis_fgf_unsupported_by_own_pairs",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "dominant_axis_notch_unsupported_by_own_pairs",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "dominant_axis_support": [
          "MDK",
          "NCL",
          "PTPRZ1",
          "SORL1",
          "GRN",
          "TNFRSF1A",
          "SORT1",
          "EGFR",
          "ERBB2"
        ],
        "dominant_signaling_axis": "secreted_growth_factor_RTK_family",
        "signaling_directionality": "substantial_signal_in_both_directions",
        "top_signal_interaction_class": "protein_ligand_receptor_dominant",
        "xspecies_signaling_richness": "many_significant_pairs_across_multiple_signalling_families"
      },
      "nTurns": 35,
      "checkpointSignals": {
        "clustering": true,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": true,
        "nerveJointAnchor": false,
        "multiomeAssembly": false,
        "depthNormalizedAtac": false,
        "regulonInference": false,
        "cd8SubsetIntegration": false,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": true,
        "orthologProjection": true,
        "epiSubsetSelection": true,
        "bidirectionalLrTest": true,
        "enrichmentFamilyRanking": true,
        "mousePseudobulkDesign": false,
        "cancerEpithelialDenominator": true,
        "geneLevelDe": false,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "trajectoryFile": "scbench-long/trajectories/gse155381_q7_xspecies_epi_communication__claude-opus-4-6__pi__r3.json",
      "runId": "claude-opus-4-6__pi__gse155381_q7_xspecies_epi_communication__r3",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-opus-4-7-claude-code",
      "modelName": "claude-opus-4-7",
      "harness": "claude-code",
      "task": "gse155381_q7_xspecies_epi_communication",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 1,
      "passed": false,
      "score": 3.75,
      "durationS": 1503.37,
      "cost": 5.38779975,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "richness_weighted",
            "role": "additive",
            "field": "$.xspecies_signaling_richness",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_weighted",
            "role": "additive",
            "field": "$.dominant_signaling_axis",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_wnt_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "WNT_FZD_LRP_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_fgf_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "FGF_FGFR_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_notch_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "NOTCH_DLL_JAG_family"
                ]
              }
            }
          },
          {
            "name": "signaling_directionality_weighted",
            "role": "additive",
            "field": "$.signaling_directionality",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "top_signal_interaction_class_weighted",
            "role": "additive",
            "field": "$.top_signal_interaction_class",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          }
        ]
      },
      "reasoning": "all_of [pass_rule=score_threshold]: FAIL\n  + richness_weighted: passed=True, score=1.5\n  + dominant_axis_weighted: passed=True, score=0.25\n  + signaling_directionality_weighted: passed=True, score=1.0\n  + top_signal_interaction_class_weighted: passed=True, score=1.0\n  + hard_fail 'dominant_axis_wnt_unsupported_by_own_pairs': not triggered\n  + hard_fail 'dominant_axis_fgf_unsupported_by_own_pairs': not triggered\n  + hard_fail 'dominant_axis_notch_unsupported_by_own_pairs': not triggered",
      "metrics": {
        "pass_rule": "score_threshold",
        "scoring_count": 4,
        "scoring_passed": 4,
        "scoring_total_score": 3.75,
        "hard_fail_triggered": []
      },
      "fieldScores": {
        "richness_weighted": 1.5,
        "dominant_axis_weighted": 0.25,
        "signaling_directionality_weighted": 1,
        "top_signal_interaction_class_weighted": 1
      },
      "graderChecks": [
        {
          "name": "richness_weighted",
          "kind": "gate",
          "passed": true,
          "score": 1.5
        },
        {
          "name": "dominant_axis_weighted",
          "kind": "gate",
          "passed": true,
          "score": 0.25
        },
        {
          "name": "signaling_directionality_weighted",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "top_signal_interaction_class_weighted",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "dominant_axis_wnt_unsupported_by_own_pairs",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "dominant_axis_fgf_unsupported_by_own_pairs",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "dominant_axis_notch_unsupported_by_own_pairs",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "dominant_axis_support": [
          "AGRN",
          "ANGPT1",
          "APOE",
          "CALR",
          "CD44",
          "CD47",
          "COL11A1",
          "COL3A1",
          "COL4A2",
          "DDR1",
          "FAM3C",
          "FBLN1",
          "FN1",
          "HMGB1",
          "ITGA4",
          "ITGA6",
          "ITGAV",
          "ITGB1",
          "ITGB3BP",
          "ITGB5",
          "LAMA1",
          "LAMA2",
          "LAMB1",
          "LAMB2",
          "LAMP1",
          "LGALS1",
          "MDK",
          "MFGE8",
          "PKM",
          "PLAU",
          "RPSA",
          "SDC1",
          "SDC2",
          "SDC3",
          "SDC4",
          "SPP1",
          "THBS2"
        ],
        "dominant_signaling_axis": "ECM_collagen_integrin_family",
        "signaling_directionality": "substantial_signal_in_both_directions",
        "top_signal_interaction_class": "protein_ligand_receptor_dominant",
        "xspecies_signaling_richness": "many_significant_pairs_across_multiple_signalling_families"
      },
      "nTurns": 83,
      "checkpointSignals": {
        "clustering": false,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": true,
        "nerveJointAnchor": false,
        "multiomeAssembly": false,
        "depthNormalizedAtac": false,
        "regulonInference": false,
        "cd8SubsetIntegration": false,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": true,
        "orthologProjection": true,
        "epiSubsetSelection": true,
        "bidirectionalLrTest": true,
        "enrichmentFamilyRanking": false,
        "mousePseudobulkDesign": false,
        "cancerEpithelialDenominator": true,
        "geneLevelDe": false,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "trajectoryFile": "scbench-long/trajectories/gse155381_q7_xspecies_epi_communication__claude-opus-4-7__claude-code__r1.json",
      "runId": "claude-opus-4-7__claude-code__gse155381_q7_xspecies_epi_communication__r1",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-opus-4-7-claude-code",
      "modelName": "claude-opus-4-7",
      "harness": "claude-code",
      "task": "gse155381_q7_xspecies_epi_communication",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 2,
      "passed": false,
      "score": 3.75,
      "durationS": 1873.46,
      "cost": 7.412116500000001,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "richness_weighted",
            "role": "additive",
            "field": "$.xspecies_signaling_richness",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_weighted",
            "role": "additive",
            "field": "$.dominant_signaling_axis",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_wnt_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "WNT_FZD_LRP_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_fgf_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "FGF_FGFR_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_notch_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "NOTCH_DLL_JAG_family"
                ]
              }
            }
          },
          {
            "name": "signaling_directionality_weighted",
            "role": "additive",
            "field": "$.signaling_directionality",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "top_signal_interaction_class_weighted",
            "role": "additive",
            "field": "$.top_signal_interaction_class",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          }
        ]
      },
      "reasoning": "all_of [pass_rule=score_threshold]: FAIL\n  + richness_weighted: passed=True, score=1.5\n  + dominant_axis_weighted: passed=True, score=0.25\n  + signaling_directionality_weighted: passed=True, score=1.0\n  + top_signal_interaction_class_weighted: passed=True, score=1.0\n  + hard_fail 'dominant_axis_wnt_unsupported_by_own_pairs': not triggered\n  + hard_fail 'dominant_axis_fgf_unsupported_by_own_pairs': not triggered\n  + hard_fail 'dominant_axis_notch_unsupported_by_own_pairs': not triggered",
      "metrics": {
        "pass_rule": "score_threshold",
        "scoring_count": 4,
        "scoring_passed": 4,
        "scoring_total_score": 3.75,
        "hard_fail_triggered": []
      },
      "fieldScores": {
        "richness_weighted": 1.5,
        "dominant_axis_weighted": 0.25,
        "signaling_directionality_weighted": 1,
        "top_signal_interaction_class_weighted": 1
      },
      "graderChecks": [
        {
          "name": "richness_weighted",
          "kind": "gate",
          "passed": true,
          "score": 1.5
        },
        {
          "name": "dominant_axis_weighted",
          "kind": "gate",
          "passed": true,
          "score": 0.25
        },
        {
          "name": "signaling_directionality_weighted",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "top_signal_interaction_class_weighted",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "dominant_axis_wnt_unsupported_by_own_pairs",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "dominant_axis_fgf_unsupported_by_own_pairs",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "dominant_axis_notch_unsupported_by_own_pairs",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "dominant_axis_support": [
          "COL1A1",
          "COL1A2",
          "COL4A1",
          "COL4A2",
          "FN1",
          "LAMA1",
          "LAMA4",
          "LAMA5",
          "LAMB1",
          "LAMC1",
          "SPP1",
          "THBS1",
          "THBS2",
          "THBS3",
          "CD47",
          "DAG1",
          "ITGA4",
          "ITGA5",
          "ITGA6",
          "ITGAV",
          "ITGB1",
          "ITGB8",
          "SDC1",
          "SDC4"
        ],
        "dominant_signaling_axis": "ECM_collagen_integrin_family",
        "signaling_directionality": "substantial_signal_in_both_directions",
        "top_signal_interaction_class": "protein_ligand_receptor_dominant",
        "xspecies_signaling_richness": "many_significant_pairs_across_multiple_signalling_families"
      },
      "nTurns": 77,
      "checkpointSignals": {
        "clustering": false,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": true,
        "nerveJointAnchor": false,
        "multiomeAssembly": true,
        "depthNormalizedAtac": false,
        "regulonInference": false,
        "cd8SubsetIntegration": true,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": true,
        "orthologProjection": true,
        "epiSubsetSelection": true,
        "bidirectionalLrTest": true,
        "enrichmentFamilyRanking": true,
        "mousePseudobulkDesign": true,
        "cancerEpithelialDenominator": true,
        "geneLevelDe": true,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "trajectoryFile": "scbench-long/trajectories/gse155381_q7_xspecies_epi_communication__claude-opus-4-7__claude-code__r2.json",
      "runId": "claude-opus-4-7__claude-code__gse155381_q7_xspecies_epi_communication__r2",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-opus-4-7-claude-code",
      "modelName": "claude-opus-4-7",
      "harness": "claude-code",
      "task": "gse155381_q7_xspecies_epi_communication",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 3,
      "passed": false,
      "score": 3.75,
      "durationS": 1455.16,
      "cost": 4.300519249999998,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "richness_weighted",
            "role": "additive",
            "field": "$.xspecies_signaling_richness",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_weighted",
            "role": "additive",
            "field": "$.dominant_signaling_axis",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_wnt_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "WNT_FZD_LRP_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_fgf_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "FGF_FGFR_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_notch_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "NOTCH_DLL_JAG_family"
                ]
              }
            }
          },
          {
            "name": "signaling_directionality_weighted",
            "role": "additive",
            "field": "$.signaling_directionality",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "top_signal_interaction_class_weighted",
            "role": "additive",
            "field": "$.top_signal_interaction_class",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          }
        ]
      },
      "reasoning": "all_of [pass_rule=score_threshold]: FAIL\n  + richness_weighted: passed=True, score=1.5\n  + dominant_axis_weighted: passed=True, score=0.25\n  + signaling_directionality_weighted: passed=True, score=1.0\n  + top_signal_interaction_class_weighted: passed=True, score=1.0\n  + hard_fail 'dominant_axis_wnt_unsupported_by_own_pairs': not triggered\n  + hard_fail 'dominant_axis_fgf_unsupported_by_own_pairs': not triggered\n  + hard_fail 'dominant_axis_notch_unsupported_by_own_pairs': not triggered",
      "metrics": {
        "pass_rule": "score_threshold",
        "scoring_count": 4,
        "scoring_passed": 4,
        "scoring_total_score": 3.75,
        "hard_fail_triggered": []
      },
      "fieldScores": {
        "richness_weighted": 1.5,
        "dominant_axis_weighted": 0.25,
        "signaling_directionality_weighted": 1,
        "top_signal_interaction_class_weighted": 1
      },
      "graderChecks": [
        {
          "name": "richness_weighted",
          "kind": "gate",
          "passed": true,
          "score": 1.5
        },
        {
          "name": "dominant_axis_weighted",
          "kind": "gate",
          "passed": true,
          "score": 0.25
        },
        {
          "name": "signaling_directionality_weighted",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "top_signal_interaction_class_weighted",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "dominant_axis_wnt_unsupported_by_own_pairs",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "dominant_axis_fgf_unsupported_by_own_pairs",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "dominant_axis_notch_unsupported_by_own_pairs",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "dominant_axis_support": [
          "MDK",
          "SDC4",
          "SDC1",
          "SDC2",
          "SDC3",
          "FBLN1",
          "ITGB1",
          "ITGB5",
          "ITGA4",
          "ITGA6",
          "ITGAV",
          "HMGB1",
          "CALR",
          "LRP1",
          "COL3A1",
          "COL4A1",
          "COL4A2",
          "COL11A1",
          "COL5A2",
          "ANGPT1",
          "FN1",
          "PSAP",
          "SERPINE2",
          "DDR1",
          "DDR2",
          "LGALS1",
          "THBS1",
          "THBS2",
          "CD47",
          "PLAU",
          "AGRN",
          "LAMB2",
          "LAMA2",
          "LAMA5",
          "RPSA",
          "MFGE8",
          "VCAN",
          "ADAM15",
          "EDIL3",
          "VCL",
          "HSPG2",
          "CLEC11A"
        ],
        "dominant_signaling_axis": "ECM_collagen_integrin_family",
        "signaling_directionality": "substantial_signal_in_both_directions",
        "top_signal_interaction_class": "protein_ligand_receptor_dominant",
        "xspecies_signaling_richness": "many_significant_pairs_across_multiple_signalling_families"
      },
      "nTurns": 65,
      "checkpointSignals": {
        "clustering": false,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": true,
        "nerveJointAnchor": false,
        "multiomeAssembly": false,
        "depthNormalizedAtac": false,
        "regulonInference": false,
        "cd8SubsetIntegration": false,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": true,
        "orthologProjection": true,
        "epiSubsetSelection": true,
        "bidirectionalLrTest": true,
        "enrichmentFamilyRanking": true,
        "mousePseudobulkDesign": false,
        "cancerEpithelialDenominator": true,
        "geneLevelDe": false,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "trajectoryFile": "scbench-long/trajectories/gse155381_q7_xspecies_epi_communication__claude-opus-4-7__claude-code__r3.json",
      "runId": "claude-opus-4-7__claude-code__gse155381_q7_xspecies_epi_communication__r3",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-opus-4-7-pi",
      "modelName": "claude-opus-4-7",
      "harness": "pi",
      "task": "gse155381_q7_xspecies_epi_communication",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 1,
      "passed": false,
      "score": 3.75,
      "durationS": 1325.48,
      "cost": 2.4964862500000002,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "richness_weighted",
            "role": "additive",
            "field": "$.xspecies_signaling_richness",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_weighted",
            "role": "additive",
            "field": "$.dominant_signaling_axis",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_wnt_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "WNT_FZD_LRP_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_fgf_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "FGF_FGFR_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_notch_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "NOTCH_DLL_JAG_family"
                ]
              }
            }
          },
          {
            "name": "signaling_directionality_weighted",
            "role": "additive",
            "field": "$.signaling_directionality",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "top_signal_interaction_class_weighted",
            "role": "additive",
            "field": "$.top_signal_interaction_class",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          }
        ]
      },
      "reasoning": "all_of [pass_rule=score_threshold]: FAIL\n  + richness_weighted: passed=True, score=1.5\n  + dominant_axis_weighted: passed=True, score=0.25\n  + signaling_directionality_weighted: passed=True, score=1.0\n  + top_signal_interaction_class_weighted: passed=True, score=1.0\n  + hard_fail 'dominant_axis_wnt_unsupported_by_own_pairs': not triggered\n  + hard_fail 'dominant_axis_fgf_unsupported_by_own_pairs': not triggered\n  + hard_fail 'dominant_axis_notch_unsupported_by_own_pairs': not triggered",
      "metrics": {
        "pass_rule": "score_threshold",
        "scoring_count": 4,
        "scoring_passed": 4,
        "scoring_total_score": 3.75,
        "hard_fail_triggered": []
      },
      "fieldScores": {
        "richness_weighted": 1.5,
        "dominant_axis_weighted": 0.25,
        "signaling_directionality_weighted": 1,
        "top_signal_interaction_class_weighted": 1
      },
      "graderChecks": [
        {
          "name": "richness_weighted",
          "kind": "gate",
          "passed": true,
          "score": 1.5
        },
        {
          "name": "dominant_axis_weighted",
          "kind": "gate",
          "passed": true,
          "score": 0.25
        },
        {
          "name": "signaling_directionality_weighted",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "top_signal_interaction_class_weighted",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "dominant_axis_wnt_unsupported_by_own_pairs",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "dominant_axis_fgf_unsupported_by_own_pairs",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "dominant_axis_notch_unsupported_by_own_pairs",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "dominant_axis_support": [
          "MDK",
          "ITGA6",
          "ITGB1",
          "FBLN1",
          "FN1",
          "COL3A1",
          "DDR1",
          "COL11A1",
          "LAMB1",
          "SPP1",
          "ITGA4",
          "THBS2",
          "CD47",
          "MFGE8",
          "ITGAV",
          "ANGPT1",
          "PLAU",
          "SDC2",
          "SDC3",
          "CD44",
          "PKM",
          "CALR"
        ],
        "dominant_signaling_axis": "ECM_collagen_integrin_family",
        "signaling_directionality": "substantial_signal_in_both_directions",
        "top_signal_interaction_class": "protein_ligand_receptor_dominant",
        "xspecies_signaling_richness": "many_significant_pairs_across_multiple_signalling_families"
      },
      "nTurns": 56,
      "checkpointSignals": {
        "clustering": false,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": true,
        "nerveJointAnchor": false,
        "multiomeAssembly": false,
        "depthNormalizedAtac": false,
        "regulonInference": false,
        "cd8SubsetIntegration": false,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": true,
        "orthologProjection": true,
        "epiSubsetSelection": true,
        "bidirectionalLrTest": true,
        "enrichmentFamilyRanking": true,
        "mousePseudobulkDesign": false,
        "cancerEpithelialDenominator": true,
        "geneLevelDe": true,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "trajectoryFile": "scbench-long/trajectories/gse155381_q7_xspecies_epi_communication__claude-opus-4-7__pi__r1.json",
      "runId": "claude-opus-4-7__pi__gse155381_q7_xspecies_epi_communication__r1",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-opus-4-7-pi",
      "modelName": "claude-opus-4-7",
      "harness": "pi",
      "task": "gse155381_q7_xspecies_epi_communication",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 2,
      "passed": false,
      "score": 4.25,
      "durationS": 559.27,
      "cost": 0.913006,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "richness_weighted",
            "role": "additive",
            "field": "$.xspecies_signaling_richness",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_weighted",
            "role": "additive",
            "field": "$.dominant_signaling_axis",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_wnt_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "WNT_FZD_LRP_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_fgf_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "FGF_FGFR_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_notch_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "NOTCH_DLL_JAG_family"
                ]
              }
            }
          },
          {
            "name": "signaling_directionality_weighted",
            "role": "additive",
            "field": "$.signaling_directionality",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "top_signal_interaction_class_weighted",
            "role": "additive",
            "field": "$.top_signal_interaction_class",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          }
        ]
      },
      "reasoning": "all_of [pass_rule=score_threshold]: FAIL\n  + richness_weighted: passed=True, score=1.5\n  + dominant_axis_weighted: passed=True, score=0.75\n  + signaling_directionality_weighted: passed=True, score=1.0\n  + top_signal_interaction_class_weighted: passed=True, score=1.0\n  + hard_fail 'dominant_axis_wnt_unsupported_by_own_pairs': not triggered\n  + hard_fail 'dominant_axis_fgf_unsupported_by_own_pairs': not triggered\n  + hard_fail 'dominant_axis_notch_unsupported_by_own_pairs': not triggered",
      "metrics": {
        "pass_rule": "score_threshold",
        "scoring_count": 4,
        "scoring_passed": 4,
        "scoring_total_score": 4.25,
        "hard_fail_triggered": []
      },
      "fieldScores": {
        "richness_weighted": 1.5,
        "dominant_axis_weighted": 0.75,
        "signaling_directionality_weighted": 1,
        "top_signal_interaction_class_weighted": 1
      },
      "graderChecks": [
        {
          "name": "richness_weighted",
          "kind": "gate",
          "passed": true,
          "score": 1.5
        },
        {
          "name": "dominant_axis_weighted",
          "kind": "gate",
          "passed": true,
          "score": 0.75
        },
        {
          "name": "signaling_directionality_weighted",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "top_signal_interaction_class_weighted",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "dominant_axis_wnt_unsupported_by_own_pairs",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "dominant_axis_fgf_unsupported_by_own_pairs",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "dominant_axis_notch_unsupported_by_own_pairs",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "dominant_axis_support": [
          "EFNA1",
          "EFNA2",
          "EFNA3",
          "EFNA4",
          "EFNA5",
          "EFNB1",
          "EPHA1",
          "EPHA2",
          "EPHA4",
          "EPHA5",
          "EPHA6",
          "EPHA7",
          "EPHB2",
          "EPHB3"
        ],
        "dominant_signaling_axis": "ephrin_EPH_axon_guidance_family",
        "signaling_directionality": "substantial_signal_in_both_directions",
        "top_signal_interaction_class": "protein_ligand_receptor_dominant",
        "xspecies_signaling_richness": "many_significant_pairs_across_multiple_signalling_families"
      },
      "nTurns": 25,
      "checkpointSignals": {
        "clustering": true,
        "markerAnnotation": true,
        "scoreBasedAnnotation": true,
        "normalization": true,
        "nerveJointAnchor": false,
        "multiomeAssembly": false,
        "depthNormalizedAtac": false,
        "regulonInference": false,
        "cd8SubsetIntegration": false,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": true,
        "orthologProjection": true,
        "epiSubsetSelection": true,
        "bidirectionalLrTest": true,
        "enrichmentFamilyRanking": true,
        "mousePseudobulkDesign": false,
        "cancerEpithelialDenominator": true,
        "geneLevelDe": false,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "trajectoryFile": "scbench-long/trajectories/gse155381_q7_xspecies_epi_communication__claude-opus-4-7__pi__r2.json",
      "runId": "claude-opus-4-7__pi__gse155381_q7_xspecies_epi_communication__r2",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-opus-4-7-pi",
      "modelName": "claude-opus-4-7",
      "harness": "pi",
      "task": "gse155381_q7_xspecies_epi_communication",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 3,
      "passed": false,
      "score": 3.75,
      "durationS": 421.12,
      "cost": 1.1443375,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "richness_weighted",
            "role": "additive",
            "field": "$.xspecies_signaling_richness",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_weighted",
            "role": "additive",
            "field": "$.dominant_signaling_axis",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_wnt_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "WNT_FZD_LRP_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_fgf_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "FGF_FGFR_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_notch_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "NOTCH_DLL_JAG_family"
                ]
              }
            }
          },
          {
            "name": "signaling_directionality_weighted",
            "role": "additive",
            "field": "$.signaling_directionality",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "top_signal_interaction_class_weighted",
            "role": "additive",
            "field": "$.top_signal_interaction_class",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          }
        ]
      },
      "reasoning": "all_of [pass_rule=score_threshold]: FAIL\n  + richness_weighted: passed=True, score=1.5\n  + dominant_axis_weighted: passed=True, score=0.25\n  + signaling_directionality_weighted: passed=True, score=1.0\n  + top_signal_interaction_class_weighted: passed=True, score=1.0\n  + hard_fail 'dominant_axis_wnt_unsupported_by_own_pairs': not triggered\n  + hard_fail 'dominant_axis_fgf_unsupported_by_own_pairs': not triggered\n  + hard_fail 'dominant_axis_notch_unsupported_by_own_pairs': not triggered",
      "metrics": {
        "pass_rule": "score_threshold",
        "scoring_count": 4,
        "scoring_passed": 4,
        "scoring_total_score": 3.75,
        "hard_fail_triggered": []
      },
      "fieldScores": {
        "richness_weighted": 1.5,
        "dominant_axis_weighted": 0.25,
        "signaling_directionality_weighted": 1,
        "top_signal_interaction_class_weighted": 1
      },
      "graderChecks": [
        {
          "name": "richness_weighted",
          "kind": "gate",
          "passed": true,
          "score": 1.5
        },
        {
          "name": "dominant_axis_weighted",
          "kind": "gate",
          "passed": true,
          "score": 0.25
        },
        {
          "name": "signaling_directionality_weighted",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "top_signal_interaction_class_weighted",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "dominant_axis_wnt_unsupported_by_own_pairs",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "dominant_axis_fgf_unsupported_by_own_pairs",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "dominant_axis_notch_unsupported_by_own_pairs",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "dominant_axis_support": [
          "FN1",
          "ITGA6",
          "ITGB1",
          "ITGA5",
          "FBLN1",
          "VCAN",
          "ANGPT1",
          "JAM3",
          "LAMA4",
          "LAMB1",
          "LAMC1",
          "ITGA7",
          "COL4A2",
          "COL4A1",
          "COL4A3",
          "COL4A4",
          "COL4A5",
          "COL4A6",
          "COL3A1",
          "ITGA10",
          "ITGB5",
          "ITGB8",
          "MFGE8",
          "ITGAV",
          "SPP1",
          "ITGA4",
          "THBS1",
          "THBS2",
          "THBS4",
          "CD47",
          "VTN",
          "NID1",
          "HSPG2",
          "FBN1",
          "EDIL3",
          "TGFBI",
          "PLAU",
          "SDC2",
          "ADAM9",
          "ADAM15",
          "ADAM17",
          "CALR"
        ],
        "dominant_signaling_axis": "ECM_collagen_integrin_family",
        "signaling_directionality": "substantial_signal_in_both_directions",
        "top_signal_interaction_class": "protein_ligand_receptor_dominant",
        "xspecies_signaling_richness": "many_significant_pairs_across_multiple_signalling_families"
      },
      "nTurns": 25,
      "checkpointSignals": {
        "clustering": false,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": true,
        "nerveJointAnchor": false,
        "multiomeAssembly": true,
        "depthNormalizedAtac": false,
        "regulonInference": false,
        "cd8SubsetIntegration": true,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": true,
        "orthologProjection": true,
        "epiSubsetSelection": true,
        "bidirectionalLrTest": true,
        "enrichmentFamilyRanking": true,
        "mousePseudobulkDesign": true,
        "cancerEpithelialDenominator": true,
        "geneLevelDe": true,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "trajectoryFile": "scbench-long/trajectories/gse155381_q7_xspecies_epi_communication__claude-opus-4-7__pi__r3.json",
      "runId": "claude-opus-4-7__pi__gse155381_q7_xspecies_epi_communication__r3",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-opus-4-8-claude-code",
      "modelName": "claude-opus-4-8",
      "harness": "claude-code",
      "task": "gse155381_q7_xspecies_epi_communication",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 1,
      "passed": false,
      "score": 3.75,
      "durationS": 1451.32,
      "cost": 4.451310250000001,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "richness_weighted",
            "role": "additive",
            "field": "$.xspecies_signaling_richness",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_weighted",
            "role": "additive",
            "field": "$.dominant_signaling_axis",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_wnt_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "WNT_FZD_LRP_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_fgf_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "FGF_FGFR_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_notch_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "NOTCH_DLL_JAG_family"
                ]
              }
            }
          },
          {
            "name": "signaling_directionality_weighted",
            "role": "additive",
            "field": "$.signaling_directionality",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "top_signal_interaction_class_weighted",
            "role": "additive",
            "field": "$.top_signal_interaction_class",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          }
        ]
      },
      "reasoning": "all_of [pass_rule=score_threshold]: FAIL\n  + richness_weighted: passed=True, score=1.5\n  + dominant_axis_weighted: passed=True, score=0.25\n  + signaling_directionality_weighted: passed=True, score=1.0\n  + top_signal_interaction_class_weighted: passed=True, score=1.0\n  + hard_fail 'dominant_axis_wnt_unsupported_by_own_pairs': not triggered\n  + hard_fail 'dominant_axis_fgf_unsupported_by_own_pairs': not triggered\n  + hard_fail 'dominant_axis_notch_unsupported_by_own_pairs': not triggered",
      "metrics": {
        "pass_rule": "score_threshold",
        "scoring_count": 4,
        "scoring_passed": 4,
        "scoring_total_score": 3.75,
        "hard_fail_triggered": []
      },
      "fieldScores": {
        "richness_weighted": 1.5,
        "dominant_axis_weighted": 0.25,
        "signaling_directionality_weighted": 1,
        "top_signal_interaction_class_weighted": 1
      },
      "graderChecks": [
        {
          "name": "richness_weighted",
          "kind": "gate",
          "passed": true,
          "score": 1.5
        },
        {
          "name": "dominant_axis_weighted",
          "kind": "gate",
          "passed": true,
          "score": 0.25
        },
        {
          "name": "signaling_directionality_weighted",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "top_signal_interaction_class_weighted",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "dominant_axis_wnt_unsupported_by_own_pairs",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "dominant_axis_fgf_unsupported_by_own_pairs",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "dominant_axis_notch_unsupported_by_own_pairs",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "dominant_axis_support": [
          "COL3A1",
          "COL5A2",
          "COL11A1",
          "COL4A2",
          "COL21A1",
          "LAMB1",
          "LAMA1",
          "LAMA4",
          "LAMB3",
          "FN1",
          "THBS2",
          "SPP1",
          "ITGB1",
          "ITGA6",
          "ITGA10",
          "ITGA7",
          "ITGA2",
          "ITGA11",
          "ITGA5",
          "ITGAV",
          "ITGB5",
          "ITGA4",
          "DDR1"
        ],
        "dominant_signaling_axis": "ECM_collagen_integrin_family",
        "signaling_directionality": "substantial_signal_in_both_directions",
        "top_signal_interaction_class": "protein_ligand_receptor_dominant",
        "xspecies_signaling_richness": "many_significant_pairs_across_multiple_signalling_families"
      },
      "nTurns": 50,
      "checkpointSignals": {
        "clustering": true,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": true,
        "nerveJointAnchor": false,
        "multiomeAssembly": true,
        "depthNormalizedAtac": false,
        "regulonInference": false,
        "cd8SubsetIntegration": true,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": true,
        "orthologProjection": true,
        "epiSubsetSelection": true,
        "bidirectionalLrTest": true,
        "enrichmentFamilyRanking": true,
        "mousePseudobulkDesign": true,
        "cancerEpithelialDenominator": false,
        "geneLevelDe": true,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "trajectoryFile": "scbench-long/trajectories/gse155381_q7_xspecies_epi_communication__claude-opus-4-8__claude-code__r1.json",
      "runId": "claude-opus-4-8__claude-code__gse155381_q7_xspecies_epi_communication__r1",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-opus-4-8-claude-code",
      "modelName": "claude-opus-4-8",
      "harness": "claude-code",
      "task": "gse155381_q7_xspecies_epi_communication",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 2,
      "passed": false,
      "score": 3.75,
      "durationS": 1423.42,
      "cost": 4.18313125,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "richness_weighted",
            "role": "additive",
            "field": "$.xspecies_signaling_richness",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_weighted",
            "role": "additive",
            "field": "$.dominant_signaling_axis",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_wnt_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "WNT_FZD_LRP_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_fgf_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "FGF_FGFR_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_notch_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "NOTCH_DLL_JAG_family"
                ]
              }
            }
          },
          {
            "name": "signaling_directionality_weighted",
            "role": "additive",
            "field": "$.signaling_directionality",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "top_signal_interaction_class_weighted",
            "role": "additive",
            "field": "$.top_signal_interaction_class",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          }
        ]
      },
      "reasoning": "all_of [pass_rule=score_threshold]: FAIL\n  + richness_weighted: passed=True, score=1.5\n  + dominant_axis_weighted: passed=True, score=0.25\n  + signaling_directionality_weighted: passed=True, score=1.0\n  + top_signal_interaction_class_weighted: passed=True, score=1.0\n  + hard_fail 'dominant_axis_wnt_unsupported_by_own_pairs': not triggered\n  + hard_fail 'dominant_axis_fgf_unsupported_by_own_pairs': not triggered\n  + hard_fail 'dominant_axis_notch_unsupported_by_own_pairs': not triggered",
      "metrics": {
        "pass_rule": "score_threshold",
        "scoring_count": 4,
        "scoring_passed": 4,
        "scoring_total_score": 3.75,
        "hard_fail_triggered": []
      },
      "fieldScores": {
        "richness_weighted": 1.5,
        "dominant_axis_weighted": 0.25,
        "signaling_directionality_weighted": 1,
        "top_signal_interaction_class_weighted": 1
      },
      "graderChecks": [
        {
          "name": "richness_weighted",
          "kind": "gate",
          "passed": true,
          "score": 1.5
        },
        {
          "name": "dominant_axis_weighted",
          "kind": "gate",
          "passed": true,
          "score": 0.25
        },
        {
          "name": "signaling_directionality_weighted",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "top_signal_interaction_class_weighted",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "dominant_axis_wnt_unsupported_by_own_pairs",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "dominant_axis_fgf_unsupported_by_own_pairs",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "dominant_axis_notch_unsupported_by_own_pairs",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "dominant_axis_support": [
          "COL3A1",
          "COL11A1",
          "MFGE8",
          "THBS2",
          "LAMA1",
          "COL5A2",
          "COL4A2",
          "ANGPT1",
          "LAMA4",
          "LAMA5",
          "COL4A6",
          "TGFB3",
          "ADAM9",
          "SPON2",
          "SPP1",
          "LAMB3",
          "COL21A1",
          "LAMB1",
          "ITGA10",
          "ITGB1",
          "DDR1",
          "ITGAV",
          "CD47",
          "ITGA6",
          "ITGA4",
          "ITGB5",
          "ITGA5",
          "ITGA7",
          "ITGA11",
          "ITGB8"
        ],
        "dominant_signaling_axis": "ECM_collagen_integrin_family",
        "signaling_directionality": "substantial_signal_in_both_directions",
        "top_signal_interaction_class": "protein_ligand_receptor_dominant",
        "xspecies_signaling_richness": "many_significant_pairs_across_multiple_signalling_families"
      },
      "nTurns": 48,
      "checkpointSignals": {
        "clustering": true,
        "markerAnnotation": true,
        "scoreBasedAnnotation": true,
        "normalization": true,
        "nerveJointAnchor": false,
        "multiomeAssembly": true,
        "depthNormalizedAtac": true,
        "regulonInference": false,
        "cd8SubsetIntegration": true,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": true,
        "orthologProjection": true,
        "epiSubsetSelection": true,
        "bidirectionalLrTest": true,
        "enrichmentFamilyRanking": true,
        "mousePseudobulkDesign": true,
        "cancerEpithelialDenominator": true,
        "geneLevelDe": true,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "trajectoryFile": "scbench-long/trajectories/gse155381_q7_xspecies_epi_communication__claude-opus-4-8__claude-code__r2.json",
      "runId": "claude-opus-4-8__claude-code__gse155381_q7_xspecies_epi_communication__r2",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-opus-4-8-claude-code",
      "modelName": "claude-opus-4-8",
      "harness": "claude-code",
      "task": "gse155381_q7_xspecies_epi_communication",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 3,
      "passed": false,
      "score": 3.75,
      "durationS": 1762.7,
      "cost": 6.574657749999999,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "richness_weighted",
            "role": "additive",
            "field": "$.xspecies_signaling_richness",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_weighted",
            "role": "additive",
            "field": "$.dominant_signaling_axis",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_wnt_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "WNT_FZD_LRP_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_fgf_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "FGF_FGFR_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_notch_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "NOTCH_DLL_JAG_family"
                ]
              }
            }
          },
          {
            "name": "signaling_directionality_weighted",
            "role": "additive",
            "field": "$.signaling_directionality",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "top_signal_interaction_class_weighted",
            "role": "additive",
            "field": "$.top_signal_interaction_class",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          }
        ]
      },
      "reasoning": "all_of [pass_rule=score_threshold]: FAIL\n  + richness_weighted: passed=True, score=1.5\n  + dominant_axis_weighted: passed=True, score=0.25\n  + signaling_directionality_weighted: passed=True, score=1.0\n  + top_signal_interaction_class_weighted: passed=True, score=1.0\n  + hard_fail 'dominant_axis_wnt_unsupported_by_own_pairs': not triggered\n  + hard_fail 'dominant_axis_fgf_unsupported_by_own_pairs': not triggered\n  + hard_fail 'dominant_axis_notch_unsupported_by_own_pairs': not triggered",
      "metrics": {
        "pass_rule": "score_threshold",
        "scoring_count": 4,
        "scoring_passed": 4,
        "scoring_total_score": 3.75,
        "hard_fail_triggered": []
      },
      "fieldScores": {
        "richness_weighted": 1.5,
        "dominant_axis_weighted": 0.25,
        "signaling_directionality_weighted": 1,
        "top_signal_interaction_class_weighted": 1
      },
      "graderChecks": [
        {
          "name": "richness_weighted",
          "kind": "gate",
          "passed": true,
          "score": 1.5
        },
        {
          "name": "dominant_axis_weighted",
          "kind": "gate",
          "passed": true,
          "score": 0.25
        },
        {
          "name": "signaling_directionality_weighted",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "top_signal_interaction_class_weighted",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "dominant_axis_wnt_unsupported_by_own_pairs",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "dominant_axis_fgf_unsupported_by_own_pairs",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "dominant_axis_notch_unsupported_by_own_pairs",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "dominant_axis_support": [
          "COL21A1",
          "COL3A1",
          "COL5A2",
          "COL11A1",
          "LAMA1",
          "LAMA2",
          "LAMA4",
          "LAMA5",
          "LAMB3",
          "FN1",
          "THBS1",
          "THBS2",
          "EDIL3",
          "FBN1",
          "RELN",
          "SPON2",
          "ITGA2",
          "ITGA3",
          "ITGA4",
          "ITGA5",
          "ITGA6",
          "ITGA7",
          "ITGA10",
          "ITGA11",
          "ITGAV",
          "ITGB1",
          "ITGB5",
          "ITGB8",
          "DDR1",
          "SDC2",
          "RPSA"
        ],
        "dominant_signaling_axis": "ECM_collagen_integrin_family",
        "signaling_directionality": "substantial_signal_in_both_directions",
        "top_signal_interaction_class": "protein_ligand_receptor_dominant",
        "xspecies_signaling_richness": "many_significant_pairs_across_multiple_signalling_families"
      },
      "nTurns": 60,
      "checkpointSignals": {
        "clustering": true,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": true,
        "nerveJointAnchor": false,
        "multiomeAssembly": true,
        "depthNormalizedAtac": false,
        "regulonInference": false,
        "cd8SubsetIntegration": true,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": true,
        "orthologProjection": true,
        "epiSubsetSelection": true,
        "bidirectionalLrTest": true,
        "enrichmentFamilyRanking": true,
        "mousePseudobulkDesign": true,
        "cancerEpithelialDenominator": true,
        "geneLevelDe": true,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "trajectoryFile": "scbench-long/trajectories/gse155381_q7_xspecies_epi_communication__claude-opus-4-8__claude-code__r3.json",
      "runId": "claude-opus-4-8__claude-code__gse155381_q7_xspecies_epi_communication__r3",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-opus-4-8-pi",
      "modelName": "claude-opus-4-8",
      "harness": "pi",
      "task": "gse155381_q7_xspecies_epi_communication",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 1,
      "passed": false,
      "score": 3.75,
      "durationS": 959.3,
      "cost": 2.6514752500000003,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "richness_weighted",
            "role": "additive",
            "field": "$.xspecies_signaling_richness",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_weighted",
            "role": "additive",
            "field": "$.dominant_signaling_axis",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_wnt_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "WNT_FZD_LRP_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_fgf_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "FGF_FGFR_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_notch_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "NOTCH_DLL_JAG_family"
                ]
              }
            }
          },
          {
            "name": "signaling_directionality_weighted",
            "role": "additive",
            "field": "$.signaling_directionality",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "top_signal_interaction_class_weighted",
            "role": "additive",
            "field": "$.top_signal_interaction_class",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          }
        ]
      },
      "reasoning": "all_of [pass_rule=score_threshold]: FAIL\n  + richness_weighted: passed=True, score=1.5\n  + dominant_axis_weighted: passed=True, score=0.25\n  + signaling_directionality_weighted: passed=True, score=1.0\n  + top_signal_interaction_class_weighted: passed=True, score=1.0\n  + hard_fail 'dominant_axis_wnt_unsupported_by_own_pairs': not triggered\n  + hard_fail 'dominant_axis_fgf_unsupported_by_own_pairs': not triggered\n  + hard_fail 'dominant_axis_notch_unsupported_by_own_pairs': not triggered",
      "metrics": {
        "pass_rule": "score_threshold",
        "scoring_count": 4,
        "scoring_passed": 4,
        "scoring_total_score": 3.75,
        "hard_fail_triggered": []
      },
      "fieldScores": {
        "richness_weighted": 1.5,
        "dominant_axis_weighted": 0.25,
        "signaling_directionality_weighted": 1,
        "top_signal_interaction_class_weighted": 1
      },
      "graderChecks": [
        {
          "name": "richness_weighted",
          "kind": "gate",
          "passed": true,
          "score": 1.5
        },
        {
          "name": "dominant_axis_weighted",
          "kind": "gate",
          "passed": true,
          "score": 0.25
        },
        {
          "name": "signaling_directionality_weighted",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "top_signal_interaction_class_weighted",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "dominant_axis_wnt_unsupported_by_own_pairs",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "dominant_axis_fgf_unsupported_by_own_pairs",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "dominant_axis_notch_unsupported_by_own_pairs",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "dominant_axis_support": [
          "COL3A1",
          "COL11A1",
          "COL6A1",
          "COL21A1",
          "FN1",
          "LAMB1",
          "LAMB2",
          "LAMA4",
          "THBS1",
          "THBS2",
          "EDIL3",
          "SPP1",
          "CCN1",
          "ANGPT1",
          "ANGPTL4",
          "FBLN1",
          "ITGB1",
          "ITGB5",
          "ITGA2",
          "ITGA4",
          "ITGA5",
          "ITGA6",
          "ITGA7",
          "ITGA10",
          "ITGA11",
          "ITGAV",
          "DDR1",
          "SDC1",
          "SDC2",
          "SDC4",
          "RPSA"
        ],
        "dominant_signaling_axis": "ECM_collagen_integrin_family",
        "signaling_directionality": "substantial_signal_in_both_directions",
        "top_signal_interaction_class": "protein_ligand_receptor_dominant",
        "xspecies_signaling_richness": "many_significant_pairs_across_multiple_signalling_families"
      },
      "nTurns": 28,
      "checkpointSignals": {
        "clustering": true,
        "markerAnnotation": true,
        "scoreBasedAnnotation": true,
        "normalization": true,
        "nerveJointAnchor": false,
        "multiomeAssembly": true,
        "depthNormalizedAtac": false,
        "regulonInference": false,
        "cd8SubsetIntegration": true,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": true,
        "orthologProjection": true,
        "epiSubsetSelection": true,
        "bidirectionalLrTest": true,
        "enrichmentFamilyRanking": true,
        "mousePseudobulkDesign": true,
        "cancerEpithelialDenominator": false,
        "geneLevelDe": true,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "trajectoryFile": "scbench-long/trajectories/gse155381_q7_xspecies_epi_communication__claude-opus-4-8__pi__r1.json",
      "runId": "claude-opus-4-8__pi__gse155381_q7_xspecies_epi_communication__r1",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-opus-4-8-pi",
      "modelName": "claude-opus-4-8",
      "harness": "pi",
      "task": "gse155381_q7_xspecies_epi_communication",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 2,
      "passed": false,
      "score": 3.75,
      "durationS": 1585.57,
      "cost": 3.127775750000001,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "richness_weighted",
            "role": "additive",
            "field": "$.xspecies_signaling_richness",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_weighted",
            "role": "additive",
            "field": "$.dominant_signaling_axis",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_wnt_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "WNT_FZD_LRP_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_fgf_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "FGF_FGFR_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_notch_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "NOTCH_DLL_JAG_family"
                ]
              }
            }
          },
          {
            "name": "signaling_directionality_weighted",
            "role": "additive",
            "field": "$.signaling_directionality",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "top_signal_interaction_class_weighted",
            "role": "additive",
            "field": "$.top_signal_interaction_class",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          }
        ]
      },
      "reasoning": "all_of [pass_rule=score_threshold]: FAIL\n  + richness_weighted: passed=True, score=1.5\n  + dominant_axis_weighted: passed=True, score=0.25\n  + signaling_directionality_weighted: passed=True, score=1.0\n  + top_signal_interaction_class_weighted: passed=True, score=1.0\n  + hard_fail 'dominant_axis_wnt_unsupported_by_own_pairs': not triggered\n  + hard_fail 'dominant_axis_fgf_unsupported_by_own_pairs': not triggered\n  + hard_fail 'dominant_axis_notch_unsupported_by_own_pairs': not triggered",
      "metrics": {
        "pass_rule": "score_threshold",
        "scoring_count": 4,
        "scoring_passed": 4,
        "scoring_total_score": 3.75,
        "hard_fail_triggered": []
      },
      "fieldScores": {
        "richness_weighted": 1.5,
        "dominant_axis_weighted": 0.25,
        "signaling_directionality_weighted": 1,
        "top_signal_interaction_class_weighted": 1
      },
      "graderChecks": [
        {
          "name": "richness_weighted",
          "kind": "gate",
          "passed": true,
          "score": 1.5
        },
        {
          "name": "dominant_axis_weighted",
          "kind": "gate",
          "passed": true,
          "score": 0.25
        },
        {
          "name": "signaling_directionality_weighted",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "top_signal_interaction_class_weighted",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "dominant_axis_wnt_unsupported_by_own_pairs",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "dominant_axis_fgf_unsupported_by_own_pairs",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "dominant_axis_notch_unsupported_by_own_pairs",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "dominant_axis_support": [
          "LAMA1",
          "LAMA2",
          "LAMB1",
          "LAMB2",
          "LAMC1",
          "COL3A1",
          "COL11A1",
          "COL5A2",
          "COL4A2",
          "COL4A1",
          "COL1A1",
          "FN1",
          "FBLN1",
          "MFGE8",
          "SPP1",
          "VCAN",
          "THBS1",
          "THBS2",
          "RPSA",
          "ITGB1",
          "ITGA1",
          "ITGA2",
          "ITGA4",
          "ITGA6",
          "ITGA10",
          "ITGAV",
          "ITGB5",
          "ITGB8",
          "DDR1",
          "DDR2",
          "SDC2",
          "CD44"
        ],
        "dominant_signaling_axis": "ECM_collagen_integrin_family",
        "signaling_directionality": "substantial_signal_in_both_directions",
        "top_signal_interaction_class": "protein_ligand_receptor_dominant",
        "xspecies_signaling_richness": "many_significant_pairs_across_multiple_signalling_families"
      },
      "nTurns": 34,
      "checkpointSignals": {
        "clustering": true,
        "markerAnnotation": true,
        "scoreBasedAnnotation": true,
        "normalization": true,
        "nerveJointAnchor": false,
        "multiomeAssembly": true,
        "depthNormalizedAtac": false,
        "regulonInference": false,
        "cd8SubsetIntegration": true,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": true,
        "orthologProjection": true,
        "epiSubsetSelection": true,
        "bidirectionalLrTest": true,
        "enrichmentFamilyRanking": true,
        "mousePseudobulkDesign": true,
        "cancerEpithelialDenominator": true,
        "geneLevelDe": true,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "trajectoryFile": "scbench-long/trajectories/gse155381_q7_xspecies_epi_communication__claude-opus-4-8__pi__r2.json",
      "runId": "claude-opus-4-8__pi__gse155381_q7_xspecies_epi_communication__r2",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-opus-4-8-pi",
      "modelName": "claude-opus-4-8",
      "harness": "pi",
      "task": "gse155381_q7_xspecies_epi_communication",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 3,
      "passed": false,
      "score": 4.25,
      "durationS": 1154.35,
      "cost": 2.38378625,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "richness_weighted",
            "role": "additive",
            "field": "$.xspecies_signaling_richness",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_weighted",
            "role": "additive",
            "field": "$.dominant_signaling_axis",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_wnt_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "WNT_FZD_LRP_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_fgf_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "FGF_FGFR_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_notch_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "NOTCH_DLL_JAG_family"
                ]
              }
            }
          },
          {
            "name": "signaling_directionality_weighted",
            "role": "additive",
            "field": "$.signaling_directionality",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "top_signal_interaction_class_weighted",
            "role": "additive",
            "field": "$.top_signal_interaction_class",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          }
        ]
      },
      "reasoning": "all_of [pass_rule=score_threshold]: FAIL\n  + richness_weighted: passed=True, score=1.5\n  + dominant_axis_weighted: passed=True, score=0.75\n  + signaling_directionality_weighted: passed=True, score=1.0\n  + top_signal_interaction_class_weighted: passed=True, score=1.0\n  + hard_fail 'dominant_axis_wnt_unsupported_by_own_pairs': not triggered\n  + hard_fail 'dominant_axis_fgf_unsupported_by_own_pairs': not triggered\n  + hard_fail 'dominant_axis_notch_unsupported_by_own_pairs': not triggered",
      "metrics": {
        "pass_rule": "score_threshold",
        "scoring_count": 4,
        "scoring_passed": 4,
        "scoring_total_score": 4.25,
        "hard_fail_triggered": []
      },
      "fieldScores": {
        "richness_weighted": 1.5,
        "dominant_axis_weighted": 0.75,
        "signaling_directionality_weighted": 1,
        "top_signal_interaction_class_weighted": 1
      },
      "graderChecks": [
        {
          "name": "richness_weighted",
          "kind": "gate",
          "passed": true,
          "score": 1.5
        },
        {
          "name": "dominant_axis_weighted",
          "kind": "gate",
          "passed": true,
          "score": 0.75
        },
        {
          "name": "signaling_directionality_weighted",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "top_signal_interaction_class_weighted",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "dominant_axis_wnt_unsupported_by_own_pairs",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "dominant_axis_fgf_unsupported_by_own_pairs",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "dominant_axis_notch_unsupported_by_own_pairs",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "dominant_axis_support": [
          "MDK",
          "NCL",
          "SDC1",
          "SDC2",
          "SDC3",
          "SDC4",
          "PTPRZ1",
          "LRP1",
          "GPC2",
          "ALK",
          "ITGA6",
          "ITGB1",
          "NOTCH2",
          "HBEGF",
          "CD9",
          "BDNF",
          "GRN",
          "MIF",
          "EGFR",
          "NTRK1",
          "VEGFA",
          "VEGFC",
          "ANGPT1",
          "PDGFRB"
        ],
        "dominant_signaling_axis": "secreted_growth_factor_RTK_family",
        "signaling_directionality": "substantial_signal_in_both_directions",
        "top_signal_interaction_class": "protein_ligand_receptor_dominant",
        "xspecies_signaling_richness": "many_significant_pairs_across_multiple_signalling_families"
      },
      "nTurns": 30,
      "checkpointSignals": {
        "clustering": true,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": true,
        "nerveJointAnchor": false,
        "multiomeAssembly": false,
        "depthNormalizedAtac": false,
        "regulonInference": false,
        "cd8SubsetIntegration": false,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": true,
        "orthologProjection": true,
        "epiSubsetSelection": true,
        "bidirectionalLrTest": true,
        "enrichmentFamilyRanking": true,
        "mousePseudobulkDesign": false,
        "cancerEpithelialDenominator": false,
        "geneLevelDe": false,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "trajectoryFile": "scbench-long/trajectories/gse155381_q7_xspecies_epi_communication__claude-opus-4-8__pi__r3.json",
      "runId": "claude-opus-4-8__pi__gse155381_q7_xspecies_epi_communication__r3",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-sonnet-4-6-pi",
      "modelName": "claude-sonnet-4-6",
      "harness": "pi",
      "task": "gse155381_q7_xspecies_epi_communication",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 1,
      "passed": false,
      "score": 3.25,
      "durationS": 2892.27,
      "cost": 4.07550375,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "richness_weighted",
            "role": "additive",
            "field": "$.xspecies_signaling_richness",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_weighted",
            "role": "additive",
            "field": "$.dominant_signaling_axis",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_wnt_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "WNT_FZD_LRP_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_fgf_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "FGF_FGFR_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_notch_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "NOTCH_DLL_JAG_family"
                ]
              }
            }
          },
          {
            "name": "signaling_directionality_weighted",
            "role": "additive",
            "field": "$.signaling_directionality",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "top_signal_interaction_class_weighted",
            "role": "additive",
            "field": "$.top_signal_interaction_class",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          }
        ]
      },
      "reasoning": "all_of [pass_rule=score_threshold]: FAIL\n  + richness_weighted: passed=True, score=1.5\n  + dominant_axis_weighted: passed=True, score=0.25\n  + signaling_directionality_weighted: passed=True, score=0.5\n  + top_signal_interaction_class_weighted: passed=True, score=1.0\n  + hard_fail 'dominant_axis_wnt_unsupported_by_own_pairs': not triggered\n  + hard_fail 'dominant_axis_fgf_unsupported_by_own_pairs': not triggered\n  + hard_fail 'dominant_axis_notch_unsupported_by_own_pairs': not triggered",
      "metrics": {
        "pass_rule": "score_threshold",
        "scoring_count": 4,
        "scoring_passed": 4,
        "scoring_total_score": 3.25,
        "hard_fail_triggered": []
      },
      "fieldScores": {
        "richness_weighted": 1.5,
        "dominant_axis_weighted": 0.25,
        "signaling_directionality_weighted": 0.5,
        "top_signal_interaction_class_weighted": 1
      },
      "graderChecks": [
        {
          "name": "richness_weighted",
          "kind": "gate",
          "passed": true,
          "score": 1.5
        },
        {
          "name": "dominant_axis_weighted",
          "kind": "gate",
          "passed": true,
          "score": 0.25
        },
        {
          "name": "signaling_directionality_weighted",
          "kind": "gate",
          "passed": true,
          "score": 0.5
        },
        {
          "name": "top_signal_interaction_class_weighted",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "dominant_axis_wnt_unsupported_by_own_pairs",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "dominant_axis_fgf_unsupported_by_own_pairs",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "dominant_axis_notch_unsupported_by_own_pairs",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "dominant_axis_support": [
          "COL3A1",
          "DDR1",
          "COL11A1",
          "FN1",
          "ITGA6",
          "THBS2",
          "CD47",
          "ANGPTL4",
          "SDC1",
          "SDC4",
          "MFGE8",
          "ITGAV",
          "CD14",
          "ITGB1",
          "ITGA4",
          "VEGFD",
          "ADAM9",
          "PKM",
          "CD44",
          "THY1"
        ],
        "dominant_signaling_axis": "ECM_collagen_integrin_family",
        "signaling_directionality": "predominantly_chimera_monkey_to_chimera_human",
        "top_signal_interaction_class": "protein_ligand_receptor_dominant",
        "xspecies_signaling_richness": "many_significant_pairs_across_multiple_signalling_families"
      },
      "nTurns": 92,
      "checkpointSignals": {
        "clustering": true,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": true,
        "nerveJointAnchor": false,
        "multiomeAssembly": false,
        "depthNormalizedAtac": false,
        "regulonInference": false,
        "cd8SubsetIntegration": true,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": true,
        "orthologProjection": true,
        "epiSubsetSelection": true,
        "bidirectionalLrTest": true,
        "enrichmentFamilyRanking": true,
        "mousePseudobulkDesign": false,
        "cancerEpithelialDenominator": true,
        "geneLevelDe": false,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "trajectoryFile": "scbench-long/trajectories/gse155381_q7_xspecies_epi_communication__claude-sonnet-4-6__pi__r1.json",
      "runId": "claude-sonnet-4-6__pi__gse155381_q7_xspecies_epi_communication__r1",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-sonnet-4-6-pi",
      "modelName": "claude-sonnet-4-6",
      "harness": "pi",
      "task": "gse155381_q7_xspecies_epi_communication",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 2,
      "passed": false,
      "score": 3.75,
      "durationS": 1911.75,
      "cost": 3.1927210500000007,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "richness_weighted",
            "role": "additive",
            "field": "$.xspecies_signaling_richness",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_weighted",
            "role": "additive",
            "field": "$.dominant_signaling_axis",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_wnt_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "WNT_FZD_LRP_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_fgf_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "FGF_FGFR_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_notch_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "NOTCH_DLL_JAG_family"
                ]
              }
            }
          },
          {
            "name": "signaling_directionality_weighted",
            "role": "additive",
            "field": "$.signaling_directionality",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "top_signal_interaction_class_weighted",
            "role": "additive",
            "field": "$.top_signal_interaction_class",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          }
        ]
      },
      "reasoning": "all_of [pass_rule=score_threshold]: FAIL\n  + richness_weighted: passed=True, score=1.5\n  + dominant_axis_weighted: passed=True, score=0.25\n  + signaling_directionality_weighted: passed=True, score=1.0\n  + top_signal_interaction_class_weighted: passed=True, score=1.0\n  + hard_fail 'dominant_axis_wnt_unsupported_by_own_pairs': not triggered\n  + hard_fail 'dominant_axis_fgf_unsupported_by_own_pairs': not triggered\n  + hard_fail 'dominant_axis_notch_unsupported_by_own_pairs': not triggered",
      "metrics": {
        "pass_rule": "score_threshold",
        "scoring_count": 4,
        "scoring_passed": 4,
        "scoring_total_score": 3.75,
        "hard_fail_triggered": []
      },
      "fieldScores": {
        "richness_weighted": 1.5,
        "dominant_axis_weighted": 0.25,
        "signaling_directionality_weighted": 1,
        "top_signal_interaction_class_weighted": 1
      },
      "graderChecks": [
        {
          "name": "richness_weighted",
          "kind": "gate",
          "passed": true,
          "score": 1.5
        },
        {
          "name": "dominant_axis_weighted",
          "kind": "gate",
          "passed": true,
          "score": 0.25
        },
        {
          "name": "signaling_directionality_weighted",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "top_signal_interaction_class_weighted",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "dominant_axis_wnt_unsupported_by_own_pairs",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "dominant_axis_fgf_unsupported_by_own_pairs",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "dominant_axis_notch_unsupported_by_own_pairs",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "dominant_axis_support": [
          "VCAN",
          "ITGB1",
          "FN1",
          "FLT4",
          "CD44",
          "COL1A1",
          "COL1A2",
          "ITGA2B"
        ],
        "dominant_signaling_axis": "ECM_collagen_integrin_family",
        "signaling_directionality": "substantial_signal_in_both_directions",
        "top_signal_interaction_class": "protein_ligand_receptor_dominant",
        "xspecies_signaling_richness": "many_significant_pairs_across_multiple_signalling_families"
      },
      "nTurns": 66,
      "checkpointSignals": {
        "clustering": true,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": true,
        "nerveJointAnchor": false,
        "multiomeAssembly": true,
        "depthNormalizedAtac": false,
        "regulonInference": false,
        "cd8SubsetIntegration": true,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": true,
        "orthologProjection": true,
        "epiSubsetSelection": true,
        "bidirectionalLrTest": true,
        "enrichmentFamilyRanking": true,
        "mousePseudobulkDesign": false,
        "cancerEpithelialDenominator": true,
        "geneLevelDe": true,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "trajectoryFile": "scbench-long/trajectories/gse155381_q7_xspecies_epi_communication__claude-sonnet-4-6__pi__r2.json",
      "runId": "claude-sonnet-4-6__pi__gse155381_q7_xspecies_epi_communication__r2",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-sonnet-4-6-pi",
      "modelName": "claude-sonnet-4-6",
      "harness": "pi",
      "task": "gse155381_q7_xspecies_epi_communication",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 3,
      "passed": false,
      "score": 3.75,
      "durationS": 2722.06,
      "cost": 3.9359955,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "richness_weighted",
            "role": "additive",
            "field": "$.xspecies_signaling_richness",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_weighted",
            "role": "additive",
            "field": "$.dominant_signaling_axis",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_wnt_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "WNT_FZD_LRP_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_fgf_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "FGF_FGFR_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_notch_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "NOTCH_DLL_JAG_family"
                ]
              }
            }
          },
          {
            "name": "signaling_directionality_weighted",
            "role": "additive",
            "field": "$.signaling_directionality",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "top_signal_interaction_class_weighted",
            "role": "additive",
            "field": "$.top_signal_interaction_class",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          }
        ]
      },
      "reasoning": "all_of [pass_rule=score_threshold]: FAIL\n  + richness_weighted: passed=True, score=1.5\n  + dominant_axis_weighted: passed=True, score=0.75\n  + signaling_directionality_weighted: passed=True, score=0.5\n  + top_signal_interaction_class_weighted: passed=True, score=1.0\n  + hard_fail 'dominant_axis_wnt_unsupported_by_own_pairs': not triggered\n  + hard_fail 'dominant_axis_fgf_unsupported_by_own_pairs': not triggered\n  + hard_fail 'dominant_axis_notch_unsupported_by_own_pairs': not triggered",
      "metrics": {
        "pass_rule": "score_threshold",
        "scoring_count": 4,
        "scoring_passed": 4,
        "scoring_total_score": 3.75,
        "hard_fail_triggered": []
      },
      "fieldScores": {
        "richness_weighted": 1.5,
        "dominant_axis_weighted": 0.75,
        "signaling_directionality_weighted": 0.5,
        "top_signal_interaction_class_weighted": 1
      },
      "graderChecks": [
        {
          "name": "richness_weighted",
          "kind": "gate",
          "passed": true,
          "score": 1.5
        },
        {
          "name": "dominant_axis_weighted",
          "kind": "gate",
          "passed": true,
          "score": 0.75
        },
        {
          "name": "signaling_directionality_weighted",
          "kind": "gate",
          "passed": true,
          "score": 0.5
        },
        {
          "name": "top_signal_interaction_class_weighted",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "dominant_axis_wnt_unsupported_by_own_pairs",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "dominant_axis_fgf_unsupported_by_own_pairs",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "dominant_axis_notch_unsupported_by_own_pairs",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "dominant_axis_support": [
          "MDK",
          "NCL",
          "PTPRZ1",
          "SDC1",
          "SDC4",
          "SDC3",
          "GPC2",
          "NOTCH2",
          "LRP1",
          "ALK"
        ],
        "dominant_signaling_axis": "secreted_growth_factor_RTK_family",
        "signaling_directionality": "predominantly_chimera_human_to_chimera_monkey",
        "top_signal_interaction_class": "protein_ligand_receptor_dominant",
        "xspecies_signaling_richness": "many_significant_pairs_across_multiple_signalling_families"
      },
      "nTurns": 82,
      "checkpointSignals": {
        "clustering": true,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": true,
        "nerveJointAnchor": false,
        "multiomeAssembly": false,
        "depthNormalizedAtac": false,
        "regulonInference": false,
        "cd8SubsetIntegration": true,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": true,
        "orthologProjection": true,
        "epiSubsetSelection": true,
        "bidirectionalLrTest": true,
        "enrichmentFamilyRanking": true,
        "mousePseudobulkDesign": true,
        "cancerEpithelialDenominator": true,
        "geneLevelDe": true,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "trajectoryFile": "scbench-long/trajectories/gse155381_q7_xspecies_epi_communication__claude-sonnet-4-6__pi__r3.json",
      "runId": "claude-sonnet-4-6__pi__gse155381_q7_xspecies_epi_communication__r3",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-gemini-3-1-pro-preview-pi",
      "modelName": "gemini-3.1-pro-preview",
      "harness": "pi",
      "task": "gse155381_q7_xspecies_epi_communication",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 1,
      "passed": false,
      "score": 3.75,
      "durationS": 1453.29,
      "cost": 1.2852881999999997,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "richness_weighted",
            "role": "additive",
            "field": "$.xspecies_signaling_richness",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_weighted",
            "role": "additive",
            "field": "$.dominant_signaling_axis",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_wnt_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "WNT_FZD_LRP_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_fgf_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "FGF_FGFR_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_notch_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "NOTCH_DLL_JAG_family"
                ]
              }
            }
          },
          {
            "name": "signaling_directionality_weighted",
            "role": "additive",
            "field": "$.signaling_directionality",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "top_signal_interaction_class_weighted",
            "role": "additive",
            "field": "$.top_signal_interaction_class",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          }
        ]
      },
      "reasoning": "all_of [pass_rule=score_threshold]: FAIL\n  + richness_weighted: passed=True, score=1.5\n  + dominant_axis_weighted: passed=True, score=0.25\n  + signaling_directionality_weighted: passed=True, score=1.0\n  + top_signal_interaction_class_weighted: passed=True, score=1.0\n  + hard_fail 'dominant_axis_wnt_unsupported_by_own_pairs': not triggered\n  + hard_fail 'dominant_axis_fgf_unsupported_by_own_pairs': not triggered\n  + hard_fail 'dominant_axis_notch_unsupported_by_own_pairs': not triggered",
      "metrics": {
        "pass_rule": "score_threshold",
        "scoring_count": 4,
        "scoring_passed": 4,
        "scoring_total_score": 3.75,
        "hard_fail_triggered": []
      },
      "fieldScores": {
        "richness_weighted": 1.5,
        "dominant_axis_weighted": 0.25,
        "signaling_directionality_weighted": 1,
        "top_signal_interaction_class_weighted": 1
      },
      "graderChecks": [
        {
          "name": "richness_weighted",
          "kind": "gate",
          "passed": true,
          "score": 1.5
        },
        {
          "name": "dominant_axis_weighted",
          "kind": "gate",
          "passed": true,
          "score": 0.25
        },
        {
          "name": "signaling_directionality_weighted",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "top_signal_interaction_class_weighted",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "dominant_axis_wnt_unsupported_by_own_pairs",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "dominant_axis_fgf_unsupported_by_own_pairs",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "dominant_axis_notch_unsupported_by_own_pairs",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "dominant_axis_support": [
          "LAMA4",
          "ITGA6",
          "ITGB1",
          "ITGB4",
          "LAMA1",
          "COL3A1",
          "ITGA10",
          "COL5A2",
          "DDR1",
          "FBN1",
          "ITGA5",
          "COL6A2",
          "ITGAV",
          "ITGB8",
          "SPP1",
          "ITGA4",
          "LAMA5",
          "ITGA7",
          "FN1",
          "SDC2"
        ],
        "dominant_signaling_axis": "ECM_collagen_integrin_family",
        "signaling_directionality": "substantial_signal_in_both_directions",
        "top_signal_interaction_class": "protein_ligand_receptor_dominant",
        "xspecies_signaling_richness": "many_significant_pairs_across_multiple_signalling_families"
      },
      "nTurns": 88,
      "checkpointSignals": {
        "clustering": true,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": true,
        "nerveJointAnchor": false,
        "multiomeAssembly": false,
        "depthNormalizedAtac": false,
        "regulonInference": false,
        "cd8SubsetIntegration": false,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": true,
        "orthologProjection": true,
        "epiSubsetSelection": true,
        "bidirectionalLrTest": true,
        "enrichmentFamilyRanking": true,
        "mousePseudobulkDesign": false,
        "cancerEpithelialDenominator": false,
        "geneLevelDe": false,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "trajectoryFile": "scbench-long/trajectories/gse155381_q7_xspecies_epi_communication__gemini-3.1-pro-preview__pi__r1.json",
      "runId": "gemini-3.1-pro-preview__pi__gse155381_q7_xspecies_epi_communication__r1",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-gemini-3-1-pro-preview-pi",
      "modelName": "gemini-3.1-pro-preview",
      "harness": "pi",
      "task": "gse155381_q7_xspecies_epi_communication",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 2,
      "passed": false,
      "score": 3.75,
      "durationS": 448.08,
      "cost": 0.74283,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "richness_weighted",
            "role": "additive",
            "field": "$.xspecies_signaling_richness",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_weighted",
            "role": "additive",
            "field": "$.dominant_signaling_axis",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_wnt_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "WNT_FZD_LRP_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_fgf_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "FGF_FGFR_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_notch_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "NOTCH_DLL_JAG_family"
                ]
              }
            }
          },
          {
            "name": "signaling_directionality_weighted",
            "role": "additive",
            "field": "$.signaling_directionality",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "top_signal_interaction_class_weighted",
            "role": "additive",
            "field": "$.top_signal_interaction_class",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          }
        ]
      },
      "reasoning": "all_of [pass_rule=score_threshold]: FAIL\n  + richness_weighted: passed=True, score=1.5\n  + dominant_axis_weighted: passed=True, score=0.25\n  + signaling_directionality_weighted: passed=True, score=1.0\n  + top_signal_interaction_class_weighted: passed=True, score=1.0\n  + hard_fail 'dominant_axis_wnt_unsupported_by_own_pairs': not triggered\n  + hard_fail 'dominant_axis_fgf_unsupported_by_own_pairs': not triggered\n  + hard_fail 'dominant_axis_notch_unsupported_by_own_pairs': not triggered",
      "metrics": {
        "pass_rule": "score_threshold",
        "scoring_count": 4,
        "scoring_passed": 4,
        "scoring_total_score": 3.75,
        "hard_fail_triggered": []
      },
      "fieldScores": {
        "richness_weighted": 1.5,
        "dominant_axis_weighted": 0.25,
        "signaling_directionality_weighted": 1,
        "top_signal_interaction_class_weighted": 1
      },
      "graderChecks": [
        {
          "name": "richness_weighted",
          "kind": "gate",
          "passed": true,
          "score": 1.5
        },
        {
          "name": "dominant_axis_weighted",
          "kind": "gate",
          "passed": true,
          "score": 0.25
        },
        {
          "name": "signaling_directionality_weighted",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "top_signal_interaction_class_weighted",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "dominant_axis_wnt_unsupported_by_own_pairs",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "dominant_axis_fgf_unsupported_by_own_pairs",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "dominant_axis_notch_unsupported_by_own_pairs",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "dominant_axis_support": [
          "SDC1",
          "COL9A2",
          "FBLN1",
          "COL1A2",
          "ITGA4",
          "ITGB5",
          "ITGA5",
          "LAMA1",
          "ITGA1",
          "LAMB2",
          "LGALS1",
          "ITGA6",
          "RPSA",
          "SPP1",
          "SDC2",
          "SDC4",
          "COL3A1",
          "ITGB8",
          "LUM",
          "MMP11",
          "LAMA2",
          "COL11A1",
          "COL1A1",
          "SDC3",
          "ITGB1",
          "FN1",
          "PLAU",
          "DDR2",
          "ITGAV",
          "MDK"
        ],
        "dominant_signaling_axis": "ECM_collagen_integrin_family",
        "signaling_directionality": "substantial_signal_in_both_directions",
        "top_signal_interaction_class": "protein_ligand_receptor_dominant",
        "xspecies_signaling_richness": "many_significant_pairs_across_multiple_signalling_families"
      },
      "nTurns": 44,
      "checkpointSignals": {
        "clustering": false,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": true,
        "nerveJointAnchor": false,
        "multiomeAssembly": false,
        "depthNormalizedAtac": false,
        "regulonInference": false,
        "cd8SubsetIntegration": false,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": true,
        "orthologProjection": true,
        "epiSubsetSelection": true,
        "bidirectionalLrTest": true,
        "enrichmentFamilyRanking": true,
        "mousePseudobulkDesign": false,
        "cancerEpithelialDenominator": false,
        "geneLevelDe": false,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "trajectoryFile": "scbench-long/trajectories/gse155381_q7_xspecies_epi_communication__gemini-3.1-pro-preview__pi__r2.json",
      "runId": "gemini-3.1-pro-preview__pi__gse155381_q7_xspecies_epi_communication__r2",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-gemini-3-1-pro-preview-pi",
      "modelName": "gemini-3.1-pro-preview",
      "harness": "pi",
      "task": "gse155381_q7_xspecies_epi_communication",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 3,
      "passed": true,
      "score": 6,
      "durationS": 675.22,
      "cost": 0.9789024000000001,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "richness_weighted",
            "role": "additive",
            "field": "$.xspecies_signaling_richness",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_weighted",
            "role": "additive",
            "field": "$.dominant_signaling_axis",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_wnt_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "WNT_FZD_LRP_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_fgf_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "FGF_FGFR_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_notch_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "NOTCH_DLL_JAG_family"
                ]
              }
            }
          },
          {
            "name": "signaling_directionality_weighted",
            "role": "additive",
            "field": "$.signaling_directionality",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "top_signal_interaction_class_weighted",
            "role": "additive",
            "field": "$.top_signal_interaction_class",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          }
        ]
      },
      "reasoning": "all_of [pass_rule=score_threshold]: PASS\n  + richness_weighted: passed=True, score=1.5\n  + dominant_axis_weighted: passed=True, score=2.5\n  + signaling_directionality_weighted: passed=True, score=1.0\n  + top_signal_interaction_class_weighted: passed=True, score=1.0\n  + hard_fail 'dominant_axis_wnt_unsupported_by_own_pairs': not triggered\n  + hard_fail 'dominant_axis_fgf_unsupported_by_own_pairs': not triggered\n  + hard_fail 'dominant_axis_notch_unsupported_by_own_pairs': not triggered",
      "metrics": {
        "pass_rule": "score_threshold",
        "scoring_count": 4,
        "scoring_passed": 4,
        "scoring_total_score": 6,
        "hard_fail_triggered": []
      },
      "fieldScores": {
        "richness_weighted": 1.5,
        "dominant_axis_weighted": 2.5,
        "signaling_directionality_weighted": 1,
        "top_signal_interaction_class_weighted": 1
      },
      "graderChecks": [
        {
          "name": "richness_weighted",
          "kind": "gate",
          "passed": true,
          "score": 1.5
        },
        {
          "name": "dominant_axis_weighted",
          "kind": "gate",
          "passed": true,
          "score": 2.5
        },
        {
          "name": "signaling_directionality_weighted",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "top_signal_interaction_class_weighted",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "dominant_axis_wnt_unsupported_by_own_pairs",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "dominant_axis_fgf_unsupported_by_own_pairs",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "dominant_axis_notch_unsupported_by_own_pairs",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "dominant_axis_support": [
          "APOE",
          "LDLR",
          "BSG",
          "SLC16A1",
          "ARPC5",
          "LRPAP1",
          "MDK",
          "ITGA4",
          "ITGB1",
          "SCARB1",
          "CALR",
          "ITGAV",
          "FAM3C",
          "LAMP1",
          "HMGB1",
          "SDC1",
          "LAMA2",
          "RPSA",
          "LAMA1",
          "SDC4",
          "GSTP1",
          "TRAF2",
          "HSPA8",
          "GPC2",
          "HLA-A",
          "APLP2",
          "ARF1",
          "PLD2",
          "PLAU",
          "AGTRAP",
          "RACK1"
        ],
        "dominant_signaling_axis": "two_or_more_families_co_dominant",
        "signaling_directionality": "substantial_signal_in_both_directions",
        "top_signal_interaction_class": "protein_ligand_receptor_dominant",
        "xspecies_signaling_richness": "many_significant_pairs_across_multiple_signalling_families"
      },
      "nTurns": 54,
      "checkpointSignals": {
        "clustering": false,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": true,
        "nerveJointAnchor": false,
        "multiomeAssembly": false,
        "depthNormalizedAtac": false,
        "regulonInference": false,
        "cd8SubsetIntegration": false,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": true,
        "orthologProjection": true,
        "epiSubsetSelection": true,
        "bidirectionalLrTest": true,
        "enrichmentFamilyRanking": true,
        "mousePseudobulkDesign": false,
        "cancerEpithelialDenominator": false,
        "geneLevelDe": false,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "trajectoryFile": "scbench-long/trajectories/gse155381_q7_xspecies_epi_communication__gemini-3.1-pro-preview__pi__r3.json",
      "runId": "gemini-3.1-pro-preview__pi__gse155381_q7_xspecies_epi_communication__r3",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-gemini-3-5-flash-pi",
      "modelName": "gemini-3.5-flash",
      "harness": "pi",
      "task": "gse155381_q7_xspecies_epi_communication",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 1,
      "passed": false,
      "score": 4.25,
      "durationS": 666.21,
      "cost": 1.48338465,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "richness_weighted",
            "role": "additive",
            "field": "$.xspecies_signaling_richness",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_weighted",
            "role": "additive",
            "field": "$.dominant_signaling_axis",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_wnt_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "WNT_FZD_LRP_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_fgf_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "FGF_FGFR_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_notch_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "NOTCH_DLL_JAG_family"
                ]
              }
            }
          },
          {
            "name": "signaling_directionality_weighted",
            "role": "additive",
            "field": "$.signaling_directionality",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "top_signal_interaction_class_weighted",
            "role": "additive",
            "field": "$.top_signal_interaction_class",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          }
        ]
      },
      "reasoning": "all_of [pass_rule=score_threshold]: FAIL\n  + richness_weighted: passed=True, score=1.5\n  + dominant_axis_weighted: passed=True, score=0.75\n  + signaling_directionality_weighted: passed=True, score=1.0\n  + top_signal_interaction_class_weighted: passed=True, score=1.0\n  + hard_fail 'dominant_axis_wnt_unsupported_by_own_pairs': not triggered\n  + hard_fail 'dominant_axis_fgf_unsupported_by_own_pairs': not triggered\n  + hard_fail 'dominant_axis_notch_unsupported_by_own_pairs': not triggered",
      "metrics": {
        "pass_rule": "score_threshold",
        "scoring_count": 4,
        "scoring_passed": 4,
        "scoring_total_score": 4.25,
        "hard_fail_triggered": []
      },
      "fieldScores": {
        "richness_weighted": 1.5,
        "dominant_axis_weighted": 0.75,
        "signaling_directionality_weighted": 1,
        "top_signal_interaction_class_weighted": 1
      },
      "graderChecks": [
        {
          "name": "richness_weighted",
          "kind": "gate",
          "passed": true,
          "score": 1.5
        },
        {
          "name": "dominant_axis_weighted",
          "kind": "gate",
          "passed": true,
          "score": 0.75
        },
        {
          "name": "signaling_directionality_weighted",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "top_signal_interaction_class_weighted",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "dominant_axis_wnt_unsupported_by_own_pairs",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "dominant_axis_fgf_unsupported_by_own_pairs",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "dominant_axis_notch_unsupported_by_own_pairs",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "dominant_axis_support": [
          "EFNA1",
          "EFNA2",
          "EFNA3",
          "EFNA4",
          "EFNA5",
          "EFNB1",
          "EPHA1",
          "EPHA2",
          "EPHA4",
          "EPHA5",
          "EPHA7",
          "EPHB1",
          "EPHB2",
          "EPHB3",
          "EPHB4"
        ],
        "dominant_signaling_axis": "ephrin_EPH_axon_guidance_family",
        "signaling_directionality": "substantial_signal_in_both_directions",
        "top_signal_interaction_class": "protein_ligand_receptor_dominant",
        "xspecies_signaling_richness": "many_significant_pairs_across_multiple_signalling_families"
      },
      "nTurns": 87,
      "checkpointSignals": {
        "clustering": false,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": true,
        "nerveJointAnchor": false,
        "multiomeAssembly": true,
        "depthNormalizedAtac": false,
        "regulonInference": false,
        "cd8SubsetIntegration": true,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": true,
        "orthologProjection": true,
        "epiSubsetSelection": true,
        "bidirectionalLrTest": true,
        "enrichmentFamilyRanking": true,
        "mousePseudobulkDesign": true,
        "cancerEpithelialDenominator": true,
        "geneLevelDe": true,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "trajectoryFile": "scbench-long/trajectories/gse155381_q7_xspecies_epi_communication__gemini-3.5-flash__pi__r1.json",
      "runId": "gemini-3.5-flash__pi__gse155381_q7_xspecies_epi_communication__r1",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-gemini-3-5-flash-pi",
      "modelName": "gemini-3.5-flash",
      "harness": "pi",
      "task": "gse155381_q7_xspecies_epi_communication",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 2,
      "passed": false,
      "score": 4.25,
      "durationS": 1111.37,
      "cost": 2.1073240500000003,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "richness_weighted",
            "role": "additive",
            "field": "$.xspecies_signaling_richness",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_weighted",
            "role": "additive",
            "field": "$.dominant_signaling_axis",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_wnt_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "WNT_FZD_LRP_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_fgf_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "FGF_FGFR_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_notch_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "NOTCH_DLL_JAG_family"
                ]
              }
            }
          },
          {
            "name": "signaling_directionality_weighted",
            "role": "additive",
            "field": "$.signaling_directionality",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "top_signal_interaction_class_weighted",
            "role": "additive",
            "field": "$.top_signal_interaction_class",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          }
        ]
      },
      "reasoning": "all_of [pass_rule=score_threshold]: FAIL\n  + richness_weighted: passed=True, score=1.5\n  + dominant_axis_weighted: passed=True, score=0.75\n  + signaling_directionality_weighted: passed=True, score=1.0\n  + top_signal_interaction_class_weighted: passed=True, score=1.0\n  + hard_fail 'dominant_axis_wnt_unsupported_by_own_pairs': not triggered\n  + hard_fail 'dominant_axis_fgf_unsupported_by_own_pairs': not triggered\n  + hard_fail 'dominant_axis_notch_unsupported_by_own_pairs': not triggered",
      "metrics": {
        "pass_rule": "score_threshold",
        "scoring_count": 4,
        "scoring_passed": 4,
        "scoring_total_score": 4.25,
        "hard_fail_triggered": []
      },
      "fieldScores": {
        "richness_weighted": 1.5,
        "dominant_axis_weighted": 0.75,
        "signaling_directionality_weighted": 1,
        "top_signal_interaction_class_weighted": 1
      },
      "graderChecks": [
        {
          "name": "richness_weighted",
          "kind": "gate",
          "passed": true,
          "score": 1.5
        },
        {
          "name": "dominant_axis_weighted",
          "kind": "gate",
          "passed": true,
          "score": 0.75
        },
        {
          "name": "signaling_directionality_weighted",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "top_signal_interaction_class_weighted",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "dominant_axis_wnt_unsupported_by_own_pairs",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "dominant_axis_fgf_unsupported_by_own_pairs",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "dominant_axis_notch_unsupported_by_own_pairs",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "dominant_axis_support": [
          "MDK",
          "NCL",
          "ITGA6",
          "ITGB1",
          "SDC2",
          "PTPRZ1",
          "LRP1",
          "SORL1",
          "NOTCH2",
          "SDC3",
          "MIF",
          "EGFR",
          "CD44",
          "CD74",
          "CXCR4"
        ],
        "dominant_signaling_axis": "secreted_growth_factor_RTK_family",
        "signaling_directionality": "substantial_signal_in_both_directions",
        "top_signal_interaction_class": "protein_ligand_receptor_dominant",
        "xspecies_signaling_richness": "many_significant_pairs_across_multiple_signalling_families"
      },
      "nTurns": 92,
      "checkpointSignals": {
        "clustering": false,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": true,
        "nerveJointAnchor": false,
        "multiomeAssembly": false,
        "depthNormalizedAtac": false,
        "regulonInference": false,
        "cd8SubsetIntegration": true,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": true,
        "orthologProjection": true,
        "epiSubsetSelection": true,
        "bidirectionalLrTest": true,
        "enrichmentFamilyRanking": true,
        "mousePseudobulkDesign": false,
        "cancerEpithelialDenominator": true,
        "geneLevelDe": false,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "trajectoryFile": "scbench-long/trajectories/gse155381_q7_xspecies_epi_communication__gemini-3.5-flash__pi__r2.json",
      "runId": "gemini-3.5-flash__pi__gse155381_q7_xspecies_epi_communication__r2",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-gemini-3-5-flash-pi",
      "modelName": "gemini-3.5-flash",
      "harness": "pi",
      "task": "gse155381_q7_xspecies_epi_communication",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 3,
      "passed": false,
      "score": 3.75,
      "durationS": 2023.55,
      "cost": 1.1352933,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "richness_weighted",
            "role": "additive",
            "field": "$.xspecies_signaling_richness",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_weighted",
            "role": "additive",
            "field": "$.dominant_signaling_axis",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_wnt_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "WNT_FZD_LRP_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_fgf_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "FGF_FGFR_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_notch_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "NOTCH_DLL_JAG_family"
                ]
              }
            }
          },
          {
            "name": "signaling_directionality_weighted",
            "role": "additive",
            "field": "$.signaling_directionality",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "top_signal_interaction_class_weighted",
            "role": "additive",
            "field": "$.top_signal_interaction_class",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          }
        ]
      },
      "reasoning": "all_of [pass_rule=score_threshold]: FAIL\n  + richness_weighted: passed=True, score=1.5\n  + dominant_axis_weighted: passed=True, score=0.25\n  + signaling_directionality_weighted: passed=True, score=1.0\n  + top_signal_interaction_class_weighted: passed=True, score=1.0\n  + hard_fail 'dominant_axis_wnt_unsupported_by_own_pairs': not triggered\n  + hard_fail 'dominant_axis_fgf_unsupported_by_own_pairs': not triggered\n  + hard_fail 'dominant_axis_notch_unsupported_by_own_pairs': not triggered",
      "metrics": {
        "pass_rule": "score_threshold",
        "scoring_count": 4,
        "scoring_passed": 4,
        "scoring_total_score": 3.75,
        "hard_fail_triggered": []
      },
      "fieldScores": {
        "richness_weighted": 1.5,
        "dominant_axis_weighted": 0.25,
        "signaling_directionality_weighted": 1,
        "top_signal_interaction_class_weighted": 1
      },
      "graderChecks": [
        {
          "name": "richness_weighted",
          "kind": "gate",
          "passed": true,
          "score": 1.5
        },
        {
          "name": "dominant_axis_weighted",
          "kind": "gate",
          "passed": true,
          "score": 0.25
        },
        {
          "name": "signaling_directionality_weighted",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "top_signal_interaction_class_weighted",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "dominant_axis_wnt_unsupported_by_own_pairs",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "dominant_axis_fgf_unsupported_by_own_pairs",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "dominant_axis_notch_unsupported_by_own_pairs",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "dominant_axis_support": [
          "COL1A2",
          "ITGB1",
          "SDC1",
          "ITGAV",
          "ITGA4",
          "ITGB8",
          "DDR2",
          "SDC4",
          "SDC3",
          "ITGA1",
          "LAMA1",
          "LAMB2",
          "RPSA",
          "APP",
          "VCAN",
          "MDK",
          "CD44",
          "SPARC"
        ],
        "dominant_signaling_axis": "ECM_collagen_integrin_family",
        "signaling_directionality": "substantial_signal_in_both_directions",
        "top_signal_interaction_class": "protein_ligand_receptor_dominant",
        "xspecies_signaling_richness": "many_significant_pairs_across_multiple_signalling_families"
      },
      "nTurns": 61,
      "checkpointSignals": {
        "clustering": true,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": true,
        "nerveJointAnchor": false,
        "multiomeAssembly": false,
        "depthNormalizedAtac": false,
        "regulonInference": false,
        "cd8SubsetIntegration": false,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": true,
        "orthologProjection": true,
        "epiSubsetSelection": true,
        "bidirectionalLrTest": true,
        "enrichmentFamilyRanking": true,
        "mousePseudobulkDesign": false,
        "cancerEpithelialDenominator": false,
        "geneLevelDe": false,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "trajectoryFile": "scbench-long/trajectories/gse155381_q7_xspecies_epi_communication__gemini-3.5-flash__pi__r3.json",
      "runId": "gemini-3.5-flash__pi__gse155381_q7_xspecies_epi_communication__r3",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-glm-5p2-pi",
      "modelName": "glm-5p2",
      "harness": "pi",
      "task": "gse155381_q7_xspecies_epi_communication",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 1,
      "passed": false,
      "score": 3.5,
      "durationS": 1449.77,
      "cost": 0.7498281600000001,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "richness_weighted",
            "role": "additive",
            "field": "$.xspecies_signaling_richness",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_weighted",
            "role": "additive",
            "field": "$.dominant_signaling_axis",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_wnt_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "WNT_FZD_LRP_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_fgf_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "FGF_FGFR_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_notch_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "NOTCH_DLL_JAG_family"
                ]
              }
            }
          },
          {
            "name": "signaling_directionality_weighted",
            "role": "additive",
            "field": "$.signaling_directionality",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "top_signal_interaction_class_weighted",
            "role": "additive",
            "field": "$.top_signal_interaction_class",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          }
        ]
      },
      "reasoning": "all_of [pass_rule=score_threshold]: FAIL\n  + richness_weighted: passed=True, score=1.5\n  + dominant_axis_weighted: passed=True, score=0.75\n  + signaling_directionality_weighted: passed=True, score=0.25\n  + top_signal_interaction_class_weighted: passed=True, score=1.0\n  + hard_fail 'dominant_axis_wnt_unsupported_by_own_pairs': not triggered\n  + hard_fail 'dominant_axis_fgf_unsupported_by_own_pairs': not triggered\n  + hard_fail 'dominant_axis_notch_unsupported_by_own_pairs': not triggered",
      "metrics": {
        "pass_rule": "score_threshold",
        "scoring_count": 4,
        "scoring_passed": 4,
        "scoring_total_score": 3.5,
        "hard_fail_triggered": []
      },
      "fieldScores": {
        "richness_weighted": 1.5,
        "dominant_axis_weighted": 0.75,
        "signaling_directionality_weighted": 0.25,
        "top_signal_interaction_class_weighted": 1
      },
      "graderChecks": [
        {
          "name": "richness_weighted",
          "kind": "gate",
          "passed": true,
          "score": 1.5
        },
        {
          "name": "dominant_axis_weighted",
          "kind": "gate",
          "passed": true,
          "score": 0.75
        },
        {
          "name": "signaling_directionality_weighted",
          "kind": "gate",
          "passed": true,
          "score": 0.25
        },
        {
          "name": "top_signal_interaction_class_weighted",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "dominant_axis_wnt_unsupported_by_own_pairs",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "dominant_axis_fgf_unsupported_by_own_pairs",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "dominant_axis_notch_unsupported_by_own_pairs",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "dominant_axis_support": [
          "MDK",
          "NCL",
          "SDC2",
          "LRP1",
          "SORL1",
          "LRP2"
        ],
        "dominant_signaling_axis": "secreted_growth_factor_RTK_family",
        "signaling_directionality": "bidirectional_but_via_distinct_families_in_each_direction",
        "top_signal_interaction_class": "protein_ligand_receptor_dominant",
        "xspecies_signaling_richness": "many_significant_pairs_across_multiple_signalling_families"
      },
      "nTurns": 97,
      "checkpointSignals": {
        "clustering": true,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": true,
        "nerveJointAnchor": false,
        "multiomeAssembly": false,
        "depthNormalizedAtac": false,
        "regulonInference": false,
        "cd8SubsetIntegration": false,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": true,
        "orthologProjection": true,
        "epiSubsetSelection": true,
        "bidirectionalLrTest": true,
        "enrichmentFamilyRanking": true,
        "mousePseudobulkDesign": false,
        "cancerEpithelialDenominator": true,
        "geneLevelDe": true,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "trajectoryFile": "scbench-long/trajectories/gse155381_q7_xspecies_epi_communication__glm-5p2__pi__r1.json",
      "runId": "glm-5p2__pi__gse155381_q7_xspecies_epi_communication__r1",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-glm-5p2-pi",
      "modelName": "glm-5p2",
      "harness": "pi",
      "task": "gse155381_q7_xspecies_epi_communication",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 2,
      "passed": false,
      "score": 3.5,
      "durationS": 1121.52,
      "cost": 0.5908144699999999,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "richness_weighted",
            "role": "additive",
            "field": "$.xspecies_signaling_richness",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_weighted",
            "role": "additive",
            "field": "$.dominant_signaling_axis",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_wnt_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "WNT_FZD_LRP_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_fgf_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "FGF_FGFR_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_notch_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "NOTCH_DLL_JAG_family"
                ]
              }
            }
          },
          {
            "name": "signaling_directionality_weighted",
            "role": "additive",
            "field": "$.signaling_directionality",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "top_signal_interaction_class_weighted",
            "role": "additive",
            "field": "$.top_signal_interaction_class",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          }
        ]
      },
      "reasoning": "all_of [pass_rule=score_threshold]: FAIL\n  + richness_weighted: passed=True, score=1.5\n  + dominant_axis_weighted: passed=True, score=0.0\n  + signaling_directionality_weighted: passed=True, score=1.0\n  + top_signal_interaction_class_weighted: passed=True, score=1.0\n  + hard_fail 'dominant_axis_wnt_unsupported_by_own_pairs': not triggered\n  + hard_fail 'dominant_axis_fgf_unsupported_by_own_pairs': not triggered\n  + hard_fail 'dominant_axis_notch_unsupported_by_own_pairs': not triggered",
      "metrics": {
        "pass_rule": "score_threshold",
        "scoring_count": 4,
        "scoring_passed": 4,
        "scoring_total_score": 3.5,
        "hard_fail_triggered": []
      },
      "fieldScores": {
        "richness_weighted": 1.5,
        "dominant_axis_weighted": 0,
        "signaling_directionality_weighted": 1,
        "top_signal_interaction_class_weighted": 1
      },
      "graderChecks": [
        {
          "name": "richness_weighted",
          "kind": "gate",
          "passed": true,
          "score": 1.5
        },
        {
          "name": "dominant_axis_weighted",
          "kind": "gate",
          "passed": true,
          "score": 0
        },
        {
          "name": "signaling_directionality_weighted",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "top_signal_interaction_class_weighted",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "dominant_axis_wnt_unsupported_by_own_pairs",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "dominant_axis_fgf_unsupported_by_own_pairs",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "dominant_axis_notch_unsupported_by_own_pairs",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "dominant_axis_support": [
          "BSG",
          "SLC16A1",
          "FAM3C",
          "LAMP1",
          "APOE",
          "LDLR",
          "LRPAP1",
          "TIMP1",
          "CD63",
          "CALM1",
          "HMMR",
          "ADAM10",
          "GPNMB",
          "HSPA8",
          "SCARB1",
          "ARF1",
          "PLD2",
          "AGTRAP",
          "RACK1",
          "CALM3",
          "RYR2",
          "GSTP1",
          "TRAF2",
          "PKM",
          "CD44",
          "CALR",
          "LRP1",
          "PSAP",
          "SORT1",
          "PTPRA",
          "GNAI2",
          "CAV1",
          "ARPC5",
          "ADRB2",
          "GRN",
          "CCN1",
          "HMGB1",
          "SDC1"
        ],
        "dominant_signaling_axis": "stress_or_metabolic_signalling_family",
        "signaling_directionality": "substantial_signal_in_both_directions",
        "top_signal_interaction_class": "protein_ligand_receptor_dominant",
        "xspecies_signaling_richness": "many_significant_pairs_across_multiple_signalling_families"
      },
      "nTurns": 61,
      "checkpointSignals": {
        "clustering": true,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": true,
        "nerveJointAnchor": false,
        "multiomeAssembly": false,
        "depthNormalizedAtac": false,
        "regulonInference": false,
        "cd8SubsetIntegration": false,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": true,
        "orthologProjection": true,
        "epiSubsetSelection": true,
        "bidirectionalLrTest": true,
        "enrichmentFamilyRanking": true,
        "mousePseudobulkDesign": false,
        "cancerEpithelialDenominator": true,
        "geneLevelDe": false,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "trajectoryFile": "scbench-long/trajectories/gse155381_q7_xspecies_epi_communication__glm-5p2__pi__r2.json",
      "runId": "glm-5p2__pi__gse155381_q7_xspecies_epi_communication__r2",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-glm-5p2-pi",
      "modelName": "glm-5p2",
      "harness": "pi",
      "task": "gse155381_q7_xspecies_epi_communication",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 3,
      "passed": false,
      "score": 3.5,
      "durationS": 866.45,
      "cost": 0.48311787999999994,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "richness_weighted",
            "role": "additive",
            "field": "$.xspecies_signaling_richness",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_weighted",
            "role": "additive",
            "field": "$.dominant_signaling_axis",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_wnt_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "WNT_FZD_LRP_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_fgf_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "FGF_FGFR_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_notch_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "NOTCH_DLL_JAG_family"
                ]
              }
            }
          },
          {
            "name": "signaling_directionality_weighted",
            "role": "additive",
            "field": "$.signaling_directionality",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "top_signal_interaction_class_weighted",
            "role": "additive",
            "field": "$.top_signal_interaction_class",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          }
        ]
      },
      "reasoning": "all_of [pass_rule=score_threshold]: FAIL\n  + richness_weighted: passed=True, score=1.5\n  + dominant_axis_weighted: passed=True, score=0.75\n  + signaling_directionality_weighted: passed=True, score=0.25\n  + top_signal_interaction_class_weighted: passed=True, score=1.0\n  + hard_fail 'dominant_axis_wnt_unsupported_by_own_pairs': not triggered\n  + hard_fail 'dominant_axis_fgf_unsupported_by_own_pairs': not triggered\n  + hard_fail 'dominant_axis_notch_unsupported_by_own_pairs': not triggered",
      "metrics": {
        "pass_rule": "score_threshold",
        "scoring_count": 4,
        "scoring_passed": 4,
        "scoring_total_score": 3.5,
        "hard_fail_triggered": []
      },
      "fieldScores": {
        "richness_weighted": 1.5,
        "dominant_axis_weighted": 0.75,
        "signaling_directionality_weighted": 0.25,
        "top_signal_interaction_class_weighted": 1
      },
      "graderChecks": [
        {
          "name": "richness_weighted",
          "kind": "gate",
          "passed": true,
          "score": 1.5
        },
        {
          "name": "dominant_axis_weighted",
          "kind": "gate",
          "passed": true,
          "score": 0.75
        },
        {
          "name": "signaling_directionality_weighted",
          "kind": "gate",
          "passed": true,
          "score": 0.25
        },
        {
          "name": "top_signal_interaction_class_weighted",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "dominant_axis_wnt_unsupported_by_own_pairs",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "dominant_axis_fgf_unsupported_by_own_pairs",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "dominant_axis_notch_unsupported_by_own_pairs",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "dominant_axis_support": [
          "MDK",
          "NCL",
          "ITGA6",
          "ITGB1",
          "PTPRZ1",
          "SDC2",
          "LRP1",
          "SORL1"
        ],
        "dominant_signaling_axis": "secreted_growth_factor_RTK_family",
        "signaling_directionality": "bidirectional_but_via_distinct_families_in_each_direction",
        "top_signal_interaction_class": "protein_ligand_receptor_dominant",
        "xspecies_signaling_richness": "many_significant_pairs_across_multiple_signalling_families"
      },
      "nTurns": 53,
      "checkpointSignals": {
        "clustering": true,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": true,
        "nerveJointAnchor": false,
        "multiomeAssembly": false,
        "depthNormalizedAtac": false,
        "regulonInference": false,
        "cd8SubsetIntegration": false,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": true,
        "orthologProjection": true,
        "epiSubsetSelection": true,
        "bidirectionalLrTest": true,
        "enrichmentFamilyRanking": true,
        "mousePseudobulkDesign": false,
        "cancerEpithelialDenominator": false,
        "geneLevelDe": false,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "trajectoryFile": "scbench-long/trajectories/gse155381_q7_xspecies_epi_communication__glm-5p2__pi__r3.json",
      "runId": "glm-5p2__pi__gse155381_q7_xspecies_epi_communication__r3",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-gpt-5-4-codex",
      "modelName": "gpt-5.4",
      "harness": "openai-codex",
      "task": "gse155381_q7_xspecies_epi_communication",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 1,
      "passed": false,
      "score": 3.25,
      "durationS": 1070.65,
      "cost": 2.2782470000000004,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "richness_weighted",
            "role": "additive",
            "field": "$.xspecies_signaling_richness",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_weighted",
            "role": "additive",
            "field": "$.dominant_signaling_axis",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_wnt_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "WNT_FZD_LRP_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_fgf_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "FGF_FGFR_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_notch_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "NOTCH_DLL_JAG_family"
                ]
              }
            }
          },
          {
            "name": "signaling_directionality_weighted",
            "role": "additive",
            "field": "$.signaling_directionality",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "top_signal_interaction_class_weighted",
            "role": "additive",
            "field": "$.top_signal_interaction_class",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          }
        ]
      },
      "reasoning": "all_of [pass_rule=score_threshold]: FAIL\n  + richness_weighted: passed=True, score=1.5\n  + dominant_axis_weighted: passed=True, score=0.25\n  + signaling_directionality_weighted: passed=True, score=0.5\n  + top_signal_interaction_class_weighted: passed=True, score=1.0\n  + hard_fail 'dominant_axis_wnt_unsupported_by_own_pairs': not triggered\n  + hard_fail 'dominant_axis_fgf_unsupported_by_own_pairs': not triggered\n  + hard_fail 'dominant_axis_notch_unsupported_by_own_pairs': not triggered",
      "metrics": {
        "pass_rule": "score_threshold",
        "scoring_count": 4,
        "scoring_passed": 4,
        "scoring_total_score": 3.25,
        "hard_fail_triggered": []
      },
      "fieldScores": {
        "richness_weighted": 1.5,
        "dominant_axis_weighted": 0.25,
        "signaling_directionality_weighted": 0.5,
        "top_signal_interaction_class_weighted": 1
      },
      "graderChecks": [
        {
          "name": "richness_weighted",
          "kind": "gate",
          "passed": true,
          "score": 1.5
        },
        {
          "name": "dominant_axis_weighted",
          "kind": "gate",
          "passed": true,
          "score": 0.25
        },
        {
          "name": "signaling_directionality_weighted",
          "kind": "gate",
          "passed": true,
          "score": 0.5
        },
        {
          "name": "top_signal_interaction_class_weighted",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "dominant_axis_wnt_unsupported_by_own_pairs",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "dominant_axis_fgf_unsupported_by_own_pairs",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "dominant_axis_notch_unsupported_by_own_pairs",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "dominant_axis_support": [
          "SPP1",
          "ITGA4",
          "FN1",
          "ITGA5",
          "COL3A1",
          "ITGA1",
          "ITGA11",
          "ITGA10",
          "ITGA2",
          "COL11A1",
          "LAMC1",
          "ITGA6"
        ],
        "dominant_signaling_axis": "ECM_collagen_integrin_family",
        "signaling_directionality": "predominantly_chimera_human_to_chimera_monkey",
        "top_signal_interaction_class": "protein_ligand_receptor_dominant",
        "xspecies_signaling_richness": "many_significant_pairs_across_multiple_signalling_families"
      },
      "nTurns": 1,
      "checkpointSignals": {
        "clustering": true,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": true,
        "nerveJointAnchor": false,
        "multiomeAssembly": true,
        "depthNormalizedAtac": false,
        "regulonInference": true,
        "cd8SubsetIntegration": false,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": true,
        "orthologProjection": true,
        "epiSubsetSelection": true,
        "bidirectionalLrTest": true,
        "enrichmentFamilyRanking": true,
        "mousePseudobulkDesign": false,
        "cancerEpithelialDenominator": true,
        "geneLevelDe": false,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "trajectoryFile": "scbench-long/trajectories/gse155381_q7_xspecies_epi_communication__gpt-5.4__openai-codex__r1.json",
      "runId": "gpt-5.4__openai-codex__gse155381_q7_xspecies_epi_communication__r1",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-gpt-5-4-codex",
      "modelName": "gpt-5.4",
      "harness": "openai-codex",
      "task": "gse155381_q7_xspecies_epi_communication",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 2,
      "passed": false,
      "score": 4.25,
      "durationS": 1540.32,
      "cost": 2.564128,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "richness_weighted",
            "role": "additive",
            "field": "$.xspecies_signaling_richness",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_weighted",
            "role": "additive",
            "field": "$.dominant_signaling_axis",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_wnt_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "WNT_FZD_LRP_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_fgf_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "FGF_FGFR_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_notch_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "NOTCH_DLL_JAG_family"
                ]
              }
            }
          },
          {
            "name": "signaling_directionality_weighted",
            "role": "additive",
            "field": "$.signaling_directionality",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "top_signal_interaction_class_weighted",
            "role": "additive",
            "field": "$.top_signal_interaction_class",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          }
        ]
      },
      "reasoning": "all_of [pass_rule=score_threshold]: FAIL\n  + richness_weighted: passed=True, score=1.5\n  + dominant_axis_weighted: passed=True, score=0.75\n  + signaling_directionality_weighted: passed=True, score=1.0\n  + top_signal_interaction_class_weighted: passed=True, score=1.0\n  + hard_fail 'dominant_axis_wnt_unsupported_by_own_pairs': not triggered\n  + hard_fail 'dominant_axis_fgf_unsupported_by_own_pairs': not triggered\n  + hard_fail 'dominant_axis_notch_unsupported_by_own_pairs': not triggered",
      "metrics": {
        "pass_rule": "score_threshold",
        "scoring_count": 4,
        "scoring_passed": 4,
        "scoring_total_score": 4.25,
        "hard_fail_triggered": []
      },
      "fieldScores": {
        "richness_weighted": 1.5,
        "dominant_axis_weighted": 0.75,
        "signaling_directionality_weighted": 1,
        "top_signal_interaction_class_weighted": 1
      },
      "graderChecks": [
        {
          "name": "richness_weighted",
          "kind": "gate",
          "passed": true,
          "score": 1.5
        },
        {
          "name": "dominant_axis_weighted",
          "kind": "gate",
          "passed": true,
          "score": 0.75
        },
        {
          "name": "signaling_directionality_weighted",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "top_signal_interaction_class_weighted",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "dominant_axis_wnt_unsupported_by_own_pairs",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "dominant_axis_fgf_unsupported_by_own_pairs",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "dominant_axis_notch_unsupported_by_own_pairs",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "dominant_axis_support": [
          "EFNA5",
          "EPHA8",
          "EPHA1",
          "EFNA4",
          "EFNA3",
          "EPHA2",
          "EPHA7",
          "EPHA4",
          "EPHB2",
          "EPHA5",
          "EFNA1",
          "EFNA2"
        ],
        "dominant_signaling_axis": "ephrin_EPH_axon_guidance_family",
        "signaling_directionality": "substantial_signal_in_both_directions",
        "top_signal_interaction_class": "protein_ligand_receptor_dominant",
        "xspecies_signaling_richness": "many_significant_pairs_across_multiple_signalling_families"
      },
      "nTurns": 1,
      "checkpointSignals": {
        "clustering": true,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": true,
        "nerveJointAnchor": false,
        "multiomeAssembly": false,
        "depthNormalizedAtac": false,
        "regulonInference": false,
        "cd8SubsetIntegration": false,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": true,
        "orthologProjection": true,
        "epiSubsetSelection": true,
        "bidirectionalLrTest": true,
        "enrichmentFamilyRanking": false,
        "mousePseudobulkDesign": false,
        "cancerEpithelialDenominator": true,
        "geneLevelDe": false,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "trajectoryFile": "scbench-long/trajectories/gse155381_q7_xspecies_epi_communication__gpt-5.4__openai-codex__r2.json",
      "runId": "gpt-5.4__openai-codex__gse155381_q7_xspecies_epi_communication__r2",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-gpt-5-4-codex",
      "modelName": "gpt-5.4",
      "harness": "openai-codex",
      "task": "gse155381_q7_xspecies_epi_communication",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 3,
      "passed": false,
      "score": 1.25,
      "durationS": 631.25,
      "cost": 1.2034595000000001,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "richness_weighted",
            "role": "additive",
            "field": "$.xspecies_signaling_richness",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_weighted",
            "role": "additive",
            "field": "$.dominant_signaling_axis",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_wnt_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "WNT_FZD_LRP_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_fgf_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "FGF_FGFR_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_notch_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "NOTCH_DLL_JAG_family"
                ]
              }
            }
          },
          {
            "name": "signaling_directionality_weighted",
            "role": "additive",
            "field": "$.signaling_directionality",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "top_signal_interaction_class_weighted",
            "role": "additive",
            "field": "$.top_signal_interaction_class",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          }
        ]
      },
      "reasoning": "all_of [pass_rule=score_threshold]: FAIL\n  + richness_weighted: passed=True, score=0.25\n  + dominant_axis_weighted: passed=True, score=0.5\n  + signaling_directionality_weighted: passed=True, score=0.25\n  + top_signal_interaction_class_weighted: passed=True, score=0.25\n  + hard_fail 'dominant_axis_wnt_unsupported_by_own_pairs': not triggered\n  + hard_fail 'dominant_axis_fgf_unsupported_by_own_pairs': not triggered\n  + hard_fail 'dominant_axis_notch_unsupported_by_own_pairs': not triggered",
      "metrics": {
        "pass_rule": "score_threshold",
        "scoring_count": 4,
        "scoring_passed": 4,
        "scoring_total_score": 1.25,
        "hard_fail_triggered": []
      },
      "fieldScores": {
        "richness_weighted": 0.25,
        "dominant_axis_weighted": 0.5,
        "signaling_directionality_weighted": 0.25,
        "top_signal_interaction_class_weighted": 0.25
      },
      "graderChecks": [
        {
          "name": "richness_weighted",
          "kind": "gate",
          "passed": true,
          "score": 0.25
        },
        {
          "name": "dominant_axis_weighted",
          "kind": "gate",
          "passed": true,
          "score": 0.5
        },
        {
          "name": "signaling_directionality_weighted",
          "kind": "gate",
          "passed": true,
          "score": 0.25
        },
        {
          "name": "top_signal_interaction_class_weighted",
          "kind": "gate",
          "passed": true,
          "score": 0.25
        },
        {
          "name": "dominant_axis_wnt_unsupported_by_own_pairs",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "dominant_axis_fgf_unsupported_by_own_pairs",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "dominant_axis_notch_unsupported_by_own_pairs",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "dominant_axis_support": [],
        "dominant_signaling_axis": "cannot_assess",
        "signaling_directionality": "cannot_assess",
        "top_signal_interaction_class": "cannot_assess",
        "xspecies_signaling_richness": "cannot_assess"
      },
      "nTurns": 1,
      "checkpointSignals": {
        "clustering": true,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": true,
        "nerveJointAnchor": false,
        "multiomeAssembly": false,
        "depthNormalizedAtac": false,
        "regulonInference": false,
        "cd8SubsetIntegration": false,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": true,
        "orthologProjection": true,
        "epiSubsetSelection": true,
        "bidirectionalLrTest": true,
        "enrichmentFamilyRanking": false,
        "mousePseudobulkDesign": false,
        "cancerEpithelialDenominator": true,
        "geneLevelDe": false,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "trajectoryFile": "scbench-long/trajectories/gse155381_q7_xspecies_epi_communication__gpt-5.4__openai-codex__r3.json",
      "runId": "gpt-5.4__openai-codex__gse155381_q7_xspecies_epi_communication__r3",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-gpt-5-4-pi",
      "modelName": "gpt-5.4",
      "harness": "pi",
      "task": "gse155381_q7_xspecies_epi_communication",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 1,
      "passed": false,
      "score": 3.75,
      "durationS": 2209.03,
      "cost": 2.593871000000001,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "richness_weighted",
            "role": "additive",
            "field": "$.xspecies_signaling_richness",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_weighted",
            "role": "additive",
            "field": "$.dominant_signaling_axis",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_wnt_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "WNT_FZD_LRP_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_fgf_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "FGF_FGFR_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_notch_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "NOTCH_DLL_JAG_family"
                ]
              }
            }
          },
          {
            "name": "signaling_directionality_weighted",
            "role": "additive",
            "field": "$.signaling_directionality",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "top_signal_interaction_class_weighted",
            "role": "additive",
            "field": "$.top_signal_interaction_class",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          }
        ]
      },
      "reasoning": "all_of [pass_rule=score_threshold]: FAIL\n  + richness_weighted: passed=True, score=1.5\n  + dominant_axis_weighted: passed=True, score=0.75\n  + signaling_directionality_weighted: passed=True, score=0.5\n  + top_signal_interaction_class_weighted: passed=True, score=1.0\n  + hard_fail 'dominant_axis_wnt_unsupported_by_own_pairs': not triggered\n  + hard_fail 'dominant_axis_fgf_unsupported_by_own_pairs': not triggered\n  + hard_fail 'dominant_axis_notch_unsupported_by_own_pairs': not triggered",
      "metrics": {
        "pass_rule": "score_threshold",
        "scoring_count": 4,
        "scoring_passed": 4,
        "scoring_total_score": 3.75,
        "hard_fail_triggered": []
      },
      "fieldScores": {
        "richness_weighted": 1.5,
        "dominant_axis_weighted": 0.75,
        "signaling_directionality_weighted": 0.5,
        "top_signal_interaction_class_weighted": 1
      },
      "graderChecks": [
        {
          "name": "richness_weighted",
          "kind": "gate",
          "passed": true,
          "score": 1.5
        },
        {
          "name": "dominant_axis_weighted",
          "kind": "gate",
          "passed": true,
          "score": 0.75
        },
        {
          "name": "signaling_directionality_weighted",
          "kind": "gate",
          "passed": true,
          "score": 0.5
        },
        {
          "name": "top_signal_interaction_class_weighted",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "dominant_axis_wnt_unsupported_by_own_pairs",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "dominant_axis_fgf_unsupported_by_own_pairs",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "dominant_axis_notch_unsupported_by_own_pairs",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "dominant_axis_support": [
          "MDK",
          "NCL",
          "SDC4",
          "ITGA6",
          "SDC1",
          "SDC2",
          "PTPRZ1",
          "LRP1",
          "ITGA4",
          "ALK"
        ],
        "dominant_signaling_axis": "secreted_growth_factor_RTK_family",
        "signaling_directionality": "predominantly_chimera_human_to_chimera_monkey",
        "top_signal_interaction_class": "protein_ligand_receptor_dominant",
        "xspecies_signaling_richness": "many_significant_pairs_across_multiple_signalling_families"
      },
      "nTurns": 72,
      "checkpointSignals": {
        "clustering": true,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": true,
        "nerveJointAnchor": false,
        "multiomeAssembly": false,
        "depthNormalizedAtac": false,
        "regulonInference": false,
        "cd8SubsetIntegration": true,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": true,
        "orthologProjection": true,
        "epiSubsetSelection": true,
        "bidirectionalLrTest": true,
        "enrichmentFamilyRanking": true,
        "mousePseudobulkDesign": false,
        "cancerEpithelialDenominator": false,
        "geneLevelDe": false,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "trajectoryFile": "scbench-long/trajectories/gse155381_q7_xspecies_epi_communication__gpt-5.4__pi__r1.json",
      "runId": "gpt-5.4__pi__gse155381_q7_xspecies_epi_communication__r1",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-gpt-5-4-pi",
      "modelName": "gpt-5.4",
      "harness": "pi",
      "task": "gse155381_q7_xspecies_epi_communication",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 2,
      "passed": false,
      "score": 3.75,
      "durationS": 1292.35,
      "cost": 1.5690480000000004,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "richness_weighted",
            "role": "additive",
            "field": "$.xspecies_signaling_richness",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_weighted",
            "role": "additive",
            "field": "$.dominant_signaling_axis",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_wnt_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "WNT_FZD_LRP_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_fgf_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "FGF_FGFR_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_notch_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "NOTCH_DLL_JAG_family"
                ]
              }
            }
          },
          {
            "name": "signaling_directionality_weighted",
            "role": "additive",
            "field": "$.signaling_directionality",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "top_signal_interaction_class_weighted",
            "role": "additive",
            "field": "$.top_signal_interaction_class",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          }
        ]
      },
      "reasoning": "all_of [pass_rule=score_threshold]: FAIL\n  + richness_weighted: passed=True, score=1.5\n  + dominant_axis_weighted: passed=True, score=0.25\n  + signaling_directionality_weighted: passed=True, score=1.0\n  + top_signal_interaction_class_weighted: passed=True, score=1.0\n  + hard_fail 'dominant_axis_wnt_unsupported_by_own_pairs': not triggered\n  + hard_fail 'dominant_axis_fgf_unsupported_by_own_pairs': not triggered\n  + hard_fail 'dominant_axis_notch_unsupported_by_own_pairs': not triggered",
      "metrics": {
        "pass_rule": "score_threshold",
        "scoring_count": 4,
        "scoring_passed": 4,
        "scoring_total_score": 3.75,
        "hard_fail_triggered": []
      },
      "fieldScores": {
        "richness_weighted": 1.5,
        "dominant_axis_weighted": 0.25,
        "signaling_directionality_weighted": 1,
        "top_signal_interaction_class_weighted": 1
      },
      "graderChecks": [
        {
          "name": "richness_weighted",
          "kind": "gate",
          "passed": true,
          "score": 1.5
        },
        {
          "name": "dominant_axis_weighted",
          "kind": "gate",
          "passed": true,
          "score": 0.25
        },
        {
          "name": "signaling_directionality_weighted",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "top_signal_interaction_class_weighted",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "dominant_axis_wnt_unsupported_by_own_pairs",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "dominant_axis_fgf_unsupported_by_own_pairs",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "dominant_axis_notch_unsupported_by_own_pairs",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "dominant_axis_support": [
          "FN1",
          "COL3A1",
          "COL11A1",
          "COL21A1",
          "COL4A2",
          "COL5A2",
          "FBN1",
          "ITGA1",
          "ITGA2",
          "ITGA5",
          "ITGA10",
          "ITGA11",
          "ITGB1"
        ],
        "dominant_signaling_axis": "ECM_collagen_integrin_family",
        "signaling_directionality": "substantial_signal_in_both_directions",
        "top_signal_interaction_class": "protein_ligand_receptor_dominant",
        "xspecies_signaling_richness": "many_significant_pairs_across_multiple_signalling_families"
      },
      "nTurns": 55,
      "checkpointSignals": {
        "clustering": true,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": true,
        "nerveJointAnchor": false,
        "multiomeAssembly": false,
        "depthNormalizedAtac": false,
        "regulonInference": false,
        "cd8SubsetIntegration": true,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": true,
        "orthologProjection": true,
        "epiSubsetSelection": true,
        "bidirectionalLrTest": true,
        "enrichmentFamilyRanking": true,
        "mousePseudobulkDesign": false,
        "cancerEpithelialDenominator": true,
        "geneLevelDe": true,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "trajectoryFile": "scbench-long/trajectories/gse155381_q7_xspecies_epi_communication__gpt-5.4__pi__r2.json",
      "runId": "gpt-5.4__pi__gse155381_q7_xspecies_epi_communication__r2",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-gpt-5-4-pi",
      "modelName": "gpt-5.4",
      "harness": "pi",
      "task": "gse155381_q7_xspecies_epi_communication",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 3,
      "passed": false,
      "score": 3.75,
      "durationS": 1783.92,
      "cost": 1.5841055,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "richness_weighted",
            "role": "additive",
            "field": "$.xspecies_signaling_richness",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_weighted",
            "role": "additive",
            "field": "$.dominant_signaling_axis",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_wnt_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "WNT_FZD_LRP_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_fgf_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "FGF_FGFR_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_notch_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "NOTCH_DLL_JAG_family"
                ]
              }
            }
          },
          {
            "name": "signaling_directionality_weighted",
            "role": "additive",
            "field": "$.signaling_directionality",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "top_signal_interaction_class_weighted",
            "role": "additive",
            "field": "$.top_signal_interaction_class",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          }
        ]
      },
      "reasoning": "all_of [pass_rule=score_threshold]: FAIL\n  + richness_weighted: passed=True, score=1.5\n  + dominant_axis_weighted: passed=True, score=0.25\n  + signaling_directionality_weighted: passed=True, score=1.0\n  + top_signal_interaction_class_weighted: passed=True, score=1.0\n  + hard_fail 'dominant_axis_wnt_unsupported_by_own_pairs': not triggered\n  + hard_fail 'dominant_axis_fgf_unsupported_by_own_pairs': not triggered\n  + hard_fail 'dominant_axis_notch_unsupported_by_own_pairs': not triggered",
      "metrics": {
        "pass_rule": "score_threshold",
        "scoring_count": 4,
        "scoring_passed": 4,
        "scoring_total_score": 3.75,
        "hard_fail_triggered": []
      },
      "fieldScores": {
        "richness_weighted": 1.5,
        "dominant_axis_weighted": 0.25,
        "signaling_directionality_weighted": 1,
        "top_signal_interaction_class_weighted": 1
      },
      "graderChecks": [
        {
          "name": "richness_weighted",
          "kind": "gate",
          "passed": true,
          "score": 1.5
        },
        {
          "name": "dominant_axis_weighted",
          "kind": "gate",
          "passed": true,
          "score": 0.25
        },
        {
          "name": "signaling_directionality_weighted",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "top_signal_interaction_class_weighted",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "dominant_axis_wnt_unsupported_by_own_pairs",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "dominant_axis_fgf_unsupported_by_own_pairs",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "dominant_axis_notch_unsupported_by_own_pairs",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "dominant_axis_support": [
          "FN1",
          "LAMB1",
          "LAMA4",
          "COL4A1",
          "COL4A2",
          "COL3A1",
          "SPP1",
          "THBS2",
          "ITGA1",
          "ITGA4",
          "ITGA5",
          "ITGA6",
          "ITGAV",
          "ITGB1",
          "CD44",
          "SDC1",
          "CD47"
        ],
        "dominant_signaling_axis": "ECM_collagen_integrin_family",
        "signaling_directionality": "substantial_signal_in_both_directions",
        "top_signal_interaction_class": "protein_ligand_receptor_dominant",
        "xspecies_signaling_richness": "many_significant_pairs_across_multiple_signalling_families"
      },
      "nTurns": 40,
      "checkpointSignals": {
        "clustering": true,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": true,
        "nerveJointAnchor": false,
        "multiomeAssembly": false,
        "depthNormalizedAtac": false,
        "regulonInference": false,
        "cd8SubsetIntegration": true,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": true,
        "orthologProjection": true,
        "epiSubsetSelection": true,
        "bidirectionalLrTest": true,
        "enrichmentFamilyRanking": true,
        "mousePseudobulkDesign": false,
        "cancerEpithelialDenominator": true,
        "geneLevelDe": false,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "trajectoryFile": "scbench-long/trajectories/gse155381_q7_xspecies_epi_communication__gpt-5.4__pi__r3.json",
      "runId": "gpt-5.4__pi__gse155381_q7_xspecies_epi_communication__r3",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-gpt-5-5-codex",
      "modelName": "gpt-5.5",
      "harness": "openai-codex",
      "task": "gse155381_q7_xspecies_epi_communication",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 1,
      "passed": false,
      "score": 3.75,
      "durationS": 703.33,
      "cost": 2.474512,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "richness_weighted",
            "role": "additive",
            "field": "$.xspecies_signaling_richness",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_weighted",
            "role": "additive",
            "field": "$.dominant_signaling_axis",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_wnt_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "WNT_FZD_LRP_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_fgf_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "FGF_FGFR_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_notch_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "NOTCH_DLL_JAG_family"
                ]
              }
            }
          },
          {
            "name": "signaling_directionality_weighted",
            "role": "additive",
            "field": "$.signaling_directionality",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "top_signal_interaction_class_weighted",
            "role": "additive",
            "field": "$.top_signal_interaction_class",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          }
        ]
      },
      "reasoning": "all_of [pass_rule=score_threshold]: FAIL\n  + richness_weighted: passed=True, score=1.5\n  + dominant_axis_weighted: passed=True, score=0.25\n  + signaling_directionality_weighted: passed=True, score=1.0\n  + top_signal_interaction_class_weighted: passed=True, score=1.0\n  + hard_fail 'dominant_axis_wnt_unsupported_by_own_pairs': not triggered\n  + hard_fail 'dominant_axis_fgf_unsupported_by_own_pairs': not triggered\n  + hard_fail 'dominant_axis_notch_unsupported_by_own_pairs': not triggered",
      "metrics": {
        "pass_rule": "score_threshold",
        "scoring_count": 4,
        "scoring_passed": 4,
        "scoring_total_score": 3.75,
        "hard_fail_triggered": []
      },
      "fieldScores": {
        "richness_weighted": 1.5,
        "dominant_axis_weighted": 0.25,
        "signaling_directionality_weighted": 1,
        "top_signal_interaction_class_weighted": 1
      },
      "graderChecks": [
        {
          "name": "richness_weighted",
          "kind": "gate",
          "passed": true,
          "score": 1.5
        },
        {
          "name": "dominant_axis_weighted",
          "kind": "gate",
          "passed": true,
          "score": 0.25
        },
        {
          "name": "signaling_directionality_weighted",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "top_signal_interaction_class_weighted",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "dominant_axis_wnt_unsupported_by_own_pairs",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "dominant_axis_fgf_unsupported_by_own_pairs",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "dominant_axis_notch_unsupported_by_own_pairs",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "dominant_axis_support": [
          "MDK",
          "ITGA6",
          "ITGB1",
          "ITGA4",
          "COL4A2",
          "SDC4",
          "FN1",
          "ITGA5",
          "SPP1",
          "ITGAV",
          "THBS1",
          "CD47",
          "LAMB1",
          "COL4A1",
          "ITGB8"
        ],
        "dominant_signaling_axis": "ECM_collagen_integrin_family",
        "signaling_directionality": "substantial_signal_in_both_directions",
        "top_signal_interaction_class": "protein_ligand_receptor_dominant",
        "xspecies_signaling_richness": "many_significant_pairs_across_multiple_signalling_families"
      },
      "nTurns": 1,
      "checkpointSignals": {
        "clustering": false,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": true,
        "nerveJointAnchor": false,
        "multiomeAssembly": false,
        "depthNormalizedAtac": false,
        "regulonInference": false,
        "cd8SubsetIntegration": false,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": true,
        "orthologProjection": true,
        "epiSubsetSelection": true,
        "bidirectionalLrTest": true,
        "enrichmentFamilyRanking": true,
        "mousePseudobulkDesign": false,
        "cancerEpithelialDenominator": true,
        "geneLevelDe": true,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "trajectoryFile": "scbench-long/trajectories/gse155381_q7_xspecies_epi_communication__gpt-5.5__openai-codex__r1.json",
      "runId": "gpt-5.5__openai-codex__gse155381_q7_xspecies_epi_communication__r1",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-gpt-5-5-codex",
      "modelName": "gpt-5.5",
      "harness": "openai-codex",
      "task": "gse155381_q7_xspecies_epi_communication",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 2,
      "passed": false,
      "score": 3.75,
      "durationS": 588.21,
      "cost": 2.555618,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "richness_weighted",
            "role": "additive",
            "field": "$.xspecies_signaling_richness",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_weighted",
            "role": "additive",
            "field": "$.dominant_signaling_axis",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_wnt_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "WNT_FZD_LRP_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_fgf_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "FGF_FGFR_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_notch_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "NOTCH_DLL_JAG_family"
                ]
              }
            }
          },
          {
            "name": "signaling_directionality_weighted",
            "role": "additive",
            "field": "$.signaling_directionality",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "top_signal_interaction_class_weighted",
            "role": "additive",
            "field": "$.top_signal_interaction_class",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          }
        ]
      },
      "reasoning": "all_of [pass_rule=score_threshold]: FAIL\n  + richness_weighted: passed=True, score=1.5\n  + dominant_axis_weighted: passed=True, score=0.25\n  + signaling_directionality_weighted: passed=True, score=1.0\n  + top_signal_interaction_class_weighted: passed=True, score=1.0\n  + hard_fail 'dominant_axis_wnt_unsupported_by_own_pairs': not triggered\n  + hard_fail 'dominant_axis_fgf_unsupported_by_own_pairs': not triggered\n  + hard_fail 'dominant_axis_notch_unsupported_by_own_pairs': not triggered",
      "metrics": {
        "pass_rule": "score_threshold",
        "scoring_count": 4,
        "scoring_passed": 4,
        "scoring_total_score": 3.75,
        "hard_fail_triggered": []
      },
      "fieldScores": {
        "richness_weighted": 1.5,
        "dominant_axis_weighted": 0.25,
        "signaling_directionality_weighted": 1,
        "top_signal_interaction_class_weighted": 1
      },
      "graderChecks": [
        {
          "name": "richness_weighted",
          "kind": "gate",
          "passed": true,
          "score": 1.5
        },
        {
          "name": "dominant_axis_weighted",
          "kind": "gate",
          "passed": true,
          "score": 0.25
        },
        {
          "name": "signaling_directionality_weighted",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "top_signal_interaction_class_weighted",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "dominant_axis_wnt_unsupported_by_own_pairs",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "dominant_axis_fgf_unsupported_by_own_pairs",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "dominant_axis_notch_unsupported_by_own_pairs",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "dominant_axis_support": [
          "VCL",
          "ITGB5",
          "SPON2",
          "ITGA4",
          "THBS2",
          "SPP1",
          "ITGB1",
          "ANGPT1",
          "ITGA5",
          "COL3A1",
          "ITGA1",
          "COL4A2",
          "FN1",
          "VCAN"
        ],
        "dominant_signaling_axis": "ECM_collagen_integrin_family",
        "signaling_directionality": "substantial_signal_in_both_directions",
        "top_signal_interaction_class": "protein_ligand_receptor_dominant",
        "xspecies_signaling_richness": "many_significant_pairs_across_multiple_signalling_families"
      },
      "nTurns": 1,
      "checkpointSignals": {
        "clustering": false,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": true,
        "nerveJointAnchor": false,
        "multiomeAssembly": false,
        "depthNormalizedAtac": false,
        "regulonInference": false,
        "cd8SubsetIntegration": true,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": true,
        "orthologProjection": true,
        "epiSubsetSelection": true,
        "bidirectionalLrTest": true,
        "enrichmentFamilyRanking": true,
        "mousePseudobulkDesign": false,
        "cancerEpithelialDenominator": true,
        "geneLevelDe": true,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "trajectoryFile": "scbench-long/trajectories/gse155381_q7_xspecies_epi_communication__gpt-5.5__openai-codex__r2.json",
      "runId": "gpt-5.5__openai-codex__gse155381_q7_xspecies_epi_communication__r2",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-gpt-5-5-codex",
      "modelName": "gpt-5.5",
      "harness": "openai-codex",
      "task": "gse155381_q7_xspecies_epi_communication",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 3,
      "passed": false,
      "score": 3.75,
      "durationS": 884.16,
      "cost": 2.503888,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "richness_weighted",
            "role": "additive",
            "field": "$.xspecies_signaling_richness",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_weighted",
            "role": "additive",
            "field": "$.dominant_signaling_axis",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_wnt_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "WNT_FZD_LRP_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_fgf_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "FGF_FGFR_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_notch_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "NOTCH_DLL_JAG_family"
                ]
              }
            }
          },
          {
            "name": "signaling_directionality_weighted",
            "role": "additive",
            "field": "$.signaling_directionality",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "top_signal_interaction_class_weighted",
            "role": "additive",
            "field": "$.top_signal_interaction_class",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          }
        ]
      },
      "reasoning": "all_of [pass_rule=score_threshold]: FAIL\n  + richness_weighted: passed=True, score=1.5\n  + dominant_axis_weighted: passed=True, score=0.25\n  + signaling_directionality_weighted: passed=True, score=1.0\n  + top_signal_interaction_class_weighted: passed=True, score=1.0\n  + hard_fail 'dominant_axis_wnt_unsupported_by_own_pairs': not triggered\n  + hard_fail 'dominant_axis_fgf_unsupported_by_own_pairs': not triggered\n  + hard_fail 'dominant_axis_notch_unsupported_by_own_pairs': not triggered",
      "metrics": {
        "pass_rule": "score_threshold",
        "scoring_count": 4,
        "scoring_passed": 4,
        "scoring_total_score": 3.75,
        "hard_fail_triggered": []
      },
      "fieldScores": {
        "richness_weighted": 1.5,
        "dominant_axis_weighted": 0.25,
        "signaling_directionality_weighted": 1,
        "top_signal_interaction_class_weighted": 1
      },
      "graderChecks": [
        {
          "name": "richness_weighted",
          "kind": "gate",
          "passed": true,
          "score": 1.5
        },
        {
          "name": "dominant_axis_weighted",
          "kind": "gate",
          "passed": true,
          "score": 0.25
        },
        {
          "name": "signaling_directionality_weighted",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "top_signal_interaction_class_weighted",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "dominant_axis_wnt_unsupported_by_own_pairs",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "dominant_axis_fgf_unsupported_by_own_pairs",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "dominant_axis_notch_unsupported_by_own_pairs",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "dominant_axis_support": [
          "SPP1",
          "ITGA4",
          "ITGB1",
          "ITGAV",
          "LAMB1",
          "ITGA6",
          "CD44",
          "COL4A2",
          "SDC4",
          "LAMA4",
          "LAMC1",
          "FN1",
          "COL1A2",
          "SDC1",
          "ITGA7"
        ],
        "dominant_signaling_axis": "ECM_collagen_integrin_family",
        "signaling_directionality": "substantial_signal_in_both_directions",
        "top_signal_interaction_class": "protein_ligand_receptor_dominant",
        "xspecies_signaling_richness": "many_significant_pairs_across_multiple_signalling_families"
      },
      "nTurns": 1,
      "checkpointSignals": {
        "clustering": false,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": true,
        "nerveJointAnchor": false,
        "multiomeAssembly": true,
        "depthNormalizedAtac": false,
        "regulonInference": true,
        "cd8SubsetIntegration": false,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": true,
        "orthologProjection": true,
        "epiSubsetSelection": true,
        "bidirectionalLrTest": true,
        "enrichmentFamilyRanking": true,
        "mousePseudobulkDesign": false,
        "cancerEpithelialDenominator": true,
        "geneLevelDe": false,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "trajectoryFile": "scbench-long/trajectories/gse155381_q7_xspecies_epi_communication__gpt-5.5__openai-codex__r3.json",
      "runId": "gpt-5.5__openai-codex__gse155381_q7_xspecies_epi_communication__r3",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-gpt-5-5-pi",
      "modelName": "gpt-5.5",
      "harness": "pi",
      "task": "gse155381_q7_xspecies_epi_communication",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 1,
      "passed": false,
      "score": 3.75,
      "durationS": 1296.23,
      "cost": 2.5331220000000005,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "richness_weighted",
            "role": "additive",
            "field": "$.xspecies_signaling_richness",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_weighted",
            "role": "additive",
            "field": "$.dominant_signaling_axis",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_wnt_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "WNT_FZD_LRP_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_fgf_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "FGF_FGFR_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_notch_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "NOTCH_DLL_JAG_family"
                ]
              }
            }
          },
          {
            "name": "signaling_directionality_weighted",
            "role": "additive",
            "field": "$.signaling_directionality",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "top_signal_interaction_class_weighted",
            "role": "additive",
            "field": "$.top_signal_interaction_class",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          }
        ]
      },
      "reasoning": "all_of [pass_rule=score_threshold]: FAIL\n  + richness_weighted: passed=True, score=1.5\n  + dominant_axis_weighted: passed=True, score=0.25\n  + signaling_directionality_weighted: passed=True, score=1.0\n  + top_signal_interaction_class_weighted: passed=True, score=1.0\n  + hard_fail 'dominant_axis_wnt_unsupported_by_own_pairs': not triggered\n  + hard_fail 'dominant_axis_fgf_unsupported_by_own_pairs': not triggered\n  + hard_fail 'dominant_axis_notch_unsupported_by_own_pairs': not triggered",
      "metrics": {
        "pass_rule": "score_threshold",
        "scoring_count": 4,
        "scoring_passed": 4,
        "scoring_total_score": 3.75,
        "hard_fail_triggered": []
      },
      "fieldScores": {
        "richness_weighted": 1.5,
        "dominant_axis_weighted": 0.25,
        "signaling_directionality_weighted": 1,
        "top_signal_interaction_class_weighted": 1
      },
      "graderChecks": [
        {
          "name": "richness_weighted",
          "kind": "gate",
          "passed": true,
          "score": 1.5
        },
        {
          "name": "dominant_axis_weighted",
          "kind": "gate",
          "passed": true,
          "score": 0.25
        },
        {
          "name": "signaling_directionality_weighted",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "top_signal_interaction_class_weighted",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "dominant_axis_wnt_unsupported_by_own_pairs",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "dominant_axis_fgf_unsupported_by_own_pairs",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "dominant_axis_notch_unsupported_by_own_pairs",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "dominant_axis_support": [
          "COL3A1",
          "ITGA1",
          "ITGB1",
          "DDR1",
          "DDR2",
          "FN1",
          "ITGA5",
          "COL4A2",
          "ITGB5",
          "COL11A1",
          "SPP1",
          "ITGA4",
          "VCAN",
          "ITGAV",
          "ITGB8",
          "LAMB1",
          "ITGA6",
          "TIMP2"
        ],
        "dominant_signaling_axis": "ECM_collagen_integrin_family",
        "signaling_directionality": "substantial_signal_in_both_directions",
        "top_signal_interaction_class": "protein_ligand_receptor_dominant",
        "xspecies_signaling_richness": "many_significant_pairs_across_multiple_signalling_families"
      },
      "nTurns": 51,
      "checkpointSignals": {
        "clustering": false,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": true,
        "nerveJointAnchor": false,
        "multiomeAssembly": false,
        "depthNormalizedAtac": false,
        "regulonInference": false,
        "cd8SubsetIntegration": false,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": true,
        "orthologProjection": true,
        "epiSubsetSelection": true,
        "bidirectionalLrTest": true,
        "enrichmentFamilyRanking": true,
        "mousePseudobulkDesign": false,
        "cancerEpithelialDenominator": true,
        "geneLevelDe": false,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "trajectoryFile": "scbench-long/trajectories/gse155381_q7_xspecies_epi_communication__gpt-5.5__pi__r1.json",
      "runId": "gpt-5.5__pi__gse155381_q7_xspecies_epi_communication__r1",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-gpt-5-5-pi",
      "modelName": "gpt-5.5",
      "harness": "pi",
      "task": "gse155381_q7_xspecies_epi_communication",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 2,
      "passed": false,
      "score": 3.75,
      "durationS": 737.33,
      "cost": 2.3167429999999998,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "richness_weighted",
            "role": "additive",
            "field": "$.xspecies_signaling_richness",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_weighted",
            "role": "additive",
            "field": "$.dominant_signaling_axis",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_wnt_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "WNT_FZD_LRP_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_fgf_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "FGF_FGFR_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_notch_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "NOTCH_DLL_JAG_family"
                ]
              }
            }
          },
          {
            "name": "signaling_directionality_weighted",
            "role": "additive",
            "field": "$.signaling_directionality",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "top_signal_interaction_class_weighted",
            "role": "additive",
            "field": "$.top_signal_interaction_class",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          }
        ]
      },
      "reasoning": "all_of [pass_rule=score_threshold]: FAIL\n  + richness_weighted: passed=True, score=1.5\n  + dominant_axis_weighted: passed=True, score=0.25\n  + signaling_directionality_weighted: passed=True, score=1.0\n  + top_signal_interaction_class_weighted: passed=True, score=1.0\n  + hard_fail 'dominant_axis_wnt_unsupported_by_own_pairs': not triggered\n  + hard_fail 'dominant_axis_fgf_unsupported_by_own_pairs': not triggered\n  + hard_fail 'dominant_axis_notch_unsupported_by_own_pairs': not triggered",
      "metrics": {
        "pass_rule": "score_threshold",
        "scoring_count": 4,
        "scoring_passed": 4,
        "scoring_total_score": 3.75,
        "hard_fail_triggered": []
      },
      "fieldScores": {
        "richness_weighted": 1.5,
        "dominant_axis_weighted": 0.25,
        "signaling_directionality_weighted": 1,
        "top_signal_interaction_class_weighted": 1
      },
      "graderChecks": [
        {
          "name": "richness_weighted",
          "kind": "gate",
          "passed": true,
          "score": 1.5
        },
        {
          "name": "dominant_axis_weighted",
          "kind": "gate",
          "passed": true,
          "score": 0.25
        },
        {
          "name": "signaling_directionality_weighted",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "top_signal_interaction_class_weighted",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "dominant_axis_wnt_unsupported_by_own_pairs",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "dominant_axis_fgf_unsupported_by_own_pairs",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "dominant_axis_notch_unsupported_by_own_pairs",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "dominant_axis_support": [
          "FN1",
          "ITGA5",
          "ITGB1",
          "SPP1",
          "ITGA4",
          "ITGAV",
          "THBS1",
          "CD47",
          "THBS2",
          "COL4A1",
          "ITGB8",
          "SDC1",
          "COL1A1",
          "COL1A2",
          "COL4A2",
          "SDC4",
          "LAMB1",
          "ITGA6",
          "ITGB4",
          "LAMA4",
          "CD44"
        ],
        "dominant_signaling_axis": "ECM_collagen_integrin_family",
        "signaling_directionality": "substantial_signal_in_both_directions",
        "top_signal_interaction_class": "protein_ligand_receptor_dominant",
        "xspecies_signaling_richness": "many_significant_pairs_across_multiple_signalling_families"
      },
      "nTurns": 35,
      "checkpointSignals": {
        "clustering": false,
        "markerAnnotation": false,
        "scoreBasedAnnotation": false,
        "normalization": true,
        "nerveJointAnchor": false,
        "multiomeAssembly": false,
        "depthNormalizedAtac": false,
        "regulonInference": false,
        "cd8SubsetIntegration": true,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": true,
        "orthologProjection": true,
        "epiSubsetSelection": true,
        "bidirectionalLrTest": true,
        "enrichmentFamilyRanking": true,
        "mousePseudobulkDesign": false,
        "cancerEpithelialDenominator": true,
        "geneLevelDe": false,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "trajectoryFile": "scbench-long/trajectories/gse155381_q7_xspecies_epi_communication__gpt-5.5__pi__r2.json",
      "runId": "gpt-5.5__pi__gse155381_q7_xspecies_epi_communication__r2",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-gpt-5-5-pi",
      "modelName": "gpt-5.5",
      "harness": "pi",
      "task": "gse155381_q7_xspecies_epi_communication",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 3,
      "passed": false,
      "score": 3.75,
      "durationS": 1084.45,
      "cost": 2.642749999999999,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "richness_weighted",
            "role": "additive",
            "field": "$.xspecies_signaling_richness",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_weighted",
            "role": "additive",
            "field": "$.dominant_signaling_axis",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_wnt_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "WNT_FZD_LRP_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_fgf_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "FGF_FGFR_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_notch_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "NOTCH_DLL_JAG_family"
                ]
              }
            }
          },
          {
            "name": "signaling_directionality_weighted",
            "role": "additive",
            "field": "$.signaling_directionality",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "top_signal_interaction_class_weighted",
            "role": "additive",
            "field": "$.top_signal_interaction_class",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          }
        ]
      },
      "reasoning": "all_of [pass_rule=score_threshold]: FAIL\n  + richness_weighted: passed=True, score=1.5\n  + dominant_axis_weighted: passed=True, score=0.25\n  + signaling_directionality_weighted: passed=True, score=1.0\n  + top_signal_interaction_class_weighted: passed=True, score=1.0\n  + hard_fail 'dominant_axis_wnt_unsupported_by_own_pairs': not triggered\n  + hard_fail 'dominant_axis_fgf_unsupported_by_own_pairs': not triggered\n  + hard_fail 'dominant_axis_notch_unsupported_by_own_pairs': not triggered",
      "metrics": {
        "pass_rule": "score_threshold",
        "scoring_count": 4,
        "scoring_passed": 4,
        "scoring_total_score": 3.75,
        "hard_fail_triggered": []
      },
      "fieldScores": {
        "richness_weighted": 1.5,
        "dominant_axis_weighted": 0.25,
        "signaling_directionality_weighted": 1,
        "top_signal_interaction_class_weighted": 1
      },
      "graderChecks": [
        {
          "name": "richness_weighted",
          "kind": "gate",
          "passed": true,
          "score": 1.5
        },
        {
          "name": "dominant_axis_weighted",
          "kind": "gate",
          "passed": true,
          "score": 0.25
        },
        {
          "name": "signaling_directionality_weighted",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "top_signal_interaction_class_weighted",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "dominant_axis_wnt_unsupported_by_own_pairs",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "dominant_axis_fgf_unsupported_by_own_pairs",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "dominant_axis_notch_unsupported_by_own_pairs",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "dominant_axis_support": [
          "MDK",
          "ITGA6",
          "ITGB1",
          "SDC4",
          "SDC1",
          "SDC2",
          "COL4A2",
          "LAMB1",
          "FN1",
          "SPP1",
          "THBS2",
          "CD47",
          "LAMC1",
          "ITGA4",
          "ITGAV",
          "LAMA4",
          "ITGA5"
        ],
        "dominant_signaling_axis": "ECM_collagen_integrin_family",
        "signaling_directionality": "substantial_signal_in_both_directions",
        "top_signal_interaction_class": "protein_ligand_receptor_dominant",
        "xspecies_signaling_richness": "many_significant_pairs_across_multiple_signalling_families"
      },
      "nTurns": 54,
      "checkpointSignals": {
        "clustering": true,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": true,
        "nerveJointAnchor": false,
        "multiomeAssembly": false,
        "depthNormalizedAtac": false,
        "regulonInference": false,
        "cd8SubsetIntegration": false,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": true,
        "orthologProjection": true,
        "epiSubsetSelection": true,
        "bidirectionalLrTest": true,
        "enrichmentFamilyRanking": true,
        "mousePseudobulkDesign": false,
        "cancerEpithelialDenominator": true,
        "geneLevelDe": true,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "trajectoryFile": "scbench-long/trajectories/gse155381_q7_xspecies_epi_communication__gpt-5.5__pi__r3.json",
      "runId": "gpt-5.5__pi__gse155381_q7_xspecies_epi_communication__r3",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-grok-4-20-beta-reasoning-pi",
      "modelName": "grok-4.20-beta-0309-reasoning",
      "harness": "pi",
      "task": "gse155381_q7_xspecies_epi_communication",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 1,
      "passed": false,
      "score": 1,
      "durationS": 78.05,
      "cost": 0.068538,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "richness_weighted",
            "role": "additive",
            "field": "$.xspecies_signaling_richness",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_weighted",
            "role": "additive",
            "field": "$.dominant_signaling_axis",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_wnt_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "WNT_FZD_LRP_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_fgf_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "FGF_FGFR_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_notch_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "NOTCH_DLL_JAG_family"
                ]
              }
            }
          },
          {
            "name": "signaling_directionality_weighted",
            "role": "additive",
            "field": "$.signaling_directionality",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "top_signal_interaction_class_weighted",
            "role": "additive",
            "field": "$.top_signal_interaction_class",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          }
        ]
      },
      "reasoning": "all_of [pass_rule=score_threshold]: FAIL\n  + richness_weighted: passed=True, score=0.0\n  + dominant_axis_weighted: passed=True, score=0.5\n  + signaling_directionality_weighted: passed=True, score=0.25\n  + top_signal_interaction_class_weighted: passed=True, score=0.25\n  + hard_fail 'dominant_axis_wnt_unsupported_by_own_pairs': not triggered\n  + hard_fail 'dominant_axis_fgf_unsupported_by_own_pairs': not triggered\n  + hard_fail 'dominant_axis_notch_unsupported_by_own_pairs': not triggered",
      "metrics": {
        "pass_rule": "score_threshold",
        "scoring_count": 4,
        "scoring_passed": 4,
        "scoring_total_score": 1,
        "hard_fail_triggered": []
      },
      "fieldScores": {
        "richness_weighted": 0,
        "dominant_axis_weighted": 0.5,
        "signaling_directionality_weighted": 0.25,
        "top_signal_interaction_class_weighted": 0.25
      },
      "graderChecks": [
        {
          "name": "richness_weighted",
          "kind": "gate",
          "passed": true,
          "score": 0
        },
        {
          "name": "dominant_axis_weighted",
          "kind": "gate",
          "passed": true,
          "score": 0.5
        },
        {
          "name": "signaling_directionality_weighted",
          "kind": "gate",
          "passed": true,
          "score": 0.25
        },
        {
          "name": "top_signal_interaction_class_weighted",
          "kind": "gate",
          "passed": true,
          "score": 0.25
        },
        {
          "name": "dominant_axis_wnt_unsupported_by_own_pairs",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "dominant_axis_fgf_unsupported_by_own_pairs",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "dominant_axis_notch_unsupported_by_own_pairs",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "dominant_axis_support": [],
        "dominant_signaling_axis": "cannot_assess",
        "signaling_directionality": "cannot_assess",
        "top_signal_interaction_class": "cannot_assess",
        "xspecies_signaling_richness": "no_significant_cross_species_signal_detected"
      },
      "nTurns": 22,
      "checkpointSignals": {
        "clustering": false,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": true,
        "nerveJointAnchor": false,
        "multiomeAssembly": false,
        "depthNormalizedAtac": false,
        "regulonInference": false,
        "cd8SubsetIntegration": false,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": true,
        "orthologProjection": false,
        "epiSubsetSelection": true,
        "bidirectionalLrTest": true,
        "enrichmentFamilyRanking": true,
        "mousePseudobulkDesign": false,
        "cancerEpithelialDenominator": false,
        "geneLevelDe": false,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "trajectoryFile": "scbench-long/trajectories/gse155381_q7_xspecies_epi_communication__grok-4.20-beta-0309-reasoning__pi__r1.json",
      "runId": "grok-4.20-beta-0309-reasoning__pi__gse155381_q7_xspecies_epi_communication__r1",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-grok-4-20-beta-reasoning-pi",
      "modelName": "grok-4.20-beta-0309-reasoning",
      "harness": "pi",
      "task": "gse155381_q7_xspecies_epi_communication",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 2,
      "passed": false,
      "score": 1,
      "durationS": 159.02,
      "cost": 0.16597570000000003,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "richness_weighted",
            "role": "additive",
            "field": "$.xspecies_signaling_richness",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_weighted",
            "role": "additive",
            "field": "$.dominant_signaling_axis",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_wnt_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "WNT_FZD_LRP_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_fgf_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "FGF_FGFR_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_notch_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "NOTCH_DLL_JAG_family"
                ]
              }
            }
          },
          {
            "name": "signaling_directionality_weighted",
            "role": "additive",
            "field": "$.signaling_directionality",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "top_signal_interaction_class_weighted",
            "role": "additive",
            "field": "$.top_signal_interaction_class",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          }
        ]
      },
      "reasoning": "all_of [pass_rule=score_threshold]: FAIL\n  + richness_weighted: passed=True, score=0.0\n  + dominant_axis_weighted: passed=True, score=0.5\n  + signaling_directionality_weighted: passed=True, score=0.25\n  + top_signal_interaction_class_weighted: passed=True, score=0.25\n  + hard_fail 'dominant_axis_wnt_unsupported_by_own_pairs': not triggered\n  + hard_fail 'dominant_axis_fgf_unsupported_by_own_pairs': not triggered\n  + hard_fail 'dominant_axis_notch_unsupported_by_own_pairs': not triggered",
      "metrics": {
        "pass_rule": "score_threshold",
        "scoring_count": 4,
        "scoring_passed": 4,
        "scoring_total_score": 1,
        "hard_fail_triggered": []
      },
      "fieldScores": {
        "richness_weighted": 0,
        "dominant_axis_weighted": 0.5,
        "signaling_directionality_weighted": 0.25,
        "top_signal_interaction_class_weighted": 0.25
      },
      "graderChecks": [
        {
          "name": "richness_weighted",
          "kind": "gate",
          "passed": true,
          "score": 0
        },
        {
          "name": "dominant_axis_weighted",
          "kind": "gate",
          "passed": true,
          "score": 0.5
        },
        {
          "name": "signaling_directionality_weighted",
          "kind": "gate",
          "passed": true,
          "score": 0.25
        },
        {
          "name": "top_signal_interaction_class_weighted",
          "kind": "gate",
          "passed": true,
          "score": 0.25
        },
        {
          "name": "dominant_axis_wnt_unsupported_by_own_pairs",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "dominant_axis_fgf_unsupported_by_own_pairs",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "dominant_axis_notch_unsupported_by_own_pairs",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "dominant_axis_support": [],
        "dominant_signaling_axis": "cannot_assess",
        "signaling_directionality": "cannot_assess",
        "top_signal_interaction_class": "cannot_assess",
        "xspecies_signaling_richness": "no_significant_cross_species_signal_detected"
      },
      "nTurns": 28,
      "checkpointSignals": {
        "clustering": false,
        "markerAnnotation": false,
        "scoreBasedAnnotation": false,
        "normalization": true,
        "nerveJointAnchor": false,
        "multiomeAssembly": false,
        "depthNormalizedAtac": false,
        "regulonInference": false,
        "cd8SubsetIntegration": true,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": true,
        "orthologProjection": true,
        "epiSubsetSelection": true,
        "bidirectionalLrTest": true,
        "enrichmentFamilyRanking": true,
        "mousePseudobulkDesign": false,
        "cancerEpithelialDenominator": true,
        "geneLevelDe": true,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "trajectoryFile": "scbench-long/trajectories/gse155381_q7_xspecies_epi_communication__grok-4.20-beta-0309-reasoning__pi__r2.json",
      "runId": "grok-4.20-beta-0309-reasoning__pi__gse155381_q7_xspecies_epi_communication__r2",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-grok-4-20-beta-reasoning-pi",
      "modelName": "grok-4.20-beta-0309-reasoning",
      "harness": "pi",
      "task": "gse155381_q7_xspecies_epi_communication",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 3,
      "passed": false,
      "score": 1,
      "durationS": 62.06,
      "cost": 0.0727161,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "richness_weighted",
            "role": "additive",
            "field": "$.xspecies_signaling_richness",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_weighted",
            "role": "additive",
            "field": "$.dominant_signaling_axis",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_wnt_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "WNT_FZD_LRP_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_fgf_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "FGF_FGFR_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_notch_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "NOTCH_DLL_JAG_family"
                ]
              }
            }
          },
          {
            "name": "signaling_directionality_weighted",
            "role": "additive",
            "field": "$.signaling_directionality",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "top_signal_interaction_class_weighted",
            "role": "additive",
            "field": "$.top_signal_interaction_class",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          }
        ]
      },
      "reasoning": "all_of [pass_rule=score_threshold]: FAIL\n  + richness_weighted: passed=True, score=0.0\n  + dominant_axis_weighted: passed=True, score=0.5\n  + signaling_directionality_weighted: passed=True, score=0.25\n  + top_signal_interaction_class_weighted: passed=True, score=0.25\n  + hard_fail 'dominant_axis_wnt_unsupported_by_own_pairs': not triggered\n  + hard_fail 'dominant_axis_fgf_unsupported_by_own_pairs': not triggered\n  + hard_fail 'dominant_axis_notch_unsupported_by_own_pairs': not triggered",
      "metrics": {
        "pass_rule": "score_threshold",
        "scoring_count": 4,
        "scoring_passed": 4,
        "scoring_total_score": 1,
        "hard_fail_triggered": []
      },
      "fieldScores": {
        "richness_weighted": 0,
        "dominant_axis_weighted": 0.5,
        "signaling_directionality_weighted": 0.25,
        "top_signal_interaction_class_weighted": 0.25
      },
      "graderChecks": [
        {
          "name": "richness_weighted",
          "kind": "gate",
          "passed": true,
          "score": 0
        },
        {
          "name": "dominant_axis_weighted",
          "kind": "gate",
          "passed": true,
          "score": 0.5
        },
        {
          "name": "signaling_directionality_weighted",
          "kind": "gate",
          "passed": true,
          "score": 0.25
        },
        {
          "name": "top_signal_interaction_class_weighted",
          "kind": "gate",
          "passed": true,
          "score": 0.25
        },
        {
          "name": "dominant_axis_wnt_unsupported_by_own_pairs",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "dominant_axis_fgf_unsupported_by_own_pairs",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "dominant_axis_notch_unsupported_by_own_pairs",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "dominant_axis_support": [],
        "dominant_signaling_axis": "cannot_assess",
        "signaling_directionality": "cannot_assess",
        "top_signal_interaction_class": "cannot_assess",
        "xspecies_signaling_richness": "no_significant_cross_species_signal_detected"
      },
      "nTurns": 18,
      "checkpointSignals": {
        "clustering": false,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": true,
        "nerveJointAnchor": false,
        "multiomeAssembly": false,
        "depthNormalizedAtac": false,
        "regulonInference": false,
        "cd8SubsetIntegration": false,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": true,
        "orthologProjection": true,
        "epiSubsetSelection": true,
        "bidirectionalLrTest": true,
        "enrichmentFamilyRanking": true,
        "mousePseudobulkDesign": false,
        "cancerEpithelialDenominator": false,
        "geneLevelDe": false,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "trajectoryFile": "scbench-long/trajectories/gse155381_q7_xspecies_epi_communication__grok-4.20-beta-0309-reasoning__pi__r3.json",
      "runId": "grok-4.20-beta-0309-reasoning__pi__gse155381_q7_xspecies_epi_communication__r3",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-grok-4-3-pi",
      "modelName": "grok-4.3",
      "harness": "pi",
      "task": "gse155381_q7_xspecies_epi_communication",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 1,
      "passed": false,
      "score": 1.25,
      "durationS": 25.01,
      "cost": 0.007651650000000001,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "richness_weighted",
            "role": "additive",
            "field": "$.xspecies_signaling_richness",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_weighted",
            "role": "additive",
            "field": "$.dominant_signaling_axis",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_wnt_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "WNT_FZD_LRP_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_fgf_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "FGF_FGFR_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_notch_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "NOTCH_DLL_JAG_family"
                ]
              }
            }
          },
          {
            "name": "signaling_directionality_weighted",
            "role": "additive",
            "field": "$.signaling_directionality",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "top_signal_interaction_class_weighted",
            "role": "additive",
            "field": "$.top_signal_interaction_class",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          }
        ]
      },
      "reasoning": "all_of [pass_rule=score_threshold]: FAIL\n  + richness_weighted: passed=True, score=0.25\n  + dominant_axis_weighted: passed=True, score=0.5\n  + signaling_directionality_weighted: passed=True, score=0.25\n  + top_signal_interaction_class_weighted: passed=True, score=0.25\n  + hard_fail 'dominant_axis_wnt_unsupported_by_own_pairs': not triggered\n  + hard_fail 'dominant_axis_fgf_unsupported_by_own_pairs': not triggered\n  + hard_fail 'dominant_axis_notch_unsupported_by_own_pairs': not triggered",
      "metrics": {
        "pass_rule": "score_threshold",
        "scoring_count": 4,
        "scoring_passed": 4,
        "scoring_total_score": 1.25,
        "hard_fail_triggered": []
      },
      "fieldScores": {
        "richness_weighted": 0.25,
        "dominant_axis_weighted": 0.5,
        "signaling_directionality_weighted": 0.25,
        "top_signal_interaction_class_weighted": 0.25
      },
      "graderChecks": [
        {
          "name": "richness_weighted",
          "kind": "gate",
          "passed": true,
          "score": 0.25
        },
        {
          "name": "dominant_axis_weighted",
          "kind": "gate",
          "passed": true,
          "score": 0.5
        },
        {
          "name": "signaling_directionality_weighted",
          "kind": "gate",
          "passed": true,
          "score": 0.25
        },
        {
          "name": "top_signal_interaction_class_weighted",
          "kind": "gate",
          "passed": true,
          "score": 0.25
        },
        {
          "name": "dominant_axis_wnt_unsupported_by_own_pairs",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "dominant_axis_fgf_unsupported_by_own_pairs",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "dominant_axis_notch_unsupported_by_own_pairs",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "dominant_axis_support": [],
        "dominant_signaling_axis": "cannot_assess",
        "signaling_directionality": "cannot_assess",
        "top_signal_interaction_class": "cannot_assess",
        "xspecies_signaling_richness": "cannot_assess"
      },
      "nTurns": 5,
      "checkpointSignals": {
        "clustering": false,
        "markerAnnotation": false,
        "scoreBasedAnnotation": false,
        "normalization": true,
        "nerveJointAnchor": false,
        "multiomeAssembly": false,
        "depthNormalizedAtac": false,
        "regulonInference": false,
        "cd8SubsetIntegration": false,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": true,
        "orthologProjection": false,
        "epiSubsetSelection": true,
        "bidirectionalLrTest": true,
        "enrichmentFamilyRanking": true,
        "mousePseudobulkDesign": false,
        "cancerEpithelialDenominator": false,
        "geneLevelDe": false,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "trajectoryFile": "scbench-long/trajectories/gse155381_q7_xspecies_epi_communication__grok-4.3__pi__r1.json",
      "runId": "grok-4.3__pi__gse155381_q7_xspecies_epi_communication__r1",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-grok-4-3-pi",
      "modelName": "grok-4.3",
      "harness": "pi",
      "task": "gse155381_q7_xspecies_epi_communication",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 2,
      "passed": false,
      "score": 1.25,
      "durationS": 30.01,
      "cost": 0.0079781,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "richness_weighted",
            "role": "additive",
            "field": "$.xspecies_signaling_richness",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_weighted",
            "role": "additive",
            "field": "$.dominant_signaling_axis",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_wnt_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "WNT_FZD_LRP_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_fgf_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "FGF_FGFR_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_notch_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "NOTCH_DLL_JAG_family"
                ]
              }
            }
          },
          {
            "name": "signaling_directionality_weighted",
            "role": "additive",
            "field": "$.signaling_directionality",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "top_signal_interaction_class_weighted",
            "role": "additive",
            "field": "$.top_signal_interaction_class",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          }
        ]
      },
      "reasoning": "all_of [pass_rule=score_threshold]: FAIL\n  + richness_weighted: passed=True, score=0.25\n  + dominant_axis_weighted: passed=True, score=0.5\n  + signaling_directionality_weighted: passed=True, score=0.25\n  + top_signal_interaction_class_weighted: passed=True, score=0.25\n  + hard_fail 'dominant_axis_wnt_unsupported_by_own_pairs': not triggered\n  + hard_fail 'dominant_axis_fgf_unsupported_by_own_pairs': not triggered\n  + hard_fail 'dominant_axis_notch_unsupported_by_own_pairs': not triggered",
      "metrics": {
        "pass_rule": "score_threshold",
        "scoring_count": 4,
        "scoring_passed": 4,
        "scoring_total_score": 1.25,
        "hard_fail_triggered": []
      },
      "fieldScores": {
        "richness_weighted": 0.25,
        "dominant_axis_weighted": 0.5,
        "signaling_directionality_weighted": 0.25,
        "top_signal_interaction_class_weighted": 0.25
      },
      "graderChecks": [
        {
          "name": "richness_weighted",
          "kind": "gate",
          "passed": true,
          "score": 0.25
        },
        {
          "name": "dominant_axis_weighted",
          "kind": "gate",
          "passed": true,
          "score": 0.5
        },
        {
          "name": "signaling_directionality_weighted",
          "kind": "gate",
          "passed": true,
          "score": 0.25
        },
        {
          "name": "top_signal_interaction_class_weighted",
          "kind": "gate",
          "passed": true,
          "score": 0.25
        },
        {
          "name": "dominant_axis_wnt_unsupported_by_own_pairs",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "dominant_axis_fgf_unsupported_by_own_pairs",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "dominant_axis_notch_unsupported_by_own_pairs",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "dominant_axis_support": [],
        "dominant_signaling_axis": "cannot_assess",
        "signaling_directionality": "cannot_assess",
        "top_signal_interaction_class": "cannot_assess",
        "xspecies_signaling_richness": "cannot_assess"
      },
      "nTurns": 6,
      "checkpointSignals": {
        "clustering": false,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": true,
        "nerveJointAnchor": false,
        "multiomeAssembly": false,
        "depthNormalizedAtac": false,
        "regulonInference": false,
        "cd8SubsetIntegration": false,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": true,
        "orthologProjection": false,
        "epiSubsetSelection": true,
        "bidirectionalLrTest": true,
        "enrichmentFamilyRanking": false,
        "mousePseudobulkDesign": false,
        "cancerEpithelialDenominator": false,
        "geneLevelDe": false,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "trajectoryFile": "scbench-long/trajectories/gse155381_q7_xspecies_epi_communication__grok-4.3__pi__r2.json",
      "runId": "grok-4.3__pi__gse155381_q7_xspecies_epi_communication__r2",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-grok-4-3-pi",
      "modelName": "grok-4.3",
      "harness": "pi",
      "task": "gse155381_q7_xspecies_epi_communication",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 3,
      "passed": false,
      "score": 1.25,
      "durationS": 26.07,
      "cost": 0.008452400000000002,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "richness_weighted",
            "role": "additive",
            "field": "$.xspecies_signaling_richness",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_weighted",
            "role": "additive",
            "field": "$.dominant_signaling_axis",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_wnt_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "WNT_FZD_LRP_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_fgf_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "FGF_FGFR_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_notch_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "NOTCH_DLL_JAG_family"
                ]
              }
            }
          },
          {
            "name": "signaling_directionality_weighted",
            "role": "additive",
            "field": "$.signaling_directionality",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "top_signal_interaction_class_weighted",
            "role": "additive",
            "field": "$.top_signal_interaction_class",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          }
        ]
      },
      "reasoning": "all_of [pass_rule=score_threshold]: FAIL\n  + richness_weighted: passed=True, score=0.25\n  + dominant_axis_weighted: passed=True, score=0.5\n  + signaling_directionality_weighted: passed=True, score=0.25\n  + top_signal_interaction_class_weighted: passed=True, score=0.25\n  + hard_fail 'dominant_axis_wnt_unsupported_by_own_pairs': not triggered\n  + hard_fail 'dominant_axis_fgf_unsupported_by_own_pairs': not triggered\n  + hard_fail 'dominant_axis_notch_unsupported_by_own_pairs': not triggered",
      "metrics": {
        "pass_rule": "score_threshold",
        "scoring_count": 4,
        "scoring_passed": 4,
        "scoring_total_score": 1.25,
        "hard_fail_triggered": []
      },
      "fieldScores": {
        "richness_weighted": 0.25,
        "dominant_axis_weighted": 0.5,
        "signaling_directionality_weighted": 0.25,
        "top_signal_interaction_class_weighted": 0.25
      },
      "graderChecks": [
        {
          "name": "richness_weighted",
          "kind": "gate",
          "passed": true,
          "score": 0.25
        },
        {
          "name": "dominant_axis_weighted",
          "kind": "gate",
          "passed": true,
          "score": 0.5
        },
        {
          "name": "signaling_directionality_weighted",
          "kind": "gate",
          "passed": true,
          "score": 0.25
        },
        {
          "name": "top_signal_interaction_class_weighted",
          "kind": "gate",
          "passed": true,
          "score": 0.25
        },
        {
          "name": "dominant_axis_wnt_unsupported_by_own_pairs",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "dominant_axis_fgf_unsupported_by_own_pairs",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "dominant_axis_notch_unsupported_by_own_pairs",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "dominant_axis_support": [],
        "dominant_signaling_axis": "cannot_assess",
        "signaling_directionality": "cannot_assess",
        "top_signal_interaction_class": "cannot_assess",
        "xspecies_signaling_richness": "cannot_assess"
      },
      "nTurns": 5,
      "checkpointSignals": {
        "clustering": false,
        "markerAnnotation": false,
        "scoreBasedAnnotation": false,
        "normalization": true,
        "nerveJointAnchor": false,
        "multiomeAssembly": false,
        "depthNormalizedAtac": false,
        "regulonInference": false,
        "cd8SubsetIntegration": false,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": true,
        "orthologProjection": false,
        "epiSubsetSelection": true,
        "bidirectionalLrTest": true,
        "enrichmentFamilyRanking": true,
        "mousePseudobulkDesign": false,
        "cancerEpithelialDenominator": false,
        "geneLevelDe": false,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "trajectoryFile": "scbench-long/trajectories/gse155381_q7_xspecies_epi_communication__grok-4.3__pi__r3.json",
      "runId": "grok-4.3__pi__gse155381_q7_xspecies_epi_communication__r3",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-kimi-k2p6-pi",
      "modelName": "kimi-k2p6",
      "harness": "pi",
      "task": "gse155381_q7_xspecies_epi_communication",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 1,
      "passed": false,
      "score": 0,
      "durationS": 2897.32,
      "cost": 0.47690201999999987,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "richness_weighted",
            "role": "additive",
            "field": "$.xspecies_signaling_richness",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_weighted",
            "role": "additive",
            "field": "$.dominant_signaling_axis",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_wnt_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "WNT_FZD_LRP_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_fgf_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "FGF_FGFR_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_notch_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "NOTCH_DLL_JAG_family"
                ]
              }
            }
          },
          {
            "name": "signaling_directionality_weighted",
            "role": "additive",
            "field": "$.signaling_directionality",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "top_signal_interaction_class_weighted",
            "role": "additive",
            "field": "$.top_signal_interaction_class",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          }
        ]
      },
      "reasoning": "",
      "metrics": {},
      "fieldScores": {},
      "graderChecks": [],
      "agentAnswer": null,
      "nTurns": 45,
      "checkpointSignals": {
        "clustering": false,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": true,
        "nerveJointAnchor": false,
        "multiomeAssembly": false,
        "depthNormalizedAtac": false,
        "regulonInference": false,
        "cd8SubsetIntegration": true,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": true,
        "orthologProjection": true,
        "epiSubsetSelection": true,
        "bidirectionalLrTest": true,
        "enrichmentFamilyRanking": true,
        "mousePseudobulkDesign": false,
        "cancerEpithelialDenominator": true,
        "geneLevelDe": true,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "trajectoryFile": "scbench-long/trajectories/gse155381_q7_xspecies_epi_communication__kimi-k2p6__pi__r1.json",
      "runId": "kimi-k2p6__pi__gse155381_q7_xspecies_epi_communication__r1",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-kimi-k2p6-pi",
      "modelName": "kimi-k2p6",
      "harness": "pi",
      "task": "gse155381_q7_xspecies_epi_communication",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 2,
      "passed": false,
      "score": 3,
      "durationS": 4303.34,
      "cost": 1.63359069,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "richness_weighted",
            "role": "additive",
            "field": "$.xspecies_signaling_richness",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_weighted",
            "role": "additive",
            "field": "$.dominant_signaling_axis",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_wnt_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "WNT_FZD_LRP_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_fgf_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "FGF_FGFR_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_notch_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "NOTCH_DLL_JAG_family"
                ]
              }
            }
          },
          {
            "name": "signaling_directionality_weighted",
            "role": "additive",
            "field": "$.signaling_directionality",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "top_signal_interaction_class_weighted",
            "role": "additive",
            "field": "$.top_signal_interaction_class",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          }
        ]
      },
      "reasoning": "all_of [pass_rule=score_threshold]: FAIL\n  + richness_weighted: passed=True, score=0.0\n  + dominant_axis_weighted: passed=True, score=2.0\n  + signaling_directionality_weighted: passed=True, score=0.0\n  + top_signal_interaction_class_weighted: passed=True, score=1.0\n  + hard_fail 'dominant_axis_wnt_unsupported_by_own_pairs': not triggered\n  + hard_fail 'dominant_axis_fgf_unsupported_by_own_pairs': not triggered\n  + hard_fail 'dominant_axis_notch_unsupported_by_own_pairs': not triggered",
      "metrics": {
        "pass_rule": "score_threshold",
        "scoring_count": 4,
        "scoring_passed": 4,
        "scoring_total_score": 3,
        "hard_fail_triggered": []
      },
      "fieldScores": {
        "richness_weighted": 0,
        "dominant_axis_weighted": 2,
        "signaling_directionality_weighted": 0,
        "top_signal_interaction_class_weighted": 1
      },
      "graderChecks": [
        {
          "name": "richness_weighted",
          "kind": "gate",
          "passed": true,
          "score": 0
        },
        {
          "name": "dominant_axis_weighted",
          "kind": "gate",
          "passed": true,
          "score": 2
        },
        {
          "name": "signaling_directionality_weighted",
          "kind": "gate",
          "passed": true,
          "score": 0
        },
        {
          "name": "top_signal_interaction_class_weighted",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "dominant_axis_wnt_unsupported_by_own_pairs",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "dominant_axis_fgf_unsupported_by_own_pairs",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "dominant_axis_notch_unsupported_by_own_pairs",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "dominant_axis_support": [
          "LAMA1",
          "RPSA",
          "LAMB2",
          "MDK",
          "NCL",
          "PKM",
          "CD44",
          "AGTRAP",
          "RACK1",
          "APOB",
          "ENO1",
          "BDNF",
          "BEX3",
          "ITGA6",
          "SDC2",
          "HBEGF",
          "CD9",
          "PTPRZ1",
          "LRP1",
          "SORL1",
          "NOTCH2",
          "SDC3",
          "GPC2",
          "ALK",
          "GSTP1",
          "TRAF2",
          "EGFR",
          "TIMP1",
          "CD63"
        ],
        "dominant_signaling_axis": "no_single_family_dominates",
        "signaling_directionality": "signal_appears_bidirectional_but_attributable_to_expression_level_asymmetry",
        "top_signal_interaction_class": "protein_ligand_receptor_dominant",
        "xspecies_signaling_richness": "signal_appears_significant_but_attributable_to_cluster_size_asymmetry"
      },
      "nTurns": 49,
      "checkpointSignals": {
        "clustering": false,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": true,
        "nerveJointAnchor": false,
        "multiomeAssembly": true,
        "depthNormalizedAtac": false,
        "regulonInference": false,
        "cd8SubsetIntegration": true,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": true,
        "orthologProjection": true,
        "epiSubsetSelection": true,
        "bidirectionalLrTest": true,
        "enrichmentFamilyRanking": true,
        "mousePseudobulkDesign": true,
        "cancerEpithelialDenominator": false,
        "geneLevelDe": true,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "trajectoryFile": "scbench-long/trajectories/gse155381_q7_xspecies_epi_communication__kimi-k2p6__pi__r2.json",
      "runId": "kimi-k2p6__pi__gse155381_q7_xspecies_epi_communication__r2",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-opus-4-6-claude-code",
      "modelName": "claude-opus-4-6",
      "harness": "claude-code",
      "task": "gse155381_q8_bidirectional_convergence",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 728,
      "cost": 1.49487075,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "chimera_human_identity_shift_target_weighted",
            "role": "additive",
            "field": "$.chimera_human_identity_shift_target",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_in_vivo_state_assignment_weighted",
            "role": "additive",
            "field": "$.chimera_human_in_vivo_state_assignment",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_shift_magnitude_weighted",
            "role": "additive",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_monkey_identity_shift_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "both_arms_null_contradicted_by_calibration",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "chimera_human_identity_shift_target": {
                "op": "equals",
                "values": [
                  "chimera_human_maintains_origin_identity_no_measurable_shift"
                ]
              },
              "chimera_monkey_identity_shift": {
                "op": "equals",
                "values": [
                  "chimera_monkey_unchanged_relative_to_control"
                ]
              }
            }
          },
          {
            "name": "chimera_monkey_paper_claim_on_confounded_input",
            "role": "hard_fail",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "value",
            "expected": "chimera_monkey_shifts_toward_chimera_human_identity"
          },
          {
            "name": "chimera_human_magnitude_wrong_attribution_signature",
            "role": "hard_fail",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "value",
            "op": "in",
            "expected": [
              "dramatic_complete_replacement_of_origin_identity",
              "magnitude_dominated_by_culture_or_metabolic_axis",
              "magnitude_dominated_by_developmental_stage_axis",
              "magnitude_dominated_by_cell_cycle_axis",
              "magnitude_dominated_by_batch_effect_axis"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-6-claude-code",
      "modelName": "claude-opus-4-6",
      "harness": "claude-code",
      "task": "gse155381_q8_bidirectional_convergence",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 778,
      "cost": 1.5629055,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "chimera_human_identity_shift_target_weighted",
            "role": "additive",
            "field": "$.chimera_human_identity_shift_target",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_in_vivo_state_assignment_weighted",
            "role": "additive",
            "field": "$.chimera_human_in_vivo_state_assignment",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_shift_magnitude_weighted",
            "role": "additive",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_monkey_identity_shift_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "both_arms_null_contradicted_by_calibration",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "chimera_human_identity_shift_target": {
                "op": "equals",
                "values": [
                  "chimera_human_maintains_origin_identity_no_measurable_shift"
                ]
              },
              "chimera_monkey_identity_shift": {
                "op": "equals",
                "values": [
                  "chimera_monkey_unchanged_relative_to_control"
                ]
              }
            }
          },
          {
            "name": "chimera_monkey_paper_claim_on_confounded_input",
            "role": "hard_fail",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "value",
            "expected": "chimera_monkey_shifts_toward_chimera_human_identity"
          },
          {
            "name": "chimera_human_magnitude_wrong_attribution_signature",
            "role": "hard_fail",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "value",
            "op": "in",
            "expected": [
              "dramatic_complete_replacement_of_origin_identity",
              "magnitude_dominated_by_culture_or_metabolic_axis",
              "magnitude_dominated_by_developmental_stage_axis",
              "magnitude_dominated_by_cell_cycle_axis",
              "magnitude_dominated_by_batch_effect_axis"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-6-claude-code",
      "modelName": "claude-opus-4-6",
      "harness": "claude-code",
      "task": "gse155381_q8_bidirectional_convergence",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1358,
      "cost": 2.7161457500000004,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "chimera_human_identity_shift_target_weighted",
            "role": "additive",
            "field": "$.chimera_human_identity_shift_target",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_in_vivo_state_assignment_weighted",
            "role": "additive",
            "field": "$.chimera_human_in_vivo_state_assignment",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_shift_magnitude_weighted",
            "role": "additive",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_monkey_identity_shift_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "both_arms_null_contradicted_by_calibration",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "chimera_human_identity_shift_target": {
                "op": "equals",
                "values": [
                  "chimera_human_maintains_origin_identity_no_measurable_shift"
                ]
              },
              "chimera_monkey_identity_shift": {
                "op": "equals",
                "values": [
                  "chimera_monkey_unchanged_relative_to_control"
                ]
              }
            }
          },
          {
            "name": "chimera_monkey_paper_claim_on_confounded_input",
            "role": "hard_fail",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "value",
            "expected": "chimera_monkey_shifts_toward_chimera_human_identity"
          },
          {
            "name": "chimera_human_magnitude_wrong_attribution_signature",
            "role": "hard_fail",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "value",
            "op": "in",
            "expected": [
              "dramatic_complete_replacement_of_origin_identity",
              "magnitude_dominated_by_culture_or_metabolic_axis",
              "magnitude_dominated_by_developmental_stage_axis",
              "magnitude_dominated_by_cell_cycle_axis",
              "magnitude_dominated_by_batch_effect_axis"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-6-pi",
      "modelName": "claude-opus-4-6",
      "harness": "pi",
      "task": "gse155381_q8_bidirectional_convergence",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 1578,
      "cost": 2.8606357499999993,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "chimera_human_identity_shift_target_weighted",
            "role": "additive",
            "field": "$.chimera_human_identity_shift_target",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_in_vivo_state_assignment_weighted",
            "role": "additive",
            "field": "$.chimera_human_in_vivo_state_assignment",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_shift_magnitude_weighted",
            "role": "additive",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_monkey_identity_shift_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "both_arms_null_contradicted_by_calibration",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "chimera_human_identity_shift_target": {
                "op": "equals",
                "values": [
                  "chimera_human_maintains_origin_identity_no_measurable_shift"
                ]
              },
              "chimera_monkey_identity_shift": {
                "op": "equals",
                "values": [
                  "chimera_monkey_unchanged_relative_to_control"
                ]
              }
            }
          },
          {
            "name": "chimera_monkey_paper_claim_on_confounded_input",
            "role": "hard_fail",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "value",
            "expected": "chimera_monkey_shifts_toward_chimera_human_identity"
          },
          {
            "name": "chimera_human_magnitude_wrong_attribution_signature",
            "role": "hard_fail",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "value",
            "op": "in",
            "expected": [
              "dramatic_complete_replacement_of_origin_identity",
              "magnitude_dominated_by_culture_or_metabolic_axis",
              "magnitude_dominated_by_developmental_stage_axis",
              "magnitude_dominated_by_cell_cycle_axis",
              "magnitude_dominated_by_batch_effect_axis"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-6-pi",
      "modelName": "claude-opus-4-6",
      "harness": "pi",
      "task": "gse155381_q8_bidirectional_convergence",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 1683,
      "cost": 3.14939875,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "chimera_human_identity_shift_target_weighted",
            "role": "additive",
            "field": "$.chimera_human_identity_shift_target",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_in_vivo_state_assignment_weighted",
            "role": "additive",
            "field": "$.chimera_human_in_vivo_state_assignment",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_shift_magnitude_weighted",
            "role": "additive",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_monkey_identity_shift_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "both_arms_null_contradicted_by_calibration",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "chimera_human_identity_shift_target": {
                "op": "equals",
                "values": [
                  "chimera_human_maintains_origin_identity_no_measurable_shift"
                ]
              },
              "chimera_monkey_identity_shift": {
                "op": "equals",
                "values": [
                  "chimera_monkey_unchanged_relative_to_control"
                ]
              }
            }
          },
          {
            "name": "chimera_monkey_paper_claim_on_confounded_input",
            "role": "hard_fail",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "value",
            "expected": "chimera_monkey_shifts_toward_chimera_human_identity"
          },
          {
            "name": "chimera_human_magnitude_wrong_attribution_signature",
            "role": "hard_fail",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "value",
            "op": "in",
            "expected": [
              "dramatic_complete_replacement_of_origin_identity",
              "magnitude_dominated_by_culture_or_metabolic_axis",
              "magnitude_dominated_by_developmental_stage_axis",
              "magnitude_dominated_by_cell_cycle_axis",
              "magnitude_dominated_by_batch_effect_axis"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-6-pi",
      "modelName": "claude-opus-4-6",
      "harness": "pi",
      "task": "gse155381_q8_bidirectional_convergence",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1534,
      "cost": 3.0496789999999994,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "chimera_human_identity_shift_target_weighted",
            "role": "additive",
            "field": "$.chimera_human_identity_shift_target",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_in_vivo_state_assignment_weighted",
            "role": "additive",
            "field": "$.chimera_human_in_vivo_state_assignment",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_shift_magnitude_weighted",
            "role": "additive",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_monkey_identity_shift_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "both_arms_null_contradicted_by_calibration",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "chimera_human_identity_shift_target": {
                "op": "equals",
                "values": [
                  "chimera_human_maintains_origin_identity_no_measurable_shift"
                ]
              },
              "chimera_monkey_identity_shift": {
                "op": "equals",
                "values": [
                  "chimera_monkey_unchanged_relative_to_control"
                ]
              }
            }
          },
          {
            "name": "chimera_monkey_paper_claim_on_confounded_input",
            "role": "hard_fail",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "value",
            "expected": "chimera_monkey_shifts_toward_chimera_human_identity"
          },
          {
            "name": "chimera_human_magnitude_wrong_attribution_signature",
            "role": "hard_fail",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "value",
            "op": "in",
            "expected": [
              "dramatic_complete_replacement_of_origin_identity",
              "magnitude_dominated_by_culture_or_metabolic_axis",
              "magnitude_dominated_by_developmental_stage_axis",
              "magnitude_dominated_by_cell_cycle_axis",
              "magnitude_dominated_by_batch_effect_axis"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-7-claude-code",
      "modelName": "claude-opus-4-7",
      "harness": "claude-code",
      "task": "gse155381_q8_bidirectional_convergence",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 1817,
      "cost": 7.434803750000001,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "chimera_human_identity_shift_target_weighted",
            "role": "additive",
            "field": "$.chimera_human_identity_shift_target",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_in_vivo_state_assignment_weighted",
            "role": "additive",
            "field": "$.chimera_human_in_vivo_state_assignment",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_shift_magnitude_weighted",
            "role": "additive",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_monkey_identity_shift_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "both_arms_null_contradicted_by_calibration",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "chimera_human_identity_shift_target": {
                "op": "equals",
                "values": [
                  "chimera_human_maintains_origin_identity_no_measurable_shift"
                ]
              },
              "chimera_monkey_identity_shift": {
                "op": "equals",
                "values": [
                  "chimera_monkey_unchanged_relative_to_control"
                ]
              }
            }
          },
          {
            "name": "chimera_monkey_paper_claim_on_confounded_input",
            "role": "hard_fail",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "value",
            "expected": "chimera_monkey_shifts_toward_chimera_human_identity"
          },
          {
            "name": "chimera_human_magnitude_wrong_attribution_signature",
            "role": "hard_fail",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "value",
            "op": "in",
            "expected": [
              "dramatic_complete_replacement_of_origin_identity",
              "magnitude_dominated_by_culture_or_metabolic_axis",
              "magnitude_dominated_by_developmental_stage_axis",
              "magnitude_dominated_by_cell_cycle_axis",
              "magnitude_dominated_by_batch_effect_axis"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-7-claude-code",
      "modelName": "claude-opus-4-7",
      "harness": "claude-code",
      "task": "gse155381_q8_bidirectional_convergence",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 2023,
      "cost": 8.580204500000002,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "chimera_human_identity_shift_target_weighted",
            "role": "additive",
            "field": "$.chimera_human_identity_shift_target",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_in_vivo_state_assignment_weighted",
            "role": "additive",
            "field": "$.chimera_human_in_vivo_state_assignment",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_shift_magnitude_weighted",
            "role": "additive",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_monkey_identity_shift_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "both_arms_null_contradicted_by_calibration",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "chimera_human_identity_shift_target": {
                "op": "equals",
                "values": [
                  "chimera_human_maintains_origin_identity_no_measurable_shift"
                ]
              },
              "chimera_monkey_identity_shift": {
                "op": "equals",
                "values": [
                  "chimera_monkey_unchanged_relative_to_control"
                ]
              }
            }
          },
          {
            "name": "chimera_monkey_paper_claim_on_confounded_input",
            "role": "hard_fail",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "value",
            "expected": "chimera_monkey_shifts_toward_chimera_human_identity"
          },
          {
            "name": "chimera_human_magnitude_wrong_attribution_signature",
            "role": "hard_fail",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "value",
            "op": "in",
            "expected": [
              "dramatic_complete_replacement_of_origin_identity",
              "magnitude_dominated_by_culture_or_metabolic_axis",
              "magnitude_dominated_by_developmental_stage_axis",
              "magnitude_dominated_by_cell_cycle_axis",
              "magnitude_dominated_by_batch_effect_axis"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-7-claude-code",
      "modelName": "claude-opus-4-7",
      "harness": "claude-code",
      "task": "gse155381_q8_bidirectional_convergence",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1617,
      "cost": 5.94424325,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "chimera_human_identity_shift_target_weighted",
            "role": "additive",
            "field": "$.chimera_human_identity_shift_target",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_in_vivo_state_assignment_weighted",
            "role": "additive",
            "field": "$.chimera_human_in_vivo_state_assignment",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_shift_magnitude_weighted",
            "role": "additive",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_monkey_identity_shift_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "both_arms_null_contradicted_by_calibration",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "chimera_human_identity_shift_target": {
                "op": "equals",
                "values": [
                  "chimera_human_maintains_origin_identity_no_measurable_shift"
                ]
              },
              "chimera_monkey_identity_shift": {
                "op": "equals",
                "values": [
                  "chimera_monkey_unchanged_relative_to_control"
                ]
              }
            }
          },
          {
            "name": "chimera_monkey_paper_claim_on_confounded_input",
            "role": "hard_fail",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "value",
            "expected": "chimera_monkey_shifts_toward_chimera_human_identity"
          },
          {
            "name": "chimera_human_magnitude_wrong_attribution_signature",
            "role": "hard_fail",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "value",
            "op": "in",
            "expected": [
              "dramatic_complete_replacement_of_origin_identity",
              "magnitude_dominated_by_culture_or_metabolic_axis",
              "magnitude_dominated_by_developmental_stage_axis",
              "magnitude_dominated_by_cell_cycle_axis",
              "magnitude_dominated_by_batch_effect_axis"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-7-pi",
      "modelName": "claude-opus-4-7",
      "harness": "pi",
      "task": "gse155381_q8_bidirectional_convergence",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 485,
      "cost": 1.2311937500000003,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "chimera_human_identity_shift_target_weighted",
            "role": "additive",
            "field": "$.chimera_human_identity_shift_target",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_in_vivo_state_assignment_weighted",
            "role": "additive",
            "field": "$.chimera_human_in_vivo_state_assignment",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_shift_magnitude_weighted",
            "role": "additive",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_monkey_identity_shift_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "both_arms_null_contradicted_by_calibration",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "chimera_human_identity_shift_target": {
                "op": "equals",
                "values": [
                  "chimera_human_maintains_origin_identity_no_measurable_shift"
                ]
              },
              "chimera_monkey_identity_shift": {
                "op": "equals",
                "values": [
                  "chimera_monkey_unchanged_relative_to_control"
                ]
              }
            }
          },
          {
            "name": "chimera_monkey_paper_claim_on_confounded_input",
            "role": "hard_fail",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "value",
            "expected": "chimera_monkey_shifts_toward_chimera_human_identity"
          },
          {
            "name": "chimera_human_magnitude_wrong_attribution_signature",
            "role": "hard_fail",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "value",
            "op": "in",
            "expected": [
              "dramatic_complete_replacement_of_origin_identity",
              "magnitude_dominated_by_culture_or_metabolic_axis",
              "magnitude_dominated_by_developmental_stage_axis",
              "magnitude_dominated_by_cell_cycle_axis",
              "magnitude_dominated_by_batch_effect_axis"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-7-pi",
      "modelName": "claude-opus-4-7",
      "harness": "pi",
      "task": "gse155381_q8_bidirectional_convergence",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 971,
      "cost": 2.491102750000001,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "chimera_human_identity_shift_target_weighted",
            "role": "additive",
            "field": "$.chimera_human_identity_shift_target",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_in_vivo_state_assignment_weighted",
            "role": "additive",
            "field": "$.chimera_human_in_vivo_state_assignment",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_shift_magnitude_weighted",
            "role": "additive",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_monkey_identity_shift_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "both_arms_null_contradicted_by_calibration",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "chimera_human_identity_shift_target": {
                "op": "equals",
                "values": [
                  "chimera_human_maintains_origin_identity_no_measurable_shift"
                ]
              },
              "chimera_monkey_identity_shift": {
                "op": "equals",
                "values": [
                  "chimera_monkey_unchanged_relative_to_control"
                ]
              }
            }
          },
          {
            "name": "chimera_monkey_paper_claim_on_confounded_input",
            "role": "hard_fail",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "value",
            "expected": "chimera_monkey_shifts_toward_chimera_human_identity"
          },
          {
            "name": "chimera_human_magnitude_wrong_attribution_signature",
            "role": "hard_fail",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "value",
            "op": "in",
            "expected": [
              "dramatic_complete_replacement_of_origin_identity",
              "magnitude_dominated_by_culture_or_metabolic_axis",
              "magnitude_dominated_by_developmental_stage_axis",
              "magnitude_dominated_by_cell_cycle_axis",
              "magnitude_dominated_by_batch_effect_axis"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-7-pi",
      "modelName": "claude-opus-4-7",
      "harness": "pi",
      "task": "gse155381_q8_bidirectional_convergence",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 827,
      "cost": 2.6287652500000007,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "chimera_human_identity_shift_target_weighted",
            "role": "additive",
            "field": "$.chimera_human_identity_shift_target",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_in_vivo_state_assignment_weighted",
            "role": "additive",
            "field": "$.chimera_human_in_vivo_state_assignment",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_shift_magnitude_weighted",
            "role": "additive",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_monkey_identity_shift_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "both_arms_null_contradicted_by_calibration",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "chimera_human_identity_shift_target": {
                "op": "equals",
                "values": [
                  "chimera_human_maintains_origin_identity_no_measurable_shift"
                ]
              },
              "chimera_monkey_identity_shift": {
                "op": "equals",
                "values": [
                  "chimera_monkey_unchanged_relative_to_control"
                ]
              }
            }
          },
          {
            "name": "chimera_monkey_paper_claim_on_confounded_input",
            "role": "hard_fail",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "value",
            "expected": "chimera_monkey_shifts_toward_chimera_human_identity"
          },
          {
            "name": "chimera_human_magnitude_wrong_attribution_signature",
            "role": "hard_fail",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "value",
            "op": "in",
            "expected": [
              "dramatic_complete_replacement_of_origin_identity",
              "magnitude_dominated_by_culture_or_metabolic_axis",
              "magnitude_dominated_by_developmental_stage_axis",
              "magnitude_dominated_by_cell_cycle_axis",
              "magnitude_dominated_by_batch_effect_axis"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-8-claude-code",
      "modelName": "claude-opus-4-8",
      "harness": "claude-code",
      "task": "gse155381_q8_bidirectional_convergence",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 2208,
      "cost": 8.128791499999998,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "chimera_human_identity_shift_target_weighted",
            "role": "additive",
            "field": "$.chimera_human_identity_shift_target",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_in_vivo_state_assignment_weighted",
            "role": "additive",
            "field": "$.chimera_human_in_vivo_state_assignment",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_shift_magnitude_weighted",
            "role": "additive",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_monkey_identity_shift_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "both_arms_null_contradicted_by_calibration",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "chimera_human_identity_shift_target": {
                "op": "equals",
                "values": [
                  "chimera_human_maintains_origin_identity_no_measurable_shift"
                ]
              },
              "chimera_monkey_identity_shift": {
                "op": "equals",
                "values": [
                  "chimera_monkey_unchanged_relative_to_control"
                ]
              }
            }
          },
          {
            "name": "chimera_monkey_paper_claim_on_confounded_input",
            "role": "hard_fail",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "value",
            "expected": "chimera_monkey_shifts_toward_chimera_human_identity"
          },
          {
            "name": "chimera_human_magnitude_wrong_attribution_signature",
            "role": "hard_fail",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "value",
            "op": "in",
            "expected": [
              "dramatic_complete_replacement_of_origin_identity",
              "magnitude_dominated_by_culture_or_metabolic_axis",
              "magnitude_dominated_by_developmental_stage_axis",
              "magnitude_dominated_by_cell_cycle_axis",
              "magnitude_dominated_by_batch_effect_axis"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-8-claude-code",
      "modelName": "claude-opus-4-8",
      "harness": "claude-code",
      "task": "gse155381_q8_bidirectional_convergence",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 2,
      "passed": true,
      "score": null,
      "durationS": 2360,
      "cost": 8.90284925,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "chimera_human_identity_shift_target_weighted",
            "role": "additive",
            "field": "$.chimera_human_identity_shift_target",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_in_vivo_state_assignment_weighted",
            "role": "additive",
            "field": "$.chimera_human_in_vivo_state_assignment",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_shift_magnitude_weighted",
            "role": "additive",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_monkey_identity_shift_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "both_arms_null_contradicted_by_calibration",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "chimera_human_identity_shift_target": {
                "op": "equals",
                "values": [
                  "chimera_human_maintains_origin_identity_no_measurable_shift"
                ]
              },
              "chimera_monkey_identity_shift": {
                "op": "equals",
                "values": [
                  "chimera_monkey_unchanged_relative_to_control"
                ]
              }
            }
          },
          {
            "name": "chimera_monkey_paper_claim_on_confounded_input",
            "role": "hard_fail",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "value",
            "expected": "chimera_monkey_shifts_toward_chimera_human_identity"
          },
          {
            "name": "chimera_human_magnitude_wrong_attribution_signature",
            "role": "hard_fail",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "value",
            "op": "in",
            "expected": [
              "dramatic_complete_replacement_of_origin_identity",
              "magnitude_dominated_by_culture_or_metabolic_axis",
              "magnitude_dominated_by_developmental_stage_axis",
              "magnitude_dominated_by_cell_cycle_axis",
              "magnitude_dominated_by_batch_effect_axis"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-8-claude-code",
      "modelName": "claude-opus-4-8",
      "harness": "claude-code",
      "task": "gse155381_q8_bidirectional_convergence",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 2569,
      "cost": 9.6295705,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "chimera_human_identity_shift_target_weighted",
            "role": "additive",
            "field": "$.chimera_human_identity_shift_target",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_in_vivo_state_assignment_weighted",
            "role": "additive",
            "field": "$.chimera_human_in_vivo_state_assignment",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_shift_magnitude_weighted",
            "role": "additive",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_monkey_identity_shift_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "both_arms_null_contradicted_by_calibration",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "chimera_human_identity_shift_target": {
                "op": "equals",
                "values": [
                  "chimera_human_maintains_origin_identity_no_measurable_shift"
                ]
              },
              "chimera_monkey_identity_shift": {
                "op": "equals",
                "values": [
                  "chimera_monkey_unchanged_relative_to_control"
                ]
              }
            }
          },
          {
            "name": "chimera_monkey_paper_claim_on_confounded_input",
            "role": "hard_fail",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "value",
            "expected": "chimera_monkey_shifts_toward_chimera_human_identity"
          },
          {
            "name": "chimera_human_magnitude_wrong_attribution_signature",
            "role": "hard_fail",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "value",
            "op": "in",
            "expected": [
              "dramatic_complete_replacement_of_origin_identity",
              "magnitude_dominated_by_culture_or_metabolic_axis",
              "magnitude_dominated_by_developmental_stage_axis",
              "magnitude_dominated_by_cell_cycle_axis",
              "magnitude_dominated_by_batch_effect_axis"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-8-pi",
      "modelName": "claude-opus-4-8",
      "harness": "pi",
      "task": "gse155381_q8_bidirectional_convergence",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 1,
      "passed": true,
      "score": null,
      "durationS": 125,
      "cost": 0.22920300000000002,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "chimera_human_identity_shift_target_weighted",
            "role": "additive",
            "field": "$.chimera_human_identity_shift_target",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_in_vivo_state_assignment_weighted",
            "role": "additive",
            "field": "$.chimera_human_in_vivo_state_assignment",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_shift_magnitude_weighted",
            "role": "additive",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_monkey_identity_shift_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "both_arms_null_contradicted_by_calibration",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "chimera_human_identity_shift_target": {
                "op": "equals",
                "values": [
                  "chimera_human_maintains_origin_identity_no_measurable_shift"
                ]
              },
              "chimera_monkey_identity_shift": {
                "op": "equals",
                "values": [
                  "chimera_monkey_unchanged_relative_to_control"
                ]
              }
            }
          },
          {
            "name": "chimera_monkey_paper_claim_on_confounded_input",
            "role": "hard_fail",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "value",
            "expected": "chimera_monkey_shifts_toward_chimera_human_identity"
          },
          {
            "name": "chimera_human_magnitude_wrong_attribution_signature",
            "role": "hard_fail",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "value",
            "op": "in",
            "expected": [
              "dramatic_complete_replacement_of_origin_identity",
              "magnitude_dominated_by_culture_or_metabolic_axis",
              "magnitude_dominated_by_developmental_stage_axis",
              "magnitude_dominated_by_cell_cycle_axis",
              "magnitude_dominated_by_batch_effect_axis"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-8-pi",
      "modelName": "claude-opus-4-8",
      "harness": "pi",
      "task": "gse155381_q8_bidirectional_convergence",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 1487,
      "cost": 3.851948750000001,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "chimera_human_identity_shift_target_weighted",
            "role": "additive",
            "field": "$.chimera_human_identity_shift_target",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_in_vivo_state_assignment_weighted",
            "role": "additive",
            "field": "$.chimera_human_in_vivo_state_assignment",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_shift_magnitude_weighted",
            "role": "additive",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_monkey_identity_shift_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "both_arms_null_contradicted_by_calibration",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "chimera_human_identity_shift_target": {
                "op": "equals",
                "values": [
                  "chimera_human_maintains_origin_identity_no_measurable_shift"
                ]
              },
              "chimera_monkey_identity_shift": {
                "op": "equals",
                "values": [
                  "chimera_monkey_unchanged_relative_to_control"
                ]
              }
            }
          },
          {
            "name": "chimera_monkey_paper_claim_on_confounded_input",
            "role": "hard_fail",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "value",
            "expected": "chimera_monkey_shifts_toward_chimera_human_identity"
          },
          {
            "name": "chimera_human_magnitude_wrong_attribution_signature",
            "role": "hard_fail",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "value",
            "op": "in",
            "expected": [
              "dramatic_complete_replacement_of_origin_identity",
              "magnitude_dominated_by_culture_or_metabolic_axis",
              "magnitude_dominated_by_developmental_stage_axis",
              "magnitude_dominated_by_cell_cycle_axis",
              "magnitude_dominated_by_batch_effect_axis"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-8-pi",
      "modelName": "claude-opus-4-8",
      "harness": "pi",
      "task": "gse155381_q8_bidirectional_convergence",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 3,
      "passed": true,
      "score": null,
      "durationS": 1881,
      "cost": 4.359586500000001,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "chimera_human_identity_shift_target_weighted",
            "role": "additive",
            "field": "$.chimera_human_identity_shift_target",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_in_vivo_state_assignment_weighted",
            "role": "additive",
            "field": "$.chimera_human_in_vivo_state_assignment",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_shift_magnitude_weighted",
            "role": "additive",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_monkey_identity_shift_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "both_arms_null_contradicted_by_calibration",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "chimera_human_identity_shift_target": {
                "op": "equals",
                "values": [
                  "chimera_human_maintains_origin_identity_no_measurable_shift"
                ]
              },
              "chimera_monkey_identity_shift": {
                "op": "equals",
                "values": [
                  "chimera_monkey_unchanged_relative_to_control"
                ]
              }
            }
          },
          {
            "name": "chimera_monkey_paper_claim_on_confounded_input",
            "role": "hard_fail",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "value",
            "expected": "chimera_monkey_shifts_toward_chimera_human_identity"
          },
          {
            "name": "chimera_human_magnitude_wrong_attribution_signature",
            "role": "hard_fail",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "value",
            "op": "in",
            "expected": [
              "dramatic_complete_replacement_of_origin_identity",
              "magnitude_dominated_by_culture_or_metabolic_axis",
              "magnitude_dominated_by_developmental_stage_axis",
              "magnitude_dominated_by_cell_cycle_axis",
              "magnitude_dominated_by_batch_effect_axis"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-4-6-pi",
      "modelName": "claude-sonnet-4-6",
      "harness": "pi",
      "task": "gse155381_q8_bidirectional_convergence",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 2112,
      "cost": 2.4417614999999993,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "chimera_human_identity_shift_target_weighted",
            "role": "additive",
            "field": "$.chimera_human_identity_shift_target",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_in_vivo_state_assignment_weighted",
            "role": "additive",
            "field": "$.chimera_human_in_vivo_state_assignment",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_shift_magnitude_weighted",
            "role": "additive",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_monkey_identity_shift_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "both_arms_null_contradicted_by_calibration",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "chimera_human_identity_shift_target": {
                "op": "equals",
                "values": [
                  "chimera_human_maintains_origin_identity_no_measurable_shift"
                ]
              },
              "chimera_monkey_identity_shift": {
                "op": "equals",
                "values": [
                  "chimera_monkey_unchanged_relative_to_control"
                ]
              }
            }
          },
          {
            "name": "chimera_monkey_paper_claim_on_confounded_input",
            "role": "hard_fail",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "value",
            "expected": "chimera_monkey_shifts_toward_chimera_human_identity"
          },
          {
            "name": "chimera_human_magnitude_wrong_attribution_signature",
            "role": "hard_fail",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "value",
            "op": "in",
            "expected": [
              "dramatic_complete_replacement_of_origin_identity",
              "magnitude_dominated_by_culture_or_metabolic_axis",
              "magnitude_dominated_by_developmental_stage_axis",
              "magnitude_dominated_by_cell_cycle_axis",
              "magnitude_dominated_by_batch_effect_axis"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-4-6-pi",
      "modelName": "claude-sonnet-4-6",
      "harness": "pi",
      "task": "gse155381_q8_bidirectional_convergence",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 1654,
      "cost": 2.3158351500000007,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "chimera_human_identity_shift_target_weighted",
            "role": "additive",
            "field": "$.chimera_human_identity_shift_target",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_in_vivo_state_assignment_weighted",
            "role": "additive",
            "field": "$.chimera_human_in_vivo_state_assignment",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_shift_magnitude_weighted",
            "role": "additive",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_monkey_identity_shift_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "both_arms_null_contradicted_by_calibration",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "chimera_human_identity_shift_target": {
                "op": "equals",
                "values": [
                  "chimera_human_maintains_origin_identity_no_measurable_shift"
                ]
              },
              "chimera_monkey_identity_shift": {
                "op": "equals",
                "values": [
                  "chimera_monkey_unchanged_relative_to_control"
                ]
              }
            }
          },
          {
            "name": "chimera_monkey_paper_claim_on_confounded_input",
            "role": "hard_fail",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "value",
            "expected": "chimera_monkey_shifts_toward_chimera_human_identity"
          },
          {
            "name": "chimera_human_magnitude_wrong_attribution_signature",
            "role": "hard_fail",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "value",
            "op": "in",
            "expected": [
              "dramatic_complete_replacement_of_origin_identity",
              "magnitude_dominated_by_culture_or_metabolic_axis",
              "magnitude_dominated_by_developmental_stage_axis",
              "magnitude_dominated_by_cell_cycle_axis",
              "magnitude_dominated_by_batch_effect_axis"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-4-6-pi",
      "modelName": "claude-sonnet-4-6",
      "harness": "pi",
      "task": "gse155381_q8_bidirectional_convergence",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1279,
      "cost": 1.6562851500000002,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "chimera_human_identity_shift_target_weighted",
            "role": "additive",
            "field": "$.chimera_human_identity_shift_target",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_in_vivo_state_assignment_weighted",
            "role": "additive",
            "field": "$.chimera_human_in_vivo_state_assignment",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_shift_magnitude_weighted",
            "role": "additive",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_monkey_identity_shift_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "both_arms_null_contradicted_by_calibration",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "chimera_human_identity_shift_target": {
                "op": "equals",
                "values": [
                  "chimera_human_maintains_origin_identity_no_measurable_shift"
                ]
              },
              "chimera_monkey_identity_shift": {
                "op": "equals",
                "values": [
                  "chimera_monkey_unchanged_relative_to_control"
                ]
              }
            }
          },
          {
            "name": "chimera_monkey_paper_claim_on_confounded_input",
            "role": "hard_fail",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "value",
            "expected": "chimera_monkey_shifts_toward_chimera_human_identity"
          },
          {
            "name": "chimera_human_magnitude_wrong_attribution_signature",
            "role": "hard_fail",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "value",
            "op": "in",
            "expected": [
              "dramatic_complete_replacement_of_origin_identity",
              "magnitude_dominated_by_culture_or_metabolic_axis",
              "magnitude_dominated_by_developmental_stage_axis",
              "magnitude_dominated_by_cell_cycle_axis",
              "magnitude_dominated_by_batch_effect_axis"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gemini-3-1-pro-preview-pi",
      "modelName": "gemini-3.1-pro-preview",
      "harness": "pi",
      "task": "gse155381_q8_bidirectional_convergence",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 384,
      "cost": 0.6622266000000002,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "chimera_human_identity_shift_target_weighted",
            "role": "additive",
            "field": "$.chimera_human_identity_shift_target",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_in_vivo_state_assignment_weighted",
            "role": "additive",
            "field": "$.chimera_human_in_vivo_state_assignment",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_shift_magnitude_weighted",
            "role": "additive",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_monkey_identity_shift_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "both_arms_null_contradicted_by_calibration",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "chimera_human_identity_shift_target": {
                "op": "equals",
                "values": [
                  "chimera_human_maintains_origin_identity_no_measurable_shift"
                ]
              },
              "chimera_monkey_identity_shift": {
                "op": "equals",
                "values": [
                  "chimera_monkey_unchanged_relative_to_control"
                ]
              }
            }
          },
          {
            "name": "chimera_monkey_paper_claim_on_confounded_input",
            "role": "hard_fail",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "value",
            "expected": "chimera_monkey_shifts_toward_chimera_human_identity"
          },
          {
            "name": "chimera_human_magnitude_wrong_attribution_signature",
            "role": "hard_fail",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "value",
            "op": "in",
            "expected": [
              "dramatic_complete_replacement_of_origin_identity",
              "magnitude_dominated_by_culture_or_metabolic_axis",
              "magnitude_dominated_by_developmental_stage_axis",
              "magnitude_dominated_by_cell_cycle_axis",
              "magnitude_dominated_by_batch_effect_axis"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gemini-3-1-pro-preview-pi",
      "modelName": "gemini-3.1-pro-preview",
      "harness": "pi",
      "task": "gse155381_q8_bidirectional_convergence",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 459,
      "cost": 0.6694521999999999,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "chimera_human_identity_shift_target_weighted",
            "role": "additive",
            "field": "$.chimera_human_identity_shift_target",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_in_vivo_state_assignment_weighted",
            "role": "additive",
            "field": "$.chimera_human_in_vivo_state_assignment",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_shift_magnitude_weighted",
            "role": "additive",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_monkey_identity_shift_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "both_arms_null_contradicted_by_calibration",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "chimera_human_identity_shift_target": {
                "op": "equals",
                "values": [
                  "chimera_human_maintains_origin_identity_no_measurable_shift"
                ]
              },
              "chimera_monkey_identity_shift": {
                "op": "equals",
                "values": [
                  "chimera_monkey_unchanged_relative_to_control"
                ]
              }
            }
          },
          {
            "name": "chimera_monkey_paper_claim_on_confounded_input",
            "role": "hard_fail",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "value",
            "expected": "chimera_monkey_shifts_toward_chimera_human_identity"
          },
          {
            "name": "chimera_human_magnitude_wrong_attribution_signature",
            "role": "hard_fail",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "value",
            "op": "in",
            "expected": [
              "dramatic_complete_replacement_of_origin_identity",
              "magnitude_dominated_by_culture_or_metabolic_axis",
              "magnitude_dominated_by_developmental_stage_axis",
              "magnitude_dominated_by_cell_cycle_axis",
              "magnitude_dominated_by_batch_effect_axis"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gemini-3-1-pro-preview-pi",
      "modelName": "gemini-3.1-pro-preview",
      "harness": "pi",
      "task": "gse155381_q8_bidirectional_convergence",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 288,
      "cost": 0.4743518000000001,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "chimera_human_identity_shift_target_weighted",
            "role": "additive",
            "field": "$.chimera_human_identity_shift_target",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_in_vivo_state_assignment_weighted",
            "role": "additive",
            "field": "$.chimera_human_in_vivo_state_assignment",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_shift_magnitude_weighted",
            "role": "additive",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_monkey_identity_shift_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "both_arms_null_contradicted_by_calibration",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "chimera_human_identity_shift_target": {
                "op": "equals",
                "values": [
                  "chimera_human_maintains_origin_identity_no_measurable_shift"
                ]
              },
              "chimera_monkey_identity_shift": {
                "op": "equals",
                "values": [
                  "chimera_monkey_unchanged_relative_to_control"
                ]
              }
            }
          },
          {
            "name": "chimera_monkey_paper_claim_on_confounded_input",
            "role": "hard_fail",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "value",
            "expected": "chimera_monkey_shifts_toward_chimera_human_identity"
          },
          {
            "name": "chimera_human_magnitude_wrong_attribution_signature",
            "role": "hard_fail",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "value",
            "op": "in",
            "expected": [
              "dramatic_complete_replacement_of_origin_identity",
              "magnitude_dominated_by_culture_or_metabolic_axis",
              "magnitude_dominated_by_developmental_stage_axis",
              "magnitude_dominated_by_cell_cycle_axis",
              "magnitude_dominated_by_batch_effect_axis"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gemini-3-5-flash-pi",
      "modelName": "gemini-3.5-flash",
      "harness": "pi",
      "task": "gse155381_q8_bidirectional_convergence",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 486,
      "cost": 1.49653275,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "chimera_human_identity_shift_target_weighted",
            "role": "additive",
            "field": "$.chimera_human_identity_shift_target",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_in_vivo_state_assignment_weighted",
            "role": "additive",
            "field": "$.chimera_human_in_vivo_state_assignment",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_shift_magnitude_weighted",
            "role": "additive",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_monkey_identity_shift_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "both_arms_null_contradicted_by_calibration",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "chimera_human_identity_shift_target": {
                "op": "equals",
                "values": [
                  "chimera_human_maintains_origin_identity_no_measurable_shift"
                ]
              },
              "chimera_monkey_identity_shift": {
                "op": "equals",
                "values": [
                  "chimera_monkey_unchanged_relative_to_control"
                ]
              }
            }
          },
          {
            "name": "chimera_monkey_paper_claim_on_confounded_input",
            "role": "hard_fail",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "value",
            "expected": "chimera_monkey_shifts_toward_chimera_human_identity"
          },
          {
            "name": "chimera_human_magnitude_wrong_attribution_signature",
            "role": "hard_fail",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "value",
            "op": "in",
            "expected": [
              "dramatic_complete_replacement_of_origin_identity",
              "magnitude_dominated_by_culture_or_metabolic_axis",
              "magnitude_dominated_by_developmental_stage_axis",
              "magnitude_dominated_by_cell_cycle_axis",
              "magnitude_dominated_by_batch_effect_axis"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gemini-3-5-flash-pi",
      "modelName": "gemini-3.5-flash",
      "harness": "pi",
      "task": "gse155381_q8_bidirectional_convergence",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 645,
      "cost": 2.3317399500000002,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "chimera_human_identity_shift_target_weighted",
            "role": "additive",
            "field": "$.chimera_human_identity_shift_target",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_in_vivo_state_assignment_weighted",
            "role": "additive",
            "field": "$.chimera_human_in_vivo_state_assignment",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_shift_magnitude_weighted",
            "role": "additive",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_monkey_identity_shift_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "both_arms_null_contradicted_by_calibration",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "chimera_human_identity_shift_target": {
                "op": "equals",
                "values": [
                  "chimera_human_maintains_origin_identity_no_measurable_shift"
                ]
              },
              "chimera_monkey_identity_shift": {
                "op": "equals",
                "values": [
                  "chimera_monkey_unchanged_relative_to_control"
                ]
              }
            }
          },
          {
            "name": "chimera_monkey_paper_claim_on_confounded_input",
            "role": "hard_fail",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "value",
            "expected": "chimera_monkey_shifts_toward_chimera_human_identity"
          },
          {
            "name": "chimera_human_magnitude_wrong_attribution_signature",
            "role": "hard_fail",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "value",
            "op": "in",
            "expected": [
              "dramatic_complete_replacement_of_origin_identity",
              "magnitude_dominated_by_culture_or_metabolic_axis",
              "magnitude_dominated_by_developmental_stage_axis",
              "magnitude_dominated_by_cell_cycle_axis",
              "magnitude_dominated_by_batch_effect_axis"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gemini-3-5-flash-pi",
      "modelName": "gemini-3.5-flash",
      "harness": "pi",
      "task": "gse155381_q8_bidirectional_convergence",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 744,
      "cost": 2.8747132499999997,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "chimera_human_identity_shift_target_weighted",
            "role": "additive",
            "field": "$.chimera_human_identity_shift_target",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_in_vivo_state_assignment_weighted",
            "role": "additive",
            "field": "$.chimera_human_in_vivo_state_assignment",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_shift_magnitude_weighted",
            "role": "additive",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_monkey_identity_shift_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "both_arms_null_contradicted_by_calibration",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "chimera_human_identity_shift_target": {
                "op": "equals",
                "values": [
                  "chimera_human_maintains_origin_identity_no_measurable_shift"
                ]
              },
              "chimera_monkey_identity_shift": {
                "op": "equals",
                "values": [
                  "chimera_monkey_unchanged_relative_to_control"
                ]
              }
            }
          },
          {
            "name": "chimera_monkey_paper_claim_on_confounded_input",
            "role": "hard_fail",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "value",
            "expected": "chimera_monkey_shifts_toward_chimera_human_identity"
          },
          {
            "name": "chimera_human_magnitude_wrong_attribution_signature",
            "role": "hard_fail",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "value",
            "op": "in",
            "expected": [
              "dramatic_complete_replacement_of_origin_identity",
              "magnitude_dominated_by_culture_or_metabolic_axis",
              "magnitude_dominated_by_developmental_stage_axis",
              "magnitude_dominated_by_cell_cycle_axis",
              "magnitude_dominated_by_batch_effect_axis"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-glm-5p2-pi",
      "modelName": "glm-5p2",
      "harness": "pi",
      "task": "gse155381_q8_bidirectional_convergence",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 2522,
      "cost": 3.7791790799999996,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "chimera_human_identity_shift_target_weighted",
            "role": "additive",
            "field": "$.chimera_human_identity_shift_target",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_in_vivo_state_assignment_weighted",
            "role": "additive",
            "field": "$.chimera_human_in_vivo_state_assignment",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_shift_magnitude_weighted",
            "role": "additive",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_monkey_identity_shift_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "both_arms_null_contradicted_by_calibration",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "chimera_human_identity_shift_target": {
                "op": "equals",
                "values": [
                  "chimera_human_maintains_origin_identity_no_measurable_shift"
                ]
              },
              "chimera_monkey_identity_shift": {
                "op": "equals",
                "values": [
                  "chimera_monkey_unchanged_relative_to_control"
                ]
              }
            }
          },
          {
            "name": "chimera_monkey_paper_claim_on_confounded_input",
            "role": "hard_fail",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "value",
            "expected": "chimera_monkey_shifts_toward_chimera_human_identity"
          },
          {
            "name": "chimera_human_magnitude_wrong_attribution_signature",
            "role": "hard_fail",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "value",
            "op": "in",
            "expected": [
              "dramatic_complete_replacement_of_origin_identity",
              "magnitude_dominated_by_culture_or_metabolic_axis",
              "magnitude_dominated_by_developmental_stage_axis",
              "magnitude_dominated_by_cell_cycle_axis",
              "magnitude_dominated_by_batch_effect_axis"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-glm-5p2-pi",
      "modelName": "glm-5p2",
      "harness": "pi",
      "task": "gse155381_q8_bidirectional_convergence",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 2924,
      "cost": 4.179632020000001,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "chimera_human_identity_shift_target_weighted",
            "role": "additive",
            "field": "$.chimera_human_identity_shift_target",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_in_vivo_state_assignment_weighted",
            "role": "additive",
            "field": "$.chimera_human_in_vivo_state_assignment",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_shift_magnitude_weighted",
            "role": "additive",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_monkey_identity_shift_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "both_arms_null_contradicted_by_calibration",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "chimera_human_identity_shift_target": {
                "op": "equals",
                "values": [
                  "chimera_human_maintains_origin_identity_no_measurable_shift"
                ]
              },
              "chimera_monkey_identity_shift": {
                "op": "equals",
                "values": [
                  "chimera_monkey_unchanged_relative_to_control"
                ]
              }
            }
          },
          {
            "name": "chimera_monkey_paper_claim_on_confounded_input",
            "role": "hard_fail",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "value",
            "expected": "chimera_monkey_shifts_toward_chimera_human_identity"
          },
          {
            "name": "chimera_human_magnitude_wrong_attribution_signature",
            "role": "hard_fail",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "value",
            "op": "in",
            "expected": [
              "dramatic_complete_replacement_of_origin_identity",
              "magnitude_dominated_by_culture_or_metabolic_axis",
              "magnitude_dominated_by_developmental_stage_axis",
              "magnitude_dominated_by_cell_cycle_axis",
              "magnitude_dominated_by_batch_effect_axis"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-glm-5p2-pi",
      "modelName": "glm-5p2",
      "harness": "pi",
      "task": "gse155381_q8_bidirectional_convergence",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 3484,
      "cost": 5.326962099999999,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "chimera_human_identity_shift_target_weighted",
            "role": "additive",
            "field": "$.chimera_human_identity_shift_target",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_in_vivo_state_assignment_weighted",
            "role": "additive",
            "field": "$.chimera_human_in_vivo_state_assignment",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_shift_magnitude_weighted",
            "role": "additive",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_monkey_identity_shift_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "both_arms_null_contradicted_by_calibration",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "chimera_human_identity_shift_target": {
                "op": "equals",
                "values": [
                  "chimera_human_maintains_origin_identity_no_measurable_shift"
                ]
              },
              "chimera_monkey_identity_shift": {
                "op": "equals",
                "values": [
                  "chimera_monkey_unchanged_relative_to_control"
                ]
              }
            }
          },
          {
            "name": "chimera_monkey_paper_claim_on_confounded_input",
            "role": "hard_fail",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "value",
            "expected": "chimera_monkey_shifts_toward_chimera_human_identity"
          },
          {
            "name": "chimera_human_magnitude_wrong_attribution_signature",
            "role": "hard_fail",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "value",
            "op": "in",
            "expected": [
              "dramatic_complete_replacement_of_origin_identity",
              "magnitude_dominated_by_culture_or_metabolic_axis",
              "magnitude_dominated_by_developmental_stage_axis",
              "magnitude_dominated_by_cell_cycle_axis",
              "magnitude_dominated_by_batch_effect_axis"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-4-codex",
      "modelName": "gpt-5.4",
      "harness": "openai-codex",
      "task": "gse155381_q8_bidirectional_convergence",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 424,
      "cost": 1.699039,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "chimera_human_identity_shift_target_weighted",
            "role": "additive",
            "field": "$.chimera_human_identity_shift_target",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_in_vivo_state_assignment_weighted",
            "role": "additive",
            "field": "$.chimera_human_in_vivo_state_assignment",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_shift_magnitude_weighted",
            "role": "additive",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_monkey_identity_shift_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "both_arms_null_contradicted_by_calibration",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "chimera_human_identity_shift_target": {
                "op": "equals",
                "values": [
                  "chimera_human_maintains_origin_identity_no_measurable_shift"
                ]
              },
              "chimera_monkey_identity_shift": {
                "op": "equals",
                "values": [
                  "chimera_monkey_unchanged_relative_to_control"
                ]
              }
            }
          },
          {
            "name": "chimera_monkey_paper_claim_on_confounded_input",
            "role": "hard_fail",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "value",
            "expected": "chimera_monkey_shifts_toward_chimera_human_identity"
          },
          {
            "name": "chimera_human_magnitude_wrong_attribution_signature",
            "role": "hard_fail",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "value",
            "op": "in",
            "expected": [
              "dramatic_complete_replacement_of_origin_identity",
              "magnitude_dominated_by_culture_or_metabolic_axis",
              "magnitude_dominated_by_developmental_stage_axis",
              "magnitude_dominated_by_cell_cycle_axis",
              "magnitude_dominated_by_batch_effect_axis"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-4-codex",
      "modelName": "gpt-5.4",
      "harness": "openai-codex",
      "task": "gse155381_q8_bidirectional_convergence",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 830,
      "cost": 3.4781895,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "chimera_human_identity_shift_target_weighted",
            "role": "additive",
            "field": "$.chimera_human_identity_shift_target",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_in_vivo_state_assignment_weighted",
            "role": "additive",
            "field": "$.chimera_human_in_vivo_state_assignment",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_shift_magnitude_weighted",
            "role": "additive",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_monkey_identity_shift_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "both_arms_null_contradicted_by_calibration",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "chimera_human_identity_shift_target": {
                "op": "equals",
                "values": [
                  "chimera_human_maintains_origin_identity_no_measurable_shift"
                ]
              },
              "chimera_monkey_identity_shift": {
                "op": "equals",
                "values": [
                  "chimera_monkey_unchanged_relative_to_control"
                ]
              }
            }
          },
          {
            "name": "chimera_monkey_paper_claim_on_confounded_input",
            "role": "hard_fail",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "value",
            "expected": "chimera_monkey_shifts_toward_chimera_human_identity"
          },
          {
            "name": "chimera_human_magnitude_wrong_attribution_signature",
            "role": "hard_fail",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "value",
            "op": "in",
            "expected": [
              "dramatic_complete_replacement_of_origin_identity",
              "magnitude_dominated_by_culture_or_metabolic_axis",
              "magnitude_dominated_by_developmental_stage_axis",
              "magnitude_dominated_by_cell_cycle_axis",
              "magnitude_dominated_by_batch_effect_axis"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-4-codex",
      "modelName": "gpt-5.4",
      "harness": "openai-codex",
      "task": "gse155381_q8_bidirectional_convergence",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 384,
      "cost": 0.6622266000000002,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "chimera_human_identity_shift_target_weighted",
            "role": "additive",
            "field": "$.chimera_human_identity_shift_target",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_in_vivo_state_assignment_weighted",
            "role": "additive",
            "field": "$.chimera_human_in_vivo_state_assignment",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_shift_magnitude_weighted",
            "role": "additive",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_monkey_identity_shift_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "both_arms_null_contradicted_by_calibration",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "chimera_human_identity_shift_target": {
                "op": "equals",
                "values": [
                  "chimera_human_maintains_origin_identity_no_measurable_shift"
                ]
              },
              "chimera_monkey_identity_shift": {
                "op": "equals",
                "values": [
                  "chimera_monkey_unchanged_relative_to_control"
                ]
              }
            }
          },
          {
            "name": "chimera_monkey_paper_claim_on_confounded_input",
            "role": "hard_fail",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "value",
            "expected": "chimera_monkey_shifts_toward_chimera_human_identity"
          },
          {
            "name": "chimera_human_magnitude_wrong_attribution_signature",
            "role": "hard_fail",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "value",
            "op": "in",
            "expected": [
              "dramatic_complete_replacement_of_origin_identity",
              "magnitude_dominated_by_culture_or_metabolic_axis",
              "magnitude_dominated_by_developmental_stage_axis",
              "magnitude_dominated_by_cell_cycle_axis",
              "magnitude_dominated_by_batch_effect_axis"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-4-pi",
      "modelName": "gpt-5.4",
      "harness": "pi",
      "task": "gse155381_q8_bidirectional_convergence",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 1235,
      "cost": 2.696919,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "chimera_human_identity_shift_target_weighted",
            "role": "additive",
            "field": "$.chimera_human_identity_shift_target",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_in_vivo_state_assignment_weighted",
            "role": "additive",
            "field": "$.chimera_human_in_vivo_state_assignment",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_shift_magnitude_weighted",
            "role": "additive",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_monkey_identity_shift_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "both_arms_null_contradicted_by_calibration",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "chimera_human_identity_shift_target": {
                "op": "equals",
                "values": [
                  "chimera_human_maintains_origin_identity_no_measurable_shift"
                ]
              },
              "chimera_monkey_identity_shift": {
                "op": "equals",
                "values": [
                  "chimera_monkey_unchanged_relative_to_control"
                ]
              }
            }
          },
          {
            "name": "chimera_monkey_paper_claim_on_confounded_input",
            "role": "hard_fail",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "value",
            "expected": "chimera_monkey_shifts_toward_chimera_human_identity"
          },
          {
            "name": "chimera_human_magnitude_wrong_attribution_signature",
            "role": "hard_fail",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "value",
            "op": "in",
            "expected": [
              "dramatic_complete_replacement_of_origin_identity",
              "magnitude_dominated_by_culture_or_metabolic_axis",
              "magnitude_dominated_by_developmental_stage_axis",
              "magnitude_dominated_by_cell_cycle_axis",
              "magnitude_dominated_by_batch_effect_axis"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-4-pi",
      "modelName": "gpt-5.4",
      "harness": "pi",
      "task": "gse155381_q8_bidirectional_convergence",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 749,
      "cost": 1.8071295,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "chimera_human_identity_shift_target_weighted",
            "role": "additive",
            "field": "$.chimera_human_identity_shift_target",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_in_vivo_state_assignment_weighted",
            "role": "additive",
            "field": "$.chimera_human_in_vivo_state_assignment",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_shift_magnitude_weighted",
            "role": "additive",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_monkey_identity_shift_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "both_arms_null_contradicted_by_calibration",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "chimera_human_identity_shift_target": {
                "op": "equals",
                "values": [
                  "chimera_human_maintains_origin_identity_no_measurable_shift"
                ]
              },
              "chimera_monkey_identity_shift": {
                "op": "equals",
                "values": [
                  "chimera_monkey_unchanged_relative_to_control"
                ]
              }
            }
          },
          {
            "name": "chimera_monkey_paper_claim_on_confounded_input",
            "role": "hard_fail",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "value",
            "expected": "chimera_monkey_shifts_toward_chimera_human_identity"
          },
          {
            "name": "chimera_human_magnitude_wrong_attribution_signature",
            "role": "hard_fail",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "value",
            "op": "in",
            "expected": [
              "dramatic_complete_replacement_of_origin_identity",
              "magnitude_dominated_by_culture_or_metabolic_axis",
              "magnitude_dominated_by_developmental_stage_axis",
              "magnitude_dominated_by_cell_cycle_axis",
              "magnitude_dominated_by_batch_effect_axis"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-4-pi",
      "modelName": "gpt-5.4",
      "harness": "pi",
      "task": "gse155381_q8_bidirectional_convergence",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1106,
      "cost": 1.732003,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "chimera_human_identity_shift_target_weighted",
            "role": "additive",
            "field": "$.chimera_human_identity_shift_target",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_in_vivo_state_assignment_weighted",
            "role": "additive",
            "field": "$.chimera_human_in_vivo_state_assignment",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_shift_magnitude_weighted",
            "role": "additive",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_monkey_identity_shift_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "both_arms_null_contradicted_by_calibration",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "chimera_human_identity_shift_target": {
                "op": "equals",
                "values": [
                  "chimera_human_maintains_origin_identity_no_measurable_shift"
                ]
              },
              "chimera_monkey_identity_shift": {
                "op": "equals",
                "values": [
                  "chimera_monkey_unchanged_relative_to_control"
                ]
              }
            }
          },
          {
            "name": "chimera_monkey_paper_claim_on_confounded_input",
            "role": "hard_fail",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "value",
            "expected": "chimera_monkey_shifts_toward_chimera_human_identity"
          },
          {
            "name": "chimera_human_magnitude_wrong_attribution_signature",
            "role": "hard_fail",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "value",
            "op": "in",
            "expected": [
              "dramatic_complete_replacement_of_origin_identity",
              "magnitude_dominated_by_culture_or_metabolic_axis",
              "magnitude_dominated_by_developmental_stage_axis",
              "magnitude_dominated_by_cell_cycle_axis",
              "magnitude_dominated_by_batch_effect_axis"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-5-codex",
      "modelName": "gpt-5.5",
      "harness": "openai-codex",
      "task": "gse155381_q8_bidirectional_convergence",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 1165,
      "cost": 2.466416,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "chimera_human_identity_shift_target_weighted",
            "role": "additive",
            "field": "$.chimera_human_identity_shift_target",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_in_vivo_state_assignment_weighted",
            "role": "additive",
            "field": "$.chimera_human_in_vivo_state_assignment",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_shift_magnitude_weighted",
            "role": "additive",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_monkey_identity_shift_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "both_arms_null_contradicted_by_calibration",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "chimera_human_identity_shift_target": {
                "op": "equals",
                "values": [
                  "chimera_human_maintains_origin_identity_no_measurable_shift"
                ]
              },
              "chimera_monkey_identity_shift": {
                "op": "equals",
                "values": [
                  "chimera_monkey_unchanged_relative_to_control"
                ]
              }
            }
          },
          {
            "name": "chimera_monkey_paper_claim_on_confounded_input",
            "role": "hard_fail",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "value",
            "expected": "chimera_monkey_shifts_toward_chimera_human_identity"
          },
          {
            "name": "chimera_human_magnitude_wrong_attribution_signature",
            "role": "hard_fail",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "value",
            "op": "in",
            "expected": [
              "dramatic_complete_replacement_of_origin_identity",
              "magnitude_dominated_by_culture_or_metabolic_axis",
              "magnitude_dominated_by_developmental_stage_axis",
              "magnitude_dominated_by_cell_cycle_axis",
              "magnitude_dominated_by_batch_effect_axis"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-5-codex",
      "modelName": "gpt-5.5",
      "harness": "openai-codex",
      "task": "gse155381_q8_bidirectional_convergence",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 694,
      "cost": 2.742553,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "chimera_human_identity_shift_target_weighted",
            "role": "additive",
            "field": "$.chimera_human_identity_shift_target",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_in_vivo_state_assignment_weighted",
            "role": "additive",
            "field": "$.chimera_human_in_vivo_state_assignment",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_shift_magnitude_weighted",
            "role": "additive",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_monkey_identity_shift_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "both_arms_null_contradicted_by_calibration",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "chimera_human_identity_shift_target": {
                "op": "equals",
                "values": [
                  "chimera_human_maintains_origin_identity_no_measurable_shift"
                ]
              },
              "chimera_monkey_identity_shift": {
                "op": "equals",
                "values": [
                  "chimera_monkey_unchanged_relative_to_control"
                ]
              }
            }
          },
          {
            "name": "chimera_monkey_paper_claim_on_confounded_input",
            "role": "hard_fail",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "value",
            "expected": "chimera_monkey_shifts_toward_chimera_human_identity"
          },
          {
            "name": "chimera_human_magnitude_wrong_attribution_signature",
            "role": "hard_fail",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "value",
            "op": "in",
            "expected": [
              "dramatic_complete_replacement_of_origin_identity",
              "magnitude_dominated_by_culture_or_metabolic_axis",
              "magnitude_dominated_by_developmental_stage_axis",
              "magnitude_dominated_by_cell_cycle_axis",
              "magnitude_dominated_by_batch_effect_axis"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-5-codex",
      "modelName": "gpt-5.5",
      "harness": "openai-codex",
      "task": "gse155381_q8_bidirectional_convergence",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 777,
      "cost": 1.848326,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "chimera_human_identity_shift_target_weighted",
            "role": "additive",
            "field": "$.chimera_human_identity_shift_target",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_in_vivo_state_assignment_weighted",
            "role": "additive",
            "field": "$.chimera_human_in_vivo_state_assignment",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_shift_magnitude_weighted",
            "role": "additive",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_monkey_identity_shift_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "both_arms_null_contradicted_by_calibration",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "chimera_human_identity_shift_target": {
                "op": "equals",
                "values": [
                  "chimera_human_maintains_origin_identity_no_measurable_shift"
                ]
              },
              "chimera_monkey_identity_shift": {
                "op": "equals",
                "values": [
                  "chimera_monkey_unchanged_relative_to_control"
                ]
              }
            }
          },
          {
            "name": "chimera_monkey_paper_claim_on_confounded_input",
            "role": "hard_fail",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "value",
            "expected": "chimera_monkey_shifts_toward_chimera_human_identity"
          },
          {
            "name": "chimera_human_magnitude_wrong_attribution_signature",
            "role": "hard_fail",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "value",
            "op": "in",
            "expected": [
              "dramatic_complete_replacement_of_origin_identity",
              "magnitude_dominated_by_culture_or_metabolic_axis",
              "magnitude_dominated_by_developmental_stage_axis",
              "magnitude_dominated_by_cell_cycle_axis",
              "magnitude_dominated_by_batch_effect_axis"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-5-pi",
      "modelName": "gpt-5.5",
      "harness": "pi",
      "task": "gse155381_q8_bidirectional_convergence",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 155,
      "cost": 0.6026520000000001,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "chimera_human_identity_shift_target_weighted",
            "role": "additive",
            "field": "$.chimera_human_identity_shift_target",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_in_vivo_state_assignment_weighted",
            "role": "additive",
            "field": "$.chimera_human_in_vivo_state_assignment",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_shift_magnitude_weighted",
            "role": "additive",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_monkey_identity_shift_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "both_arms_null_contradicted_by_calibration",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "chimera_human_identity_shift_target": {
                "op": "equals",
                "values": [
                  "chimera_human_maintains_origin_identity_no_measurable_shift"
                ]
              },
              "chimera_monkey_identity_shift": {
                "op": "equals",
                "values": [
                  "chimera_monkey_unchanged_relative_to_control"
                ]
              }
            }
          },
          {
            "name": "chimera_monkey_paper_claim_on_confounded_input",
            "role": "hard_fail",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "value",
            "expected": "chimera_monkey_shifts_toward_chimera_human_identity"
          },
          {
            "name": "chimera_human_magnitude_wrong_attribution_signature",
            "role": "hard_fail",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "value",
            "op": "in",
            "expected": [
              "dramatic_complete_replacement_of_origin_identity",
              "magnitude_dominated_by_culture_or_metabolic_axis",
              "magnitude_dominated_by_developmental_stage_axis",
              "magnitude_dominated_by_cell_cycle_axis",
              "magnitude_dominated_by_batch_effect_axis"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-5-pi",
      "modelName": "gpt-5.5",
      "harness": "pi",
      "task": "gse155381_q8_bidirectional_convergence",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 990,
      "cost": 4.3162650000000005,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "chimera_human_identity_shift_target_weighted",
            "role": "additive",
            "field": "$.chimera_human_identity_shift_target",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_in_vivo_state_assignment_weighted",
            "role": "additive",
            "field": "$.chimera_human_in_vivo_state_assignment",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_shift_magnitude_weighted",
            "role": "additive",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_monkey_identity_shift_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "both_arms_null_contradicted_by_calibration",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "chimera_human_identity_shift_target": {
                "op": "equals",
                "values": [
                  "chimera_human_maintains_origin_identity_no_measurable_shift"
                ]
              },
              "chimera_monkey_identity_shift": {
                "op": "equals",
                "values": [
                  "chimera_monkey_unchanged_relative_to_control"
                ]
              }
            }
          },
          {
            "name": "chimera_monkey_paper_claim_on_confounded_input",
            "role": "hard_fail",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "value",
            "expected": "chimera_monkey_shifts_toward_chimera_human_identity"
          },
          {
            "name": "chimera_human_magnitude_wrong_attribution_signature",
            "role": "hard_fail",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "value",
            "op": "in",
            "expected": [
              "dramatic_complete_replacement_of_origin_identity",
              "magnitude_dominated_by_culture_or_metabolic_axis",
              "magnitude_dominated_by_developmental_stage_axis",
              "magnitude_dominated_by_cell_cycle_axis",
              "magnitude_dominated_by_batch_effect_axis"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-5-pi",
      "modelName": "gpt-5.5",
      "harness": "pi",
      "task": "gse155381_q8_bidirectional_convergence",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1084,
      "cost": 4.705443999999999,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "chimera_human_identity_shift_target_weighted",
            "role": "additive",
            "field": "$.chimera_human_identity_shift_target",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_in_vivo_state_assignment_weighted",
            "role": "additive",
            "field": "$.chimera_human_in_vivo_state_assignment",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_shift_magnitude_weighted",
            "role": "additive",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_monkey_identity_shift_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "both_arms_null_contradicted_by_calibration",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "chimera_human_identity_shift_target": {
                "op": "equals",
                "values": [
                  "chimera_human_maintains_origin_identity_no_measurable_shift"
                ]
              },
              "chimera_monkey_identity_shift": {
                "op": "equals",
                "values": [
                  "chimera_monkey_unchanged_relative_to_control"
                ]
              }
            }
          },
          {
            "name": "chimera_monkey_paper_claim_on_confounded_input",
            "role": "hard_fail",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "value",
            "expected": "chimera_monkey_shifts_toward_chimera_human_identity"
          },
          {
            "name": "chimera_human_magnitude_wrong_attribution_signature",
            "role": "hard_fail",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "value",
            "op": "in",
            "expected": [
              "dramatic_complete_replacement_of_origin_identity",
              "magnitude_dominated_by_culture_or_metabolic_axis",
              "magnitude_dominated_by_developmental_stage_axis",
              "magnitude_dominated_by_cell_cycle_axis",
              "magnitude_dominated_by_batch_effect_axis"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-20-beta-reasoning-pi",
      "modelName": "grok-4.20-beta-0309-reasoning",
      "harness": "pi",
      "task": "gse155381_q8_bidirectional_convergence",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 135,
      "cost": 0.06529900000000002,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "chimera_human_identity_shift_target_weighted",
            "role": "additive",
            "field": "$.chimera_human_identity_shift_target",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_in_vivo_state_assignment_weighted",
            "role": "additive",
            "field": "$.chimera_human_in_vivo_state_assignment",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_shift_magnitude_weighted",
            "role": "additive",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_monkey_identity_shift_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "both_arms_null_contradicted_by_calibration",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "chimera_human_identity_shift_target": {
                "op": "equals",
                "values": [
                  "chimera_human_maintains_origin_identity_no_measurable_shift"
                ]
              },
              "chimera_monkey_identity_shift": {
                "op": "equals",
                "values": [
                  "chimera_monkey_unchanged_relative_to_control"
                ]
              }
            }
          },
          {
            "name": "chimera_monkey_paper_claim_on_confounded_input",
            "role": "hard_fail",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "value",
            "expected": "chimera_monkey_shifts_toward_chimera_human_identity"
          },
          {
            "name": "chimera_human_magnitude_wrong_attribution_signature",
            "role": "hard_fail",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "value",
            "op": "in",
            "expected": [
              "dramatic_complete_replacement_of_origin_identity",
              "magnitude_dominated_by_culture_or_metabolic_axis",
              "magnitude_dominated_by_developmental_stage_axis",
              "magnitude_dominated_by_cell_cycle_axis",
              "magnitude_dominated_by_batch_effect_axis"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-20-beta-reasoning-pi",
      "modelName": "grok-4.20-beta-0309-reasoning",
      "harness": "pi",
      "task": "gse155381_q8_bidirectional_convergence",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 148,
      "cost": 0.07072605000000001,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "chimera_human_identity_shift_target_weighted",
            "role": "additive",
            "field": "$.chimera_human_identity_shift_target",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_in_vivo_state_assignment_weighted",
            "role": "additive",
            "field": "$.chimera_human_in_vivo_state_assignment",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_shift_magnitude_weighted",
            "role": "additive",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_monkey_identity_shift_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "both_arms_null_contradicted_by_calibration",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "chimera_human_identity_shift_target": {
                "op": "equals",
                "values": [
                  "chimera_human_maintains_origin_identity_no_measurable_shift"
                ]
              },
              "chimera_monkey_identity_shift": {
                "op": "equals",
                "values": [
                  "chimera_monkey_unchanged_relative_to_control"
                ]
              }
            }
          },
          {
            "name": "chimera_monkey_paper_claim_on_confounded_input",
            "role": "hard_fail",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "value",
            "expected": "chimera_monkey_shifts_toward_chimera_human_identity"
          },
          {
            "name": "chimera_human_magnitude_wrong_attribution_signature",
            "role": "hard_fail",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "value",
            "op": "in",
            "expected": [
              "dramatic_complete_replacement_of_origin_identity",
              "magnitude_dominated_by_culture_or_metabolic_axis",
              "magnitude_dominated_by_developmental_stage_axis",
              "magnitude_dominated_by_cell_cycle_axis",
              "magnitude_dominated_by_batch_effect_axis"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-20-beta-reasoning-pi",
      "modelName": "grok-4.20-beta-0309-reasoning",
      "harness": "pi",
      "task": "gse155381_q8_bidirectional_convergence",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 172,
      "cost": 0.04186695000000001,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "chimera_human_identity_shift_target_weighted",
            "role": "additive",
            "field": "$.chimera_human_identity_shift_target",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_in_vivo_state_assignment_weighted",
            "role": "additive",
            "field": "$.chimera_human_in_vivo_state_assignment",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_shift_magnitude_weighted",
            "role": "additive",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_monkey_identity_shift_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "both_arms_null_contradicted_by_calibration",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "chimera_human_identity_shift_target": {
                "op": "equals",
                "values": [
                  "chimera_human_maintains_origin_identity_no_measurable_shift"
                ]
              },
              "chimera_monkey_identity_shift": {
                "op": "equals",
                "values": [
                  "chimera_monkey_unchanged_relative_to_control"
                ]
              }
            }
          },
          {
            "name": "chimera_monkey_paper_claim_on_confounded_input",
            "role": "hard_fail",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "value",
            "expected": "chimera_monkey_shifts_toward_chimera_human_identity"
          },
          {
            "name": "chimera_human_magnitude_wrong_attribution_signature",
            "role": "hard_fail",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "value",
            "op": "in",
            "expected": [
              "dramatic_complete_replacement_of_origin_identity",
              "magnitude_dominated_by_culture_or_metabolic_axis",
              "magnitude_dominated_by_developmental_stage_axis",
              "magnitude_dominated_by_cell_cycle_axis",
              "magnitude_dominated_by_batch_effect_axis"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-3-pi",
      "modelName": "grok-4.3",
      "harness": "pi",
      "task": "gse155381_q8_bidirectional_convergence",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 23,
      "cost": 0.0069094,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "chimera_human_identity_shift_target_weighted",
            "role": "additive",
            "field": "$.chimera_human_identity_shift_target",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_in_vivo_state_assignment_weighted",
            "role": "additive",
            "field": "$.chimera_human_in_vivo_state_assignment",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_shift_magnitude_weighted",
            "role": "additive",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_monkey_identity_shift_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "both_arms_null_contradicted_by_calibration",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "chimera_human_identity_shift_target": {
                "op": "equals",
                "values": [
                  "chimera_human_maintains_origin_identity_no_measurable_shift"
                ]
              },
              "chimera_monkey_identity_shift": {
                "op": "equals",
                "values": [
                  "chimera_monkey_unchanged_relative_to_control"
                ]
              }
            }
          },
          {
            "name": "chimera_monkey_paper_claim_on_confounded_input",
            "role": "hard_fail",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "value",
            "expected": "chimera_monkey_shifts_toward_chimera_human_identity"
          },
          {
            "name": "chimera_human_magnitude_wrong_attribution_signature",
            "role": "hard_fail",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "value",
            "op": "in",
            "expected": [
              "dramatic_complete_replacement_of_origin_identity",
              "magnitude_dominated_by_culture_or_metabolic_axis",
              "magnitude_dominated_by_developmental_stage_axis",
              "magnitude_dominated_by_cell_cycle_axis",
              "magnitude_dominated_by_batch_effect_axis"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-3-pi",
      "modelName": "grok-4.3",
      "harness": "pi",
      "task": "gse155381_q8_bidirectional_convergence",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 16,
      "cost": 0.00635055,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "chimera_human_identity_shift_target_weighted",
            "role": "additive",
            "field": "$.chimera_human_identity_shift_target",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_in_vivo_state_assignment_weighted",
            "role": "additive",
            "field": "$.chimera_human_in_vivo_state_assignment",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_shift_magnitude_weighted",
            "role": "additive",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_monkey_identity_shift_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "both_arms_null_contradicted_by_calibration",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "chimera_human_identity_shift_target": {
                "op": "equals",
                "values": [
                  "chimera_human_maintains_origin_identity_no_measurable_shift"
                ]
              },
              "chimera_monkey_identity_shift": {
                "op": "equals",
                "values": [
                  "chimera_monkey_unchanged_relative_to_control"
                ]
              }
            }
          },
          {
            "name": "chimera_monkey_paper_claim_on_confounded_input",
            "role": "hard_fail",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "value",
            "expected": "chimera_monkey_shifts_toward_chimera_human_identity"
          },
          {
            "name": "chimera_human_magnitude_wrong_attribution_signature",
            "role": "hard_fail",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "value",
            "op": "in",
            "expected": [
              "dramatic_complete_replacement_of_origin_identity",
              "magnitude_dominated_by_culture_or_metabolic_axis",
              "magnitude_dominated_by_developmental_stage_axis",
              "magnitude_dominated_by_cell_cycle_axis",
              "magnitude_dominated_by_batch_effect_axis"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-3-pi",
      "modelName": "grok-4.3",
      "harness": "pi",
      "task": "gse155381_q8_bidirectional_convergence",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 57,
      "cost": 0.012308600000000001,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "chimera_human_identity_shift_target_weighted",
            "role": "additive",
            "field": "$.chimera_human_identity_shift_target",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_in_vivo_state_assignment_weighted",
            "role": "additive",
            "field": "$.chimera_human_in_vivo_state_assignment",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_shift_magnitude_weighted",
            "role": "additive",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_monkey_identity_shift_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "both_arms_null_contradicted_by_calibration",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "chimera_human_identity_shift_target": {
                "op": "equals",
                "values": [
                  "chimera_human_maintains_origin_identity_no_measurable_shift"
                ]
              },
              "chimera_monkey_identity_shift": {
                "op": "equals",
                "values": [
                  "chimera_monkey_unchanged_relative_to_control"
                ]
              }
            }
          },
          {
            "name": "chimera_monkey_paper_claim_on_confounded_input",
            "role": "hard_fail",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "value",
            "expected": "chimera_monkey_shifts_toward_chimera_human_identity"
          },
          {
            "name": "chimera_human_magnitude_wrong_attribution_signature",
            "role": "hard_fail",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "value",
            "op": "in",
            "expected": [
              "dramatic_complete_replacement_of_origin_identity",
              "magnitude_dominated_by_culture_or_metabolic_axis",
              "magnitude_dominated_by_developmental_stage_axis",
              "magnitude_dominated_by_cell_cycle_axis",
              "magnitude_dominated_by_batch_effect_axis"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-kimi-k2p6-pi",
      "modelName": "kimi-k2p6",
      "harness": "pi",
      "task": "gse155381_q8_bidirectional_convergence",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 1,
      "passed": true,
      "score": null,
      "durationS": 1826,
      "cost": 0.47328527,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "chimera_human_identity_shift_target_weighted",
            "role": "additive",
            "field": "$.chimera_human_identity_shift_target",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_in_vivo_state_assignment_weighted",
            "role": "additive",
            "field": "$.chimera_human_in_vivo_state_assignment",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_shift_magnitude_weighted",
            "role": "additive",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_monkey_identity_shift_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "both_arms_null_contradicted_by_calibration",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "chimera_human_identity_shift_target": {
                "op": "equals",
                "values": [
                  "chimera_human_maintains_origin_identity_no_measurable_shift"
                ]
              },
              "chimera_monkey_identity_shift": {
                "op": "equals",
                "values": [
                  "chimera_monkey_unchanged_relative_to_control"
                ]
              }
            }
          },
          {
            "name": "chimera_monkey_paper_claim_on_confounded_input",
            "role": "hard_fail",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "value",
            "expected": "chimera_monkey_shifts_toward_chimera_human_identity"
          },
          {
            "name": "chimera_human_magnitude_wrong_attribution_signature",
            "role": "hard_fail",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "value",
            "op": "in",
            "expected": [
              "dramatic_complete_replacement_of_origin_identity",
              "magnitude_dominated_by_culture_or_metabolic_axis",
              "magnitude_dominated_by_developmental_stage_axis",
              "magnitude_dominated_by_cell_cycle_axis",
              "magnitude_dominated_by_batch_effect_axis"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-kimi-k2p6-pi",
      "modelName": "kimi-k2p6",
      "harness": "pi",
      "task": "gse155381_q8_bidirectional_convergence",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 3454,
      "cost": 1.3030025999999997,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "chimera_human_identity_shift_target_weighted",
            "role": "additive",
            "field": "$.chimera_human_identity_shift_target",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_in_vivo_state_assignment_weighted",
            "role": "additive",
            "field": "$.chimera_human_in_vivo_state_assignment",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_shift_magnitude_weighted",
            "role": "additive",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_monkey_identity_shift_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "both_arms_null_contradicted_by_calibration",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "chimera_human_identity_shift_target": {
                "op": "equals",
                "values": [
                  "chimera_human_maintains_origin_identity_no_measurable_shift"
                ]
              },
              "chimera_monkey_identity_shift": {
                "op": "equals",
                "values": [
                  "chimera_monkey_unchanged_relative_to_control"
                ]
              }
            }
          },
          {
            "name": "chimera_monkey_paper_claim_on_confounded_input",
            "role": "hard_fail",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "value",
            "expected": "chimera_monkey_shifts_toward_chimera_human_identity"
          },
          {
            "name": "chimera_human_magnitude_wrong_attribution_signature",
            "role": "hard_fail",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "value",
            "op": "in",
            "expected": [
              "dramatic_complete_replacement_of_origin_identity",
              "magnitude_dominated_by_culture_or_metabolic_axis",
              "magnitude_dominated_by_developmental_stage_axis",
              "magnitude_dominated_by_cell_cycle_axis",
              "magnitude_dominated_by_batch_effect_axis"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-kimi-k2p6-pi",
      "modelName": "kimi-k2p6",
      "harness": "pi",
      "task": "gse155381_q8_bidirectional_convergence",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 2791,
      "cost": 1.52509572,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "chimera_human_identity_shift_target_weighted",
            "role": "additive",
            "field": "$.chimera_human_identity_shift_target",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_in_vivo_state_assignment_weighted",
            "role": "additive",
            "field": "$.chimera_human_in_vivo_state_assignment",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_shift_magnitude_weighted",
            "role": "additive",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_monkey_identity_shift_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "both_arms_null_contradicted_by_calibration",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "chimera_human_identity_shift_target": {
                "op": "equals",
                "values": [
                  "chimera_human_maintains_origin_identity_no_measurable_shift"
                ]
              },
              "chimera_monkey_identity_shift": {
                "op": "equals",
                "values": [
                  "chimera_monkey_unchanged_relative_to_control"
                ]
              }
            }
          },
          {
            "name": "chimera_monkey_paper_claim_on_confounded_input",
            "role": "hard_fail",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "value",
            "expected": "chimera_monkey_shifts_toward_chimera_human_identity"
          },
          {
            "name": "chimera_human_magnitude_wrong_attribution_signature",
            "role": "hard_fail",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "value",
            "op": "in",
            "expected": [
              "dramatic_complete_replacement_of_origin_identity",
              "magnitude_dominated_by_culture_or_metabolic_axis",
              "magnitude_dominated_by_developmental_stage_axis",
              "magnitude_dominated_by_cell_cycle_axis",
              "magnitude_dominated_by_batch_effect_axis"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-6-claude-code",
      "modelName": "claude-opus-4-6",
      "harness": "claude-code",
      "task": "gse171524_ectopic_airway_rare_cell",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 3230,
      "cost": 3.4379387500000003,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "focus_population_identity_gate",
            "role": "gate",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "chemosensory_marker_expressing_population_alpha"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "focus_population_marker_calls",
            "kind": "list",
            "matchKey": "marker",
            "k": 3,
            "expected": [
              {
                "key": "POU2F3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "CHAT",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "TRPM5",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "LRMP",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNAT3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "AVIL",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNG13",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              }
            ]
          },
          {
            "name": "focus_population_condition_distribution_gate",
            "role": "gate",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_enriched_with_substantial_control_presence",
              "covid_predominant",
              "covid_only",
              "covid_enriched_trend_not_significant"
            ]
          },
          {
            "name": "focus_population_location_gate",
            "role": "gate",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "ectopic_parenchyma_in_covid",
              "parenchyma_only",
              "bronchiolized_or_peribronchial_parenchyma",
              "airway_and_parenchyma"
            ]
          },
          {
            "name": "focus_population_recovery_mode_gate",
            "role": "gate",
            "field": "$.focus_population_recovery_mode",
            "kind": "value",
            "op": "in",
            "expected": [
              "continuum_not_discrete_cluster"
            ]
          },
          {
            "name": "focus_population_identity_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "ionocyte_like_beta",
              "neuroendocrine_like_gamma",
              "basal_stem_progenitor_like_delta",
              "aberrant_basaloid_transitional_like_epsilon",
              "neuronal_or_neural_gene_minor_population_zeta",
              "deuterosomal_ciliated_progenitor_like_eta",
              "hillock_transitional_like_theta",
              "low_quality_doublet_or_debris",
              "no_rare_cell_recovered",
              "not_assessed"
            ]
          },
          {
            "name": "cf_wrong_lineage_marker_called_positive",
            "role": "hard_fail",
            "field": "$.focus_population_marker_calls[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "focus_population_condition_distribution_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "equally_distributed",
              "control_predominant",
              "not_assessed"
            ]
          },
          {
            "name": "focus_population_location_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "airway_only",
              "not_assessed"
            ]
          },
          {
            "name": "cross_axis_canonical_identity_with_airway_only_location_incoherence",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "focus_population_identity": {
                "op": "in",
                "values": [
                  "chemosensory_marker_expressing_population_alpha"
                ]
              },
              "focus_population_location": {
                "op": "in",
                "values": [
                  "airway_only"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-6-claude-code",
      "modelName": "claude-opus-4-6",
      "harness": "claude-code",
      "task": "gse171524_ectopic_airway_rare_cell",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 1714,
      "cost": 2.3211802500000007,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "focus_population_identity_gate",
            "role": "gate",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "chemosensory_marker_expressing_population_alpha"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "focus_population_marker_calls",
            "kind": "list",
            "matchKey": "marker",
            "k": 3,
            "expected": [
              {
                "key": "POU2F3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "CHAT",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "TRPM5",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "LRMP",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNAT3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "AVIL",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNG13",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              }
            ]
          },
          {
            "name": "focus_population_condition_distribution_gate",
            "role": "gate",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_enriched_with_substantial_control_presence",
              "covid_predominant",
              "covid_only",
              "covid_enriched_trend_not_significant"
            ]
          },
          {
            "name": "focus_population_location_gate",
            "role": "gate",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "ectopic_parenchyma_in_covid",
              "parenchyma_only",
              "bronchiolized_or_peribronchial_parenchyma",
              "airway_and_parenchyma"
            ]
          },
          {
            "name": "focus_population_recovery_mode_gate",
            "role": "gate",
            "field": "$.focus_population_recovery_mode",
            "kind": "value",
            "op": "in",
            "expected": [
              "continuum_not_discrete_cluster"
            ]
          },
          {
            "name": "focus_population_identity_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "ionocyte_like_beta",
              "neuroendocrine_like_gamma",
              "basal_stem_progenitor_like_delta",
              "aberrant_basaloid_transitional_like_epsilon",
              "neuronal_or_neural_gene_minor_population_zeta",
              "deuterosomal_ciliated_progenitor_like_eta",
              "hillock_transitional_like_theta",
              "low_quality_doublet_or_debris",
              "no_rare_cell_recovered",
              "not_assessed"
            ]
          },
          {
            "name": "cf_wrong_lineage_marker_called_positive",
            "role": "hard_fail",
            "field": "$.focus_population_marker_calls[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "focus_population_condition_distribution_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "equally_distributed",
              "control_predominant",
              "not_assessed"
            ]
          },
          {
            "name": "focus_population_location_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "airway_only",
              "not_assessed"
            ]
          },
          {
            "name": "cross_axis_canonical_identity_with_airway_only_location_incoherence",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "focus_population_identity": {
                "op": "in",
                "values": [
                  "chemosensory_marker_expressing_population_alpha"
                ]
              },
              "focus_population_location": {
                "op": "in",
                "values": [
                  "airway_only"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-6-claude-code",
      "modelName": "claude-opus-4-6",
      "harness": "claude-code",
      "task": "gse171524_ectopic_airway_rare_cell",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1361,
      "cost": 1.4596845,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "focus_population_identity_gate",
            "role": "gate",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "chemosensory_marker_expressing_population_alpha"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "focus_population_marker_calls",
            "kind": "list",
            "matchKey": "marker",
            "k": 3,
            "expected": [
              {
                "key": "POU2F3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "CHAT",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "TRPM5",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "LRMP",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNAT3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "AVIL",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNG13",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              }
            ]
          },
          {
            "name": "focus_population_condition_distribution_gate",
            "role": "gate",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_enriched_with_substantial_control_presence",
              "covid_predominant",
              "covid_only",
              "covid_enriched_trend_not_significant"
            ]
          },
          {
            "name": "focus_population_location_gate",
            "role": "gate",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "ectopic_parenchyma_in_covid",
              "parenchyma_only",
              "bronchiolized_or_peribronchial_parenchyma",
              "airway_and_parenchyma"
            ]
          },
          {
            "name": "focus_population_recovery_mode_gate",
            "role": "gate",
            "field": "$.focus_population_recovery_mode",
            "kind": "value",
            "op": "in",
            "expected": [
              "continuum_not_discrete_cluster"
            ]
          },
          {
            "name": "focus_population_identity_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "ionocyte_like_beta",
              "neuroendocrine_like_gamma",
              "basal_stem_progenitor_like_delta",
              "aberrant_basaloid_transitional_like_epsilon",
              "neuronal_or_neural_gene_minor_population_zeta",
              "deuterosomal_ciliated_progenitor_like_eta",
              "hillock_transitional_like_theta",
              "low_quality_doublet_or_debris",
              "no_rare_cell_recovered",
              "not_assessed"
            ]
          },
          {
            "name": "cf_wrong_lineage_marker_called_positive",
            "role": "hard_fail",
            "field": "$.focus_population_marker_calls[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "focus_population_condition_distribution_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "equally_distributed",
              "control_predominant",
              "not_assessed"
            ]
          },
          {
            "name": "focus_population_location_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "airway_only",
              "not_assessed"
            ]
          },
          {
            "name": "cross_axis_canonical_identity_with_airway_only_location_incoherence",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "focus_population_identity": {
                "op": "in",
                "values": [
                  "chemosensory_marker_expressing_population_alpha"
                ]
              },
              "focus_population_location": {
                "op": "in",
                "values": [
                  "airway_only"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-6-pi",
      "modelName": "claude-opus-4-6",
      "harness": "pi",
      "task": "gse171524_ectopic_airway_rare_cell",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 1958,
      "cost": 1.3793950000000001,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "focus_population_identity_gate",
            "role": "gate",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "chemosensory_marker_expressing_population_alpha"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "focus_population_marker_calls",
            "kind": "list",
            "matchKey": "marker",
            "k": 3,
            "expected": [
              {
                "key": "POU2F3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "CHAT",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "TRPM5",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "LRMP",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNAT3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "AVIL",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNG13",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              }
            ]
          },
          {
            "name": "focus_population_condition_distribution_gate",
            "role": "gate",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_enriched_with_substantial_control_presence",
              "covid_predominant",
              "covid_only",
              "covid_enriched_trend_not_significant"
            ]
          },
          {
            "name": "focus_population_location_gate",
            "role": "gate",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "ectopic_parenchyma_in_covid",
              "parenchyma_only",
              "bronchiolized_or_peribronchial_parenchyma",
              "airway_and_parenchyma"
            ]
          },
          {
            "name": "focus_population_recovery_mode_gate",
            "role": "gate",
            "field": "$.focus_population_recovery_mode",
            "kind": "value",
            "op": "in",
            "expected": [
              "continuum_not_discrete_cluster"
            ]
          },
          {
            "name": "focus_population_identity_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "ionocyte_like_beta",
              "neuroendocrine_like_gamma",
              "basal_stem_progenitor_like_delta",
              "aberrant_basaloid_transitional_like_epsilon",
              "neuronal_or_neural_gene_minor_population_zeta",
              "deuterosomal_ciliated_progenitor_like_eta",
              "hillock_transitional_like_theta",
              "low_quality_doublet_or_debris",
              "no_rare_cell_recovered",
              "not_assessed"
            ]
          },
          {
            "name": "cf_wrong_lineage_marker_called_positive",
            "role": "hard_fail",
            "field": "$.focus_population_marker_calls[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "focus_population_condition_distribution_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "equally_distributed",
              "control_predominant",
              "not_assessed"
            ]
          },
          {
            "name": "focus_population_location_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "airway_only",
              "not_assessed"
            ]
          },
          {
            "name": "cross_axis_canonical_identity_with_airway_only_location_incoherence",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "focus_population_identity": {
                "op": "in",
                "values": [
                  "chemosensory_marker_expressing_population_alpha"
                ]
              },
              "focus_population_location": {
                "op": "in",
                "values": [
                  "airway_only"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-6-pi",
      "modelName": "claude-opus-4-6",
      "harness": "pi",
      "task": "gse171524_ectopic_airway_rare_cell",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 2094,
      "cost": 3.132942000000001,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "focus_population_identity_gate",
            "role": "gate",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "chemosensory_marker_expressing_population_alpha"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "focus_population_marker_calls",
            "kind": "list",
            "matchKey": "marker",
            "k": 3,
            "expected": [
              {
                "key": "POU2F3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "CHAT",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "TRPM5",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "LRMP",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNAT3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "AVIL",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNG13",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              }
            ]
          },
          {
            "name": "focus_population_condition_distribution_gate",
            "role": "gate",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_enriched_with_substantial_control_presence",
              "covid_predominant",
              "covid_only",
              "covid_enriched_trend_not_significant"
            ]
          },
          {
            "name": "focus_population_location_gate",
            "role": "gate",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "ectopic_parenchyma_in_covid",
              "parenchyma_only",
              "bronchiolized_or_peribronchial_parenchyma",
              "airway_and_parenchyma"
            ]
          },
          {
            "name": "focus_population_recovery_mode_gate",
            "role": "gate",
            "field": "$.focus_population_recovery_mode",
            "kind": "value",
            "op": "in",
            "expected": [
              "continuum_not_discrete_cluster"
            ]
          },
          {
            "name": "focus_population_identity_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "ionocyte_like_beta",
              "neuroendocrine_like_gamma",
              "basal_stem_progenitor_like_delta",
              "aberrant_basaloid_transitional_like_epsilon",
              "neuronal_or_neural_gene_minor_population_zeta",
              "deuterosomal_ciliated_progenitor_like_eta",
              "hillock_transitional_like_theta",
              "low_quality_doublet_or_debris",
              "no_rare_cell_recovered",
              "not_assessed"
            ]
          },
          {
            "name": "cf_wrong_lineage_marker_called_positive",
            "role": "hard_fail",
            "field": "$.focus_population_marker_calls[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "focus_population_condition_distribution_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "equally_distributed",
              "control_predominant",
              "not_assessed"
            ]
          },
          {
            "name": "focus_population_location_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "airway_only",
              "not_assessed"
            ]
          },
          {
            "name": "cross_axis_canonical_identity_with_airway_only_location_incoherence",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "focus_population_identity": {
                "op": "in",
                "values": [
                  "chemosensory_marker_expressing_population_alpha"
                ]
              },
              "focus_population_location": {
                "op": "in",
                "values": [
                  "airway_only"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-6-pi",
      "modelName": "claude-opus-4-6",
      "harness": "pi",
      "task": "gse171524_ectopic_airway_rare_cell",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 4167,
      "cost": 2.7498112500000005,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "focus_population_identity_gate",
            "role": "gate",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "chemosensory_marker_expressing_population_alpha"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "focus_population_marker_calls",
            "kind": "list",
            "matchKey": "marker",
            "k": 3,
            "expected": [
              {
                "key": "POU2F3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "CHAT",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "TRPM5",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "LRMP",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNAT3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "AVIL",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNG13",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              }
            ]
          },
          {
            "name": "focus_population_condition_distribution_gate",
            "role": "gate",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_enriched_with_substantial_control_presence",
              "covid_predominant",
              "covid_only",
              "covid_enriched_trend_not_significant"
            ]
          },
          {
            "name": "focus_population_location_gate",
            "role": "gate",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "ectopic_parenchyma_in_covid",
              "parenchyma_only",
              "bronchiolized_or_peribronchial_parenchyma",
              "airway_and_parenchyma"
            ]
          },
          {
            "name": "focus_population_recovery_mode_gate",
            "role": "gate",
            "field": "$.focus_population_recovery_mode",
            "kind": "value",
            "op": "in",
            "expected": [
              "continuum_not_discrete_cluster"
            ]
          },
          {
            "name": "focus_population_identity_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "ionocyte_like_beta",
              "neuroendocrine_like_gamma",
              "basal_stem_progenitor_like_delta",
              "aberrant_basaloid_transitional_like_epsilon",
              "neuronal_or_neural_gene_minor_population_zeta",
              "deuterosomal_ciliated_progenitor_like_eta",
              "hillock_transitional_like_theta",
              "low_quality_doublet_or_debris",
              "no_rare_cell_recovered",
              "not_assessed"
            ]
          },
          {
            "name": "cf_wrong_lineage_marker_called_positive",
            "role": "hard_fail",
            "field": "$.focus_population_marker_calls[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "focus_population_condition_distribution_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "equally_distributed",
              "control_predominant",
              "not_assessed"
            ]
          },
          {
            "name": "focus_population_location_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "airway_only",
              "not_assessed"
            ]
          },
          {
            "name": "cross_axis_canonical_identity_with_airway_only_location_incoherence",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "focus_population_identity": {
                "op": "in",
                "values": [
                  "chemosensory_marker_expressing_population_alpha"
                ]
              },
              "focus_population_location": {
                "op": "in",
                "values": [
                  "airway_only"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-7-claude-code",
      "modelName": "claude-opus-4-7",
      "harness": "claude-code",
      "task": "gse171524_ectopic_airway_rare_cell",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 304,
      "cost": 0.8119132499999999,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "focus_population_identity_gate",
            "role": "gate",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "chemosensory_marker_expressing_population_alpha"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "focus_population_marker_calls",
            "kind": "list",
            "matchKey": "marker",
            "k": 3,
            "expected": [
              {
                "key": "POU2F3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "CHAT",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "TRPM5",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "LRMP",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNAT3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "AVIL",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNG13",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              }
            ]
          },
          {
            "name": "focus_population_condition_distribution_gate",
            "role": "gate",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_enriched_with_substantial_control_presence",
              "covid_predominant",
              "covid_only",
              "covid_enriched_trend_not_significant"
            ]
          },
          {
            "name": "focus_population_location_gate",
            "role": "gate",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "ectopic_parenchyma_in_covid",
              "parenchyma_only",
              "bronchiolized_or_peribronchial_parenchyma",
              "airway_and_parenchyma"
            ]
          },
          {
            "name": "focus_population_recovery_mode_gate",
            "role": "gate",
            "field": "$.focus_population_recovery_mode",
            "kind": "value",
            "op": "in",
            "expected": [
              "continuum_not_discrete_cluster"
            ]
          },
          {
            "name": "focus_population_identity_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "ionocyte_like_beta",
              "neuroendocrine_like_gamma",
              "basal_stem_progenitor_like_delta",
              "aberrant_basaloid_transitional_like_epsilon",
              "neuronal_or_neural_gene_minor_population_zeta",
              "deuterosomal_ciliated_progenitor_like_eta",
              "hillock_transitional_like_theta",
              "low_quality_doublet_or_debris",
              "no_rare_cell_recovered",
              "not_assessed"
            ]
          },
          {
            "name": "cf_wrong_lineage_marker_called_positive",
            "role": "hard_fail",
            "field": "$.focus_population_marker_calls[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "focus_population_condition_distribution_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "equally_distributed",
              "control_predominant",
              "not_assessed"
            ]
          },
          {
            "name": "focus_population_location_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "airway_only",
              "not_assessed"
            ]
          },
          {
            "name": "cross_axis_canonical_identity_with_airway_only_location_incoherence",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "focus_population_identity": {
                "op": "in",
                "values": [
                  "chemosensory_marker_expressing_population_alpha"
                ]
              },
              "focus_population_location": {
                "op": "in",
                "values": [
                  "airway_only"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-7-claude-code",
      "modelName": "claude-opus-4-7",
      "harness": "claude-code",
      "task": "gse171524_ectopic_airway_rare_cell",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 251,
      "cost": 0.6324847500000002,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "focus_population_identity_gate",
            "role": "gate",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "chemosensory_marker_expressing_population_alpha"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "focus_population_marker_calls",
            "kind": "list",
            "matchKey": "marker",
            "k": 3,
            "expected": [
              {
                "key": "POU2F3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "CHAT",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "TRPM5",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "LRMP",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNAT3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "AVIL",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNG13",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              }
            ]
          },
          {
            "name": "focus_population_condition_distribution_gate",
            "role": "gate",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_enriched_with_substantial_control_presence",
              "covid_predominant",
              "covid_only",
              "covid_enriched_trend_not_significant"
            ]
          },
          {
            "name": "focus_population_location_gate",
            "role": "gate",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "ectopic_parenchyma_in_covid",
              "parenchyma_only",
              "bronchiolized_or_peribronchial_parenchyma",
              "airway_and_parenchyma"
            ]
          },
          {
            "name": "focus_population_recovery_mode_gate",
            "role": "gate",
            "field": "$.focus_population_recovery_mode",
            "kind": "value",
            "op": "in",
            "expected": [
              "continuum_not_discrete_cluster"
            ]
          },
          {
            "name": "focus_population_identity_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "ionocyte_like_beta",
              "neuroendocrine_like_gamma",
              "basal_stem_progenitor_like_delta",
              "aberrant_basaloid_transitional_like_epsilon",
              "neuronal_or_neural_gene_minor_population_zeta",
              "deuterosomal_ciliated_progenitor_like_eta",
              "hillock_transitional_like_theta",
              "low_quality_doublet_or_debris",
              "no_rare_cell_recovered",
              "not_assessed"
            ]
          },
          {
            "name": "cf_wrong_lineage_marker_called_positive",
            "role": "hard_fail",
            "field": "$.focus_population_marker_calls[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "focus_population_condition_distribution_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "equally_distributed",
              "control_predominant",
              "not_assessed"
            ]
          },
          {
            "name": "focus_population_location_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "airway_only",
              "not_assessed"
            ]
          },
          {
            "name": "cross_axis_canonical_identity_with_airway_only_location_incoherence",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "focus_population_identity": {
                "op": "in",
                "values": [
                  "chemosensory_marker_expressing_population_alpha"
                ]
              },
              "focus_population_location": {
                "op": "in",
                "values": [
                  "airway_only"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-7-claude-code",
      "modelName": "claude-opus-4-7",
      "harness": "claude-code",
      "task": "gse171524_ectopic_airway_rare_cell",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1696,
      "cost": 8.493597499999998,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "focus_population_identity_gate",
            "role": "gate",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "chemosensory_marker_expressing_population_alpha"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "focus_population_marker_calls",
            "kind": "list",
            "matchKey": "marker",
            "k": 3,
            "expected": [
              {
                "key": "POU2F3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "CHAT",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "TRPM5",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "LRMP",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNAT3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "AVIL",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNG13",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              }
            ]
          },
          {
            "name": "focus_population_condition_distribution_gate",
            "role": "gate",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_enriched_with_substantial_control_presence",
              "covid_predominant",
              "covid_only",
              "covid_enriched_trend_not_significant"
            ]
          },
          {
            "name": "focus_population_location_gate",
            "role": "gate",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "ectopic_parenchyma_in_covid",
              "parenchyma_only",
              "bronchiolized_or_peribronchial_parenchyma",
              "airway_and_parenchyma"
            ]
          },
          {
            "name": "focus_population_recovery_mode_gate",
            "role": "gate",
            "field": "$.focus_population_recovery_mode",
            "kind": "value",
            "op": "in",
            "expected": [
              "continuum_not_discrete_cluster"
            ]
          },
          {
            "name": "focus_population_identity_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "ionocyte_like_beta",
              "neuroendocrine_like_gamma",
              "basal_stem_progenitor_like_delta",
              "aberrant_basaloid_transitional_like_epsilon",
              "neuronal_or_neural_gene_minor_population_zeta",
              "deuterosomal_ciliated_progenitor_like_eta",
              "hillock_transitional_like_theta",
              "low_quality_doublet_or_debris",
              "no_rare_cell_recovered",
              "not_assessed"
            ]
          },
          {
            "name": "cf_wrong_lineage_marker_called_positive",
            "role": "hard_fail",
            "field": "$.focus_population_marker_calls[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "focus_population_condition_distribution_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "equally_distributed",
              "control_predominant",
              "not_assessed"
            ]
          },
          {
            "name": "focus_population_location_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "airway_only",
              "not_assessed"
            ]
          },
          {
            "name": "cross_axis_canonical_identity_with_airway_only_location_incoherence",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "focus_population_identity": {
                "op": "in",
                "values": [
                  "chemosensory_marker_expressing_population_alpha"
                ]
              },
              "focus_population_location": {
                "op": "in",
                "values": [
                  "airway_only"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-7-pi",
      "modelName": "claude-opus-4-7",
      "harness": "pi",
      "task": "gse171524_ectopic_airway_rare_cell",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 1411,
      "cost": 0.9079662499999999,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "focus_population_identity_gate",
            "role": "gate",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "chemosensory_marker_expressing_population_alpha"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "focus_population_marker_calls",
            "kind": "list",
            "matchKey": "marker",
            "k": 3,
            "expected": [
              {
                "key": "POU2F3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "CHAT",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "TRPM5",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "LRMP",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNAT3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "AVIL",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNG13",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              }
            ]
          },
          {
            "name": "focus_population_condition_distribution_gate",
            "role": "gate",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_enriched_with_substantial_control_presence",
              "covid_predominant",
              "covid_only",
              "covid_enriched_trend_not_significant"
            ]
          },
          {
            "name": "focus_population_location_gate",
            "role": "gate",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "ectopic_parenchyma_in_covid",
              "parenchyma_only",
              "bronchiolized_or_peribronchial_parenchyma",
              "airway_and_parenchyma"
            ]
          },
          {
            "name": "focus_population_recovery_mode_gate",
            "role": "gate",
            "field": "$.focus_population_recovery_mode",
            "kind": "value",
            "op": "in",
            "expected": [
              "continuum_not_discrete_cluster"
            ]
          },
          {
            "name": "focus_population_identity_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "ionocyte_like_beta",
              "neuroendocrine_like_gamma",
              "basal_stem_progenitor_like_delta",
              "aberrant_basaloid_transitional_like_epsilon",
              "neuronal_or_neural_gene_minor_population_zeta",
              "deuterosomal_ciliated_progenitor_like_eta",
              "hillock_transitional_like_theta",
              "low_quality_doublet_or_debris",
              "no_rare_cell_recovered",
              "not_assessed"
            ]
          },
          {
            "name": "cf_wrong_lineage_marker_called_positive",
            "role": "hard_fail",
            "field": "$.focus_population_marker_calls[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "focus_population_condition_distribution_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "equally_distributed",
              "control_predominant",
              "not_assessed"
            ]
          },
          {
            "name": "focus_population_location_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "airway_only",
              "not_assessed"
            ]
          },
          {
            "name": "cross_axis_canonical_identity_with_airway_only_location_incoherence",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "focus_population_identity": {
                "op": "in",
                "values": [
                  "chemosensory_marker_expressing_population_alpha"
                ]
              },
              "focus_population_location": {
                "op": "in",
                "values": [
                  "airway_only"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-7-pi",
      "modelName": "claude-opus-4-7",
      "harness": "pi",
      "task": "gse171524_ectopic_airway_rare_cell",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 2058,
      "cost": 1.2005972500000002,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "focus_population_identity_gate",
            "role": "gate",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "chemosensory_marker_expressing_population_alpha"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "focus_population_marker_calls",
            "kind": "list",
            "matchKey": "marker",
            "k": 3,
            "expected": [
              {
                "key": "POU2F3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "CHAT",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "TRPM5",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "LRMP",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNAT3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "AVIL",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNG13",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              }
            ]
          },
          {
            "name": "focus_population_condition_distribution_gate",
            "role": "gate",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_enriched_with_substantial_control_presence",
              "covid_predominant",
              "covid_only",
              "covid_enriched_trend_not_significant"
            ]
          },
          {
            "name": "focus_population_location_gate",
            "role": "gate",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "ectopic_parenchyma_in_covid",
              "parenchyma_only",
              "bronchiolized_or_peribronchial_parenchyma",
              "airway_and_parenchyma"
            ]
          },
          {
            "name": "focus_population_recovery_mode_gate",
            "role": "gate",
            "field": "$.focus_population_recovery_mode",
            "kind": "value",
            "op": "in",
            "expected": [
              "continuum_not_discrete_cluster"
            ]
          },
          {
            "name": "focus_population_identity_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "ionocyte_like_beta",
              "neuroendocrine_like_gamma",
              "basal_stem_progenitor_like_delta",
              "aberrant_basaloid_transitional_like_epsilon",
              "neuronal_or_neural_gene_minor_population_zeta",
              "deuterosomal_ciliated_progenitor_like_eta",
              "hillock_transitional_like_theta",
              "low_quality_doublet_or_debris",
              "no_rare_cell_recovered",
              "not_assessed"
            ]
          },
          {
            "name": "cf_wrong_lineage_marker_called_positive",
            "role": "hard_fail",
            "field": "$.focus_population_marker_calls[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "focus_population_condition_distribution_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "equally_distributed",
              "control_predominant",
              "not_assessed"
            ]
          },
          {
            "name": "focus_population_location_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "airway_only",
              "not_assessed"
            ]
          },
          {
            "name": "cross_axis_canonical_identity_with_airway_only_location_incoherence",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "focus_population_identity": {
                "op": "in",
                "values": [
                  "chemosensory_marker_expressing_population_alpha"
                ]
              },
              "focus_population_location": {
                "op": "in",
                "values": [
                  "airway_only"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-7-pi",
      "modelName": "claude-opus-4-7",
      "harness": "pi",
      "task": "gse171524_ectopic_airway_rare_cell",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1788,
      "cost": 1.45557725,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "focus_population_identity_gate",
            "role": "gate",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "chemosensory_marker_expressing_population_alpha"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "focus_population_marker_calls",
            "kind": "list",
            "matchKey": "marker",
            "k": 3,
            "expected": [
              {
                "key": "POU2F3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "CHAT",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "TRPM5",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "LRMP",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNAT3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "AVIL",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNG13",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              }
            ]
          },
          {
            "name": "focus_population_condition_distribution_gate",
            "role": "gate",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_enriched_with_substantial_control_presence",
              "covid_predominant",
              "covid_only",
              "covid_enriched_trend_not_significant"
            ]
          },
          {
            "name": "focus_population_location_gate",
            "role": "gate",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "ectopic_parenchyma_in_covid",
              "parenchyma_only",
              "bronchiolized_or_peribronchial_parenchyma",
              "airway_and_parenchyma"
            ]
          },
          {
            "name": "focus_population_recovery_mode_gate",
            "role": "gate",
            "field": "$.focus_population_recovery_mode",
            "kind": "value",
            "op": "in",
            "expected": [
              "continuum_not_discrete_cluster"
            ]
          },
          {
            "name": "focus_population_identity_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "ionocyte_like_beta",
              "neuroendocrine_like_gamma",
              "basal_stem_progenitor_like_delta",
              "aberrant_basaloid_transitional_like_epsilon",
              "neuronal_or_neural_gene_minor_population_zeta",
              "deuterosomal_ciliated_progenitor_like_eta",
              "hillock_transitional_like_theta",
              "low_quality_doublet_or_debris",
              "no_rare_cell_recovered",
              "not_assessed"
            ]
          },
          {
            "name": "cf_wrong_lineage_marker_called_positive",
            "role": "hard_fail",
            "field": "$.focus_population_marker_calls[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "focus_population_condition_distribution_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "equally_distributed",
              "control_predominant",
              "not_assessed"
            ]
          },
          {
            "name": "focus_population_location_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "airway_only",
              "not_assessed"
            ]
          },
          {
            "name": "cross_axis_canonical_identity_with_airway_only_location_incoherence",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "focus_population_identity": {
                "op": "in",
                "values": [
                  "chemosensory_marker_expressing_population_alpha"
                ]
              },
              "focus_population_location": {
                "op": "in",
                "values": [
                  "airway_only"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-8-claude-code",
      "modelName": "claude-opus-4-8",
      "harness": "claude-code",
      "task": "gse171524_ectopic_airway_rare_cell",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 1557,
      "cost": 4.0954715,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "focus_population_identity_gate",
            "role": "gate",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "chemosensory_marker_expressing_population_alpha"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "focus_population_marker_calls",
            "kind": "list",
            "matchKey": "marker",
            "k": 3,
            "expected": [
              {
                "key": "POU2F3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "CHAT",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "TRPM5",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "LRMP",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNAT3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "AVIL",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNG13",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              }
            ]
          },
          {
            "name": "focus_population_condition_distribution_gate",
            "role": "gate",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_enriched_with_substantial_control_presence",
              "covid_predominant",
              "covid_only",
              "covid_enriched_trend_not_significant"
            ]
          },
          {
            "name": "focus_population_location_gate",
            "role": "gate",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "ectopic_parenchyma_in_covid",
              "parenchyma_only",
              "bronchiolized_or_peribronchial_parenchyma",
              "airway_and_parenchyma"
            ]
          },
          {
            "name": "focus_population_recovery_mode_gate",
            "role": "gate",
            "field": "$.focus_population_recovery_mode",
            "kind": "value",
            "op": "in",
            "expected": [
              "continuum_not_discrete_cluster"
            ]
          },
          {
            "name": "focus_population_identity_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "ionocyte_like_beta",
              "neuroendocrine_like_gamma",
              "basal_stem_progenitor_like_delta",
              "aberrant_basaloid_transitional_like_epsilon",
              "neuronal_or_neural_gene_minor_population_zeta",
              "deuterosomal_ciliated_progenitor_like_eta",
              "hillock_transitional_like_theta",
              "low_quality_doublet_or_debris",
              "no_rare_cell_recovered",
              "not_assessed"
            ]
          },
          {
            "name": "cf_wrong_lineage_marker_called_positive",
            "role": "hard_fail",
            "field": "$.focus_population_marker_calls[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "focus_population_condition_distribution_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "equally_distributed",
              "control_predominant",
              "not_assessed"
            ]
          },
          {
            "name": "focus_population_location_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "airway_only",
              "not_assessed"
            ]
          },
          {
            "name": "cross_axis_canonical_identity_with_airway_only_location_incoherence",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "focus_population_identity": {
                "op": "in",
                "values": [
                  "chemosensory_marker_expressing_population_alpha"
                ]
              },
              "focus_population_location": {
                "op": "in",
                "values": [
                  "airway_only"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-8-claude-code",
      "modelName": "claude-opus-4-8",
      "harness": "claude-code",
      "task": "gse171524_ectopic_airway_rare_cell",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 1632,
      "cost": 3.6860572499999993,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "focus_population_identity_gate",
            "role": "gate",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "chemosensory_marker_expressing_population_alpha"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "focus_population_marker_calls",
            "kind": "list",
            "matchKey": "marker",
            "k": 3,
            "expected": [
              {
                "key": "POU2F3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "CHAT",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "TRPM5",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "LRMP",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNAT3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "AVIL",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNG13",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              }
            ]
          },
          {
            "name": "focus_population_condition_distribution_gate",
            "role": "gate",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_enriched_with_substantial_control_presence",
              "covid_predominant",
              "covid_only",
              "covid_enriched_trend_not_significant"
            ]
          },
          {
            "name": "focus_population_location_gate",
            "role": "gate",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "ectopic_parenchyma_in_covid",
              "parenchyma_only",
              "bronchiolized_or_peribronchial_parenchyma",
              "airway_and_parenchyma"
            ]
          },
          {
            "name": "focus_population_recovery_mode_gate",
            "role": "gate",
            "field": "$.focus_population_recovery_mode",
            "kind": "value",
            "op": "in",
            "expected": [
              "continuum_not_discrete_cluster"
            ]
          },
          {
            "name": "focus_population_identity_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "ionocyte_like_beta",
              "neuroendocrine_like_gamma",
              "basal_stem_progenitor_like_delta",
              "aberrant_basaloid_transitional_like_epsilon",
              "neuronal_or_neural_gene_minor_population_zeta",
              "deuterosomal_ciliated_progenitor_like_eta",
              "hillock_transitional_like_theta",
              "low_quality_doublet_or_debris",
              "no_rare_cell_recovered",
              "not_assessed"
            ]
          },
          {
            "name": "cf_wrong_lineage_marker_called_positive",
            "role": "hard_fail",
            "field": "$.focus_population_marker_calls[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "focus_population_condition_distribution_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "equally_distributed",
              "control_predominant",
              "not_assessed"
            ]
          },
          {
            "name": "focus_population_location_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "airway_only",
              "not_assessed"
            ]
          },
          {
            "name": "cross_axis_canonical_identity_with_airway_only_location_incoherence",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "focus_population_identity": {
                "op": "in",
                "values": [
                  "chemosensory_marker_expressing_population_alpha"
                ]
              },
              "focus_population_location": {
                "op": "in",
                "values": [
                  "airway_only"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-8-claude-code",
      "modelName": "claude-opus-4-8",
      "harness": "claude-code",
      "task": "gse171524_ectopic_airway_rare_cell",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 916,
      "cost": 4.305160249999999,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "focus_population_identity_gate",
            "role": "gate",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "chemosensory_marker_expressing_population_alpha"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "focus_population_marker_calls",
            "kind": "list",
            "matchKey": "marker",
            "k": 3,
            "expected": [
              {
                "key": "POU2F3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "CHAT",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "TRPM5",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "LRMP",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNAT3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "AVIL",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNG13",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              }
            ]
          },
          {
            "name": "focus_population_condition_distribution_gate",
            "role": "gate",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_enriched_with_substantial_control_presence",
              "covid_predominant",
              "covid_only",
              "covid_enriched_trend_not_significant"
            ]
          },
          {
            "name": "focus_population_location_gate",
            "role": "gate",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "ectopic_parenchyma_in_covid",
              "parenchyma_only",
              "bronchiolized_or_peribronchial_parenchyma",
              "airway_and_parenchyma"
            ]
          },
          {
            "name": "focus_population_recovery_mode_gate",
            "role": "gate",
            "field": "$.focus_population_recovery_mode",
            "kind": "value",
            "op": "in",
            "expected": [
              "continuum_not_discrete_cluster"
            ]
          },
          {
            "name": "focus_population_identity_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "ionocyte_like_beta",
              "neuroendocrine_like_gamma",
              "basal_stem_progenitor_like_delta",
              "aberrant_basaloid_transitional_like_epsilon",
              "neuronal_or_neural_gene_minor_population_zeta",
              "deuterosomal_ciliated_progenitor_like_eta",
              "hillock_transitional_like_theta",
              "low_quality_doublet_or_debris",
              "no_rare_cell_recovered",
              "not_assessed"
            ]
          },
          {
            "name": "cf_wrong_lineage_marker_called_positive",
            "role": "hard_fail",
            "field": "$.focus_population_marker_calls[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "focus_population_condition_distribution_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "equally_distributed",
              "control_predominant",
              "not_assessed"
            ]
          },
          {
            "name": "focus_population_location_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "airway_only",
              "not_assessed"
            ]
          },
          {
            "name": "cross_axis_canonical_identity_with_airway_only_location_incoherence",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "focus_population_identity": {
                "op": "in",
                "values": [
                  "chemosensory_marker_expressing_population_alpha"
                ]
              },
              "focus_population_location": {
                "op": "in",
                "values": [
                  "airway_only"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-8-pi",
      "modelName": "claude-opus-4-8",
      "harness": "pi",
      "task": "gse171524_ectopic_airway_rare_cell",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 1120,
      "cost": 1.9430135,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "focus_population_identity_gate",
            "role": "gate",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "chemosensory_marker_expressing_population_alpha"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "focus_population_marker_calls",
            "kind": "list",
            "matchKey": "marker",
            "k": 3,
            "expected": [
              {
                "key": "POU2F3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "CHAT",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "TRPM5",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "LRMP",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNAT3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "AVIL",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNG13",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              }
            ]
          },
          {
            "name": "focus_population_condition_distribution_gate",
            "role": "gate",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_enriched_with_substantial_control_presence",
              "covid_predominant",
              "covid_only",
              "covid_enriched_trend_not_significant"
            ]
          },
          {
            "name": "focus_population_location_gate",
            "role": "gate",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "ectopic_parenchyma_in_covid",
              "parenchyma_only",
              "bronchiolized_or_peribronchial_parenchyma",
              "airway_and_parenchyma"
            ]
          },
          {
            "name": "focus_population_recovery_mode_gate",
            "role": "gate",
            "field": "$.focus_population_recovery_mode",
            "kind": "value",
            "op": "in",
            "expected": [
              "continuum_not_discrete_cluster"
            ]
          },
          {
            "name": "focus_population_identity_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "ionocyte_like_beta",
              "neuroendocrine_like_gamma",
              "basal_stem_progenitor_like_delta",
              "aberrant_basaloid_transitional_like_epsilon",
              "neuronal_or_neural_gene_minor_population_zeta",
              "deuterosomal_ciliated_progenitor_like_eta",
              "hillock_transitional_like_theta",
              "low_quality_doublet_or_debris",
              "no_rare_cell_recovered",
              "not_assessed"
            ]
          },
          {
            "name": "cf_wrong_lineage_marker_called_positive",
            "role": "hard_fail",
            "field": "$.focus_population_marker_calls[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "focus_population_condition_distribution_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "equally_distributed",
              "control_predominant",
              "not_assessed"
            ]
          },
          {
            "name": "focus_population_location_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "airway_only",
              "not_assessed"
            ]
          },
          {
            "name": "cross_axis_canonical_identity_with_airway_only_location_incoherence",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "focus_population_identity": {
                "op": "in",
                "values": [
                  "chemosensory_marker_expressing_population_alpha"
                ]
              },
              "focus_population_location": {
                "op": "in",
                "values": [
                  "airway_only"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-8-pi",
      "modelName": "claude-opus-4-8",
      "harness": "pi",
      "task": "gse171524_ectopic_airway_rare_cell",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 1183,
      "cost": 1.7583460000000002,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "focus_population_identity_gate",
            "role": "gate",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "chemosensory_marker_expressing_population_alpha"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "focus_population_marker_calls",
            "kind": "list",
            "matchKey": "marker",
            "k": 3,
            "expected": [
              {
                "key": "POU2F3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "CHAT",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "TRPM5",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "LRMP",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNAT3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "AVIL",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNG13",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              }
            ]
          },
          {
            "name": "focus_population_condition_distribution_gate",
            "role": "gate",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_enriched_with_substantial_control_presence",
              "covid_predominant",
              "covid_only",
              "covid_enriched_trend_not_significant"
            ]
          },
          {
            "name": "focus_population_location_gate",
            "role": "gate",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "ectopic_parenchyma_in_covid",
              "parenchyma_only",
              "bronchiolized_or_peribronchial_parenchyma",
              "airway_and_parenchyma"
            ]
          },
          {
            "name": "focus_population_recovery_mode_gate",
            "role": "gate",
            "field": "$.focus_population_recovery_mode",
            "kind": "value",
            "op": "in",
            "expected": [
              "continuum_not_discrete_cluster"
            ]
          },
          {
            "name": "focus_population_identity_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "ionocyte_like_beta",
              "neuroendocrine_like_gamma",
              "basal_stem_progenitor_like_delta",
              "aberrant_basaloid_transitional_like_epsilon",
              "neuronal_or_neural_gene_minor_population_zeta",
              "deuterosomal_ciliated_progenitor_like_eta",
              "hillock_transitional_like_theta",
              "low_quality_doublet_or_debris",
              "no_rare_cell_recovered",
              "not_assessed"
            ]
          },
          {
            "name": "cf_wrong_lineage_marker_called_positive",
            "role": "hard_fail",
            "field": "$.focus_population_marker_calls[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "focus_population_condition_distribution_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "equally_distributed",
              "control_predominant",
              "not_assessed"
            ]
          },
          {
            "name": "focus_population_location_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "airway_only",
              "not_assessed"
            ]
          },
          {
            "name": "cross_axis_canonical_identity_with_airway_only_location_incoherence",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "focus_population_identity": {
                "op": "in",
                "values": [
                  "chemosensory_marker_expressing_population_alpha"
                ]
              },
              "focus_population_location": {
                "op": "in",
                "values": [
                  "airway_only"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-8-pi",
      "modelName": "claude-opus-4-8",
      "harness": "pi",
      "task": "gse171524_ectopic_airway_rare_cell",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1080,
      "cost": 1.9019560000000002,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "focus_population_identity_gate",
            "role": "gate",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "chemosensory_marker_expressing_population_alpha"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "focus_population_marker_calls",
            "kind": "list",
            "matchKey": "marker",
            "k": 3,
            "expected": [
              {
                "key": "POU2F3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "CHAT",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "TRPM5",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "LRMP",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNAT3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "AVIL",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNG13",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              }
            ]
          },
          {
            "name": "focus_population_condition_distribution_gate",
            "role": "gate",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_enriched_with_substantial_control_presence",
              "covid_predominant",
              "covid_only",
              "covid_enriched_trend_not_significant"
            ]
          },
          {
            "name": "focus_population_location_gate",
            "role": "gate",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "ectopic_parenchyma_in_covid",
              "parenchyma_only",
              "bronchiolized_or_peribronchial_parenchyma",
              "airway_and_parenchyma"
            ]
          },
          {
            "name": "focus_population_recovery_mode_gate",
            "role": "gate",
            "field": "$.focus_population_recovery_mode",
            "kind": "value",
            "op": "in",
            "expected": [
              "continuum_not_discrete_cluster"
            ]
          },
          {
            "name": "focus_population_identity_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "ionocyte_like_beta",
              "neuroendocrine_like_gamma",
              "basal_stem_progenitor_like_delta",
              "aberrant_basaloid_transitional_like_epsilon",
              "neuronal_or_neural_gene_minor_population_zeta",
              "deuterosomal_ciliated_progenitor_like_eta",
              "hillock_transitional_like_theta",
              "low_quality_doublet_or_debris",
              "no_rare_cell_recovered",
              "not_assessed"
            ]
          },
          {
            "name": "cf_wrong_lineage_marker_called_positive",
            "role": "hard_fail",
            "field": "$.focus_population_marker_calls[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "focus_population_condition_distribution_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "equally_distributed",
              "control_predominant",
              "not_assessed"
            ]
          },
          {
            "name": "focus_population_location_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "airway_only",
              "not_assessed"
            ]
          },
          {
            "name": "cross_axis_canonical_identity_with_airway_only_location_incoherence",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "focus_population_identity": {
                "op": "in",
                "values": [
                  "chemosensory_marker_expressing_population_alpha"
                ]
              },
              "focus_population_location": {
                "op": "in",
                "values": [
                  "airway_only"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-4-6-pi",
      "modelName": "claude-sonnet-4-6",
      "harness": "pi",
      "task": "gse171524_ectopic_airway_rare_cell",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 2763,
      "cost": 2.2707735,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "focus_population_identity_gate",
            "role": "gate",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "chemosensory_marker_expressing_population_alpha"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "focus_population_marker_calls",
            "kind": "list",
            "matchKey": "marker",
            "k": 3,
            "expected": [
              {
                "key": "POU2F3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "CHAT",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "TRPM5",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "LRMP",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNAT3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "AVIL",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNG13",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              }
            ]
          },
          {
            "name": "focus_population_condition_distribution_gate",
            "role": "gate",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_enriched_with_substantial_control_presence",
              "covid_predominant",
              "covid_only",
              "covid_enriched_trend_not_significant"
            ]
          },
          {
            "name": "focus_population_location_gate",
            "role": "gate",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "ectopic_parenchyma_in_covid",
              "parenchyma_only",
              "bronchiolized_or_peribronchial_parenchyma",
              "airway_and_parenchyma"
            ]
          },
          {
            "name": "focus_population_recovery_mode_gate",
            "role": "gate",
            "field": "$.focus_population_recovery_mode",
            "kind": "value",
            "op": "in",
            "expected": [
              "continuum_not_discrete_cluster"
            ]
          },
          {
            "name": "focus_population_identity_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "ionocyte_like_beta",
              "neuroendocrine_like_gamma",
              "basal_stem_progenitor_like_delta",
              "aberrant_basaloid_transitional_like_epsilon",
              "neuronal_or_neural_gene_minor_population_zeta",
              "deuterosomal_ciliated_progenitor_like_eta",
              "hillock_transitional_like_theta",
              "low_quality_doublet_or_debris",
              "no_rare_cell_recovered",
              "not_assessed"
            ]
          },
          {
            "name": "cf_wrong_lineage_marker_called_positive",
            "role": "hard_fail",
            "field": "$.focus_population_marker_calls[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "focus_population_condition_distribution_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "equally_distributed",
              "control_predominant",
              "not_assessed"
            ]
          },
          {
            "name": "focus_population_location_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "airway_only",
              "not_assessed"
            ]
          },
          {
            "name": "cross_axis_canonical_identity_with_airway_only_location_incoherence",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "focus_population_identity": {
                "op": "in",
                "values": [
                  "chemosensory_marker_expressing_population_alpha"
                ]
              },
              "focus_population_location": {
                "op": "in",
                "values": [
                  "airway_only"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-4-6-pi",
      "modelName": "claude-sonnet-4-6",
      "harness": "pi",
      "task": "gse171524_ectopic_airway_rare_cell",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 4031,
      "cost": 3.5729179500000003,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "focus_population_identity_gate",
            "role": "gate",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "chemosensory_marker_expressing_population_alpha"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "focus_population_marker_calls",
            "kind": "list",
            "matchKey": "marker",
            "k": 3,
            "expected": [
              {
                "key": "POU2F3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "CHAT",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "TRPM5",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "LRMP",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNAT3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "AVIL",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNG13",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              }
            ]
          },
          {
            "name": "focus_population_condition_distribution_gate",
            "role": "gate",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_enriched_with_substantial_control_presence",
              "covid_predominant",
              "covid_only",
              "covid_enriched_trend_not_significant"
            ]
          },
          {
            "name": "focus_population_location_gate",
            "role": "gate",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "ectopic_parenchyma_in_covid",
              "parenchyma_only",
              "bronchiolized_or_peribronchial_parenchyma",
              "airway_and_parenchyma"
            ]
          },
          {
            "name": "focus_population_recovery_mode_gate",
            "role": "gate",
            "field": "$.focus_population_recovery_mode",
            "kind": "value",
            "op": "in",
            "expected": [
              "continuum_not_discrete_cluster"
            ]
          },
          {
            "name": "focus_population_identity_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "ionocyte_like_beta",
              "neuroendocrine_like_gamma",
              "basal_stem_progenitor_like_delta",
              "aberrant_basaloid_transitional_like_epsilon",
              "neuronal_or_neural_gene_minor_population_zeta",
              "deuterosomal_ciliated_progenitor_like_eta",
              "hillock_transitional_like_theta",
              "low_quality_doublet_or_debris",
              "no_rare_cell_recovered",
              "not_assessed"
            ]
          },
          {
            "name": "cf_wrong_lineage_marker_called_positive",
            "role": "hard_fail",
            "field": "$.focus_population_marker_calls[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "focus_population_condition_distribution_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "equally_distributed",
              "control_predominant",
              "not_assessed"
            ]
          },
          {
            "name": "focus_population_location_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "airway_only",
              "not_assessed"
            ]
          },
          {
            "name": "cross_axis_canonical_identity_with_airway_only_location_incoherence",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "focus_population_identity": {
                "op": "in",
                "values": [
                  "chemosensory_marker_expressing_population_alpha"
                ]
              },
              "focus_population_location": {
                "op": "in",
                "values": [
                  "airway_only"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-4-6-pi",
      "modelName": "claude-sonnet-4-6",
      "harness": "pi",
      "task": "gse171524_ectopic_airway_rare_cell",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 3511,
      "cost": 2.56468515,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "focus_population_identity_gate",
            "role": "gate",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "chemosensory_marker_expressing_population_alpha"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "focus_population_marker_calls",
            "kind": "list",
            "matchKey": "marker",
            "k": 3,
            "expected": [
              {
                "key": "POU2F3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "CHAT",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "TRPM5",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "LRMP",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNAT3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "AVIL",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNG13",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              }
            ]
          },
          {
            "name": "focus_population_condition_distribution_gate",
            "role": "gate",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_enriched_with_substantial_control_presence",
              "covid_predominant",
              "covid_only",
              "covid_enriched_trend_not_significant"
            ]
          },
          {
            "name": "focus_population_location_gate",
            "role": "gate",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "ectopic_parenchyma_in_covid",
              "parenchyma_only",
              "bronchiolized_or_peribronchial_parenchyma",
              "airway_and_parenchyma"
            ]
          },
          {
            "name": "focus_population_recovery_mode_gate",
            "role": "gate",
            "field": "$.focus_population_recovery_mode",
            "kind": "value",
            "op": "in",
            "expected": [
              "continuum_not_discrete_cluster"
            ]
          },
          {
            "name": "focus_population_identity_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "ionocyte_like_beta",
              "neuroendocrine_like_gamma",
              "basal_stem_progenitor_like_delta",
              "aberrant_basaloid_transitional_like_epsilon",
              "neuronal_or_neural_gene_minor_population_zeta",
              "deuterosomal_ciliated_progenitor_like_eta",
              "hillock_transitional_like_theta",
              "low_quality_doublet_or_debris",
              "no_rare_cell_recovered",
              "not_assessed"
            ]
          },
          {
            "name": "cf_wrong_lineage_marker_called_positive",
            "role": "hard_fail",
            "field": "$.focus_population_marker_calls[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "focus_population_condition_distribution_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "equally_distributed",
              "control_predominant",
              "not_assessed"
            ]
          },
          {
            "name": "focus_population_location_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "airway_only",
              "not_assessed"
            ]
          },
          {
            "name": "cross_axis_canonical_identity_with_airway_only_location_incoherence",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "focus_population_identity": {
                "op": "in",
                "values": [
                  "chemosensory_marker_expressing_population_alpha"
                ]
              },
              "focus_population_location": {
                "op": "in",
                "values": [
                  "airway_only"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-gemini-3-1-pro-preview-pi",
      "modelName": "gemini-3.1-pro-preview",
      "harness": "pi",
      "task": "gse171524_ectopic_airway_rare_cell",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 830,
      "cost": 0.8555478,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "focus_population_identity_gate",
            "role": "gate",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "chemosensory_marker_expressing_population_alpha"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "focus_population_marker_calls",
            "kind": "list",
            "matchKey": "marker",
            "k": 3,
            "expected": [
              {
                "key": "POU2F3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "CHAT",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "TRPM5",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "LRMP",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNAT3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "AVIL",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNG13",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              }
            ]
          },
          {
            "name": "focus_population_condition_distribution_gate",
            "role": "gate",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_enriched_with_substantial_control_presence",
              "covid_predominant",
              "covid_only",
              "covid_enriched_trend_not_significant"
            ]
          },
          {
            "name": "focus_population_location_gate",
            "role": "gate",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "ectopic_parenchyma_in_covid",
              "parenchyma_only",
              "bronchiolized_or_peribronchial_parenchyma",
              "airway_and_parenchyma"
            ]
          },
          {
            "name": "focus_population_recovery_mode_gate",
            "role": "gate",
            "field": "$.focus_population_recovery_mode",
            "kind": "value",
            "op": "in",
            "expected": [
              "continuum_not_discrete_cluster"
            ]
          },
          {
            "name": "focus_population_identity_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "ionocyte_like_beta",
              "neuroendocrine_like_gamma",
              "basal_stem_progenitor_like_delta",
              "aberrant_basaloid_transitional_like_epsilon",
              "neuronal_or_neural_gene_minor_population_zeta",
              "deuterosomal_ciliated_progenitor_like_eta",
              "hillock_transitional_like_theta",
              "low_quality_doublet_or_debris",
              "no_rare_cell_recovered",
              "not_assessed"
            ]
          },
          {
            "name": "cf_wrong_lineage_marker_called_positive",
            "role": "hard_fail",
            "field": "$.focus_population_marker_calls[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "focus_population_condition_distribution_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "equally_distributed",
              "control_predominant",
              "not_assessed"
            ]
          },
          {
            "name": "focus_population_location_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "airway_only",
              "not_assessed"
            ]
          },
          {
            "name": "cross_axis_canonical_identity_with_airway_only_location_incoherence",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "focus_population_identity": {
                "op": "in",
                "values": [
                  "chemosensory_marker_expressing_population_alpha"
                ]
              },
              "focus_population_location": {
                "op": "in",
                "values": [
                  "airway_only"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-gemini-3-1-pro-preview-pi",
      "modelName": "gemini-3.1-pro-preview",
      "harness": "pi",
      "task": "gse171524_ectopic_airway_rare_cell",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 1630,
      "cost": 1.2404699999999997,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "focus_population_identity_gate",
            "role": "gate",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "chemosensory_marker_expressing_population_alpha"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "focus_population_marker_calls",
            "kind": "list",
            "matchKey": "marker",
            "k": 3,
            "expected": [
              {
                "key": "POU2F3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "CHAT",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "TRPM5",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "LRMP",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNAT3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "AVIL",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNG13",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              }
            ]
          },
          {
            "name": "focus_population_condition_distribution_gate",
            "role": "gate",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_enriched_with_substantial_control_presence",
              "covid_predominant",
              "covid_only",
              "covid_enriched_trend_not_significant"
            ]
          },
          {
            "name": "focus_population_location_gate",
            "role": "gate",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "ectopic_parenchyma_in_covid",
              "parenchyma_only",
              "bronchiolized_or_peribronchial_parenchyma",
              "airway_and_parenchyma"
            ]
          },
          {
            "name": "focus_population_recovery_mode_gate",
            "role": "gate",
            "field": "$.focus_population_recovery_mode",
            "kind": "value",
            "op": "in",
            "expected": [
              "continuum_not_discrete_cluster"
            ]
          },
          {
            "name": "focus_population_identity_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "ionocyte_like_beta",
              "neuroendocrine_like_gamma",
              "basal_stem_progenitor_like_delta",
              "aberrant_basaloid_transitional_like_epsilon",
              "neuronal_or_neural_gene_minor_population_zeta",
              "deuterosomal_ciliated_progenitor_like_eta",
              "hillock_transitional_like_theta",
              "low_quality_doublet_or_debris",
              "no_rare_cell_recovered",
              "not_assessed"
            ]
          },
          {
            "name": "cf_wrong_lineage_marker_called_positive",
            "role": "hard_fail",
            "field": "$.focus_population_marker_calls[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "focus_population_condition_distribution_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "equally_distributed",
              "control_predominant",
              "not_assessed"
            ]
          },
          {
            "name": "focus_population_location_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "airway_only",
              "not_assessed"
            ]
          },
          {
            "name": "cross_axis_canonical_identity_with_airway_only_location_incoherence",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "focus_population_identity": {
                "op": "in",
                "values": [
                  "chemosensory_marker_expressing_population_alpha"
                ]
              },
              "focus_population_location": {
                "op": "in",
                "values": [
                  "airway_only"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-gemini-3-1-pro-preview-pi",
      "modelName": "gemini-3.1-pro-preview",
      "harness": "pi",
      "task": "gse171524_ectopic_airway_rare_cell",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1404,
      "cost": 0.9069434000000001,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "focus_population_identity_gate",
            "role": "gate",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "chemosensory_marker_expressing_population_alpha"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "focus_population_marker_calls",
            "kind": "list",
            "matchKey": "marker",
            "k": 3,
            "expected": [
              {
                "key": "POU2F3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "CHAT",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "TRPM5",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "LRMP",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNAT3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "AVIL",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNG13",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              }
            ]
          },
          {
            "name": "focus_population_condition_distribution_gate",
            "role": "gate",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_enriched_with_substantial_control_presence",
              "covid_predominant",
              "covid_only",
              "covid_enriched_trend_not_significant"
            ]
          },
          {
            "name": "focus_population_location_gate",
            "role": "gate",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "ectopic_parenchyma_in_covid",
              "parenchyma_only",
              "bronchiolized_or_peribronchial_parenchyma",
              "airway_and_parenchyma"
            ]
          },
          {
            "name": "focus_population_recovery_mode_gate",
            "role": "gate",
            "field": "$.focus_population_recovery_mode",
            "kind": "value",
            "op": "in",
            "expected": [
              "continuum_not_discrete_cluster"
            ]
          },
          {
            "name": "focus_population_identity_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "ionocyte_like_beta",
              "neuroendocrine_like_gamma",
              "basal_stem_progenitor_like_delta",
              "aberrant_basaloid_transitional_like_epsilon",
              "neuronal_or_neural_gene_minor_population_zeta",
              "deuterosomal_ciliated_progenitor_like_eta",
              "hillock_transitional_like_theta",
              "low_quality_doublet_or_debris",
              "no_rare_cell_recovered",
              "not_assessed"
            ]
          },
          {
            "name": "cf_wrong_lineage_marker_called_positive",
            "role": "hard_fail",
            "field": "$.focus_population_marker_calls[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "focus_population_condition_distribution_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "equally_distributed",
              "control_predominant",
              "not_assessed"
            ]
          },
          {
            "name": "focus_population_location_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "airway_only",
              "not_assessed"
            ]
          },
          {
            "name": "cross_axis_canonical_identity_with_airway_only_location_incoherence",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "focus_population_identity": {
                "op": "in",
                "values": [
                  "chemosensory_marker_expressing_population_alpha"
                ]
              },
              "focus_population_location": {
                "op": "in",
                "values": [
                  "airway_only"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-gemini-3-5-flash-pi",
      "modelName": "gemini-3.5-flash",
      "harness": "pi",
      "task": "gse171524_ectopic_airway_rare_cell",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 1548,
      "cost": 1.1083519499999999,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "focus_population_identity_gate",
            "role": "gate",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "chemosensory_marker_expressing_population_alpha"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "focus_population_marker_calls",
            "kind": "list",
            "matchKey": "marker",
            "k": 3,
            "expected": [
              {
                "key": "POU2F3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "CHAT",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "TRPM5",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "LRMP",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNAT3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "AVIL",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNG13",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              }
            ]
          },
          {
            "name": "focus_population_condition_distribution_gate",
            "role": "gate",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_enriched_with_substantial_control_presence",
              "covid_predominant",
              "covid_only",
              "covid_enriched_trend_not_significant"
            ]
          },
          {
            "name": "focus_population_location_gate",
            "role": "gate",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "ectopic_parenchyma_in_covid",
              "parenchyma_only",
              "bronchiolized_or_peribronchial_parenchyma",
              "airway_and_parenchyma"
            ]
          },
          {
            "name": "focus_population_recovery_mode_gate",
            "role": "gate",
            "field": "$.focus_population_recovery_mode",
            "kind": "value",
            "op": "in",
            "expected": [
              "continuum_not_discrete_cluster"
            ]
          },
          {
            "name": "focus_population_identity_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "ionocyte_like_beta",
              "neuroendocrine_like_gamma",
              "basal_stem_progenitor_like_delta",
              "aberrant_basaloid_transitional_like_epsilon",
              "neuronal_or_neural_gene_minor_population_zeta",
              "deuterosomal_ciliated_progenitor_like_eta",
              "hillock_transitional_like_theta",
              "low_quality_doublet_or_debris",
              "no_rare_cell_recovered",
              "not_assessed"
            ]
          },
          {
            "name": "cf_wrong_lineage_marker_called_positive",
            "role": "hard_fail",
            "field": "$.focus_population_marker_calls[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "focus_population_condition_distribution_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "equally_distributed",
              "control_predominant",
              "not_assessed"
            ]
          },
          {
            "name": "focus_population_location_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "airway_only",
              "not_assessed"
            ]
          },
          {
            "name": "cross_axis_canonical_identity_with_airway_only_location_incoherence",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "focus_population_identity": {
                "op": "in",
                "values": [
                  "chemosensory_marker_expressing_population_alpha"
                ]
              },
              "focus_population_location": {
                "op": "in",
                "values": [
                  "airway_only"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-gemini-3-5-flash-pi",
      "modelName": "gemini-3.5-flash",
      "harness": "pi",
      "task": "gse171524_ectopic_airway_rare_cell",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 1256,
      "cost": 0.9535927500000001,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "focus_population_identity_gate",
            "role": "gate",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "chemosensory_marker_expressing_population_alpha"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "focus_population_marker_calls",
            "kind": "list",
            "matchKey": "marker",
            "k": 3,
            "expected": [
              {
                "key": "POU2F3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "CHAT",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "TRPM5",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "LRMP",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNAT3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "AVIL",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNG13",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              }
            ]
          },
          {
            "name": "focus_population_condition_distribution_gate",
            "role": "gate",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_enriched_with_substantial_control_presence",
              "covid_predominant",
              "covid_only",
              "covid_enriched_trend_not_significant"
            ]
          },
          {
            "name": "focus_population_location_gate",
            "role": "gate",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "ectopic_parenchyma_in_covid",
              "parenchyma_only",
              "bronchiolized_or_peribronchial_parenchyma",
              "airway_and_parenchyma"
            ]
          },
          {
            "name": "focus_population_recovery_mode_gate",
            "role": "gate",
            "field": "$.focus_population_recovery_mode",
            "kind": "value",
            "op": "in",
            "expected": [
              "continuum_not_discrete_cluster"
            ]
          },
          {
            "name": "focus_population_identity_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "ionocyte_like_beta",
              "neuroendocrine_like_gamma",
              "basal_stem_progenitor_like_delta",
              "aberrant_basaloid_transitional_like_epsilon",
              "neuronal_or_neural_gene_minor_population_zeta",
              "deuterosomal_ciliated_progenitor_like_eta",
              "hillock_transitional_like_theta",
              "low_quality_doublet_or_debris",
              "no_rare_cell_recovered",
              "not_assessed"
            ]
          },
          {
            "name": "cf_wrong_lineage_marker_called_positive",
            "role": "hard_fail",
            "field": "$.focus_population_marker_calls[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "focus_population_condition_distribution_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "equally_distributed",
              "control_predominant",
              "not_assessed"
            ]
          },
          {
            "name": "focus_population_location_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "airway_only",
              "not_assessed"
            ]
          },
          {
            "name": "cross_axis_canonical_identity_with_airway_only_location_incoherence",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "focus_population_identity": {
                "op": "in",
                "values": [
                  "chemosensory_marker_expressing_population_alpha"
                ]
              },
              "focus_population_location": {
                "op": "in",
                "values": [
                  "airway_only"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-gemini-3-5-flash-pi",
      "modelName": "gemini-3.5-flash",
      "harness": "pi",
      "task": "gse171524_ectopic_airway_rare_cell",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1652,
      "cost": 1.5108375,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "focus_population_identity_gate",
            "role": "gate",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "chemosensory_marker_expressing_population_alpha"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "focus_population_marker_calls",
            "kind": "list",
            "matchKey": "marker",
            "k": 3,
            "expected": [
              {
                "key": "POU2F3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "CHAT",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "TRPM5",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "LRMP",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNAT3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "AVIL",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNG13",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              }
            ]
          },
          {
            "name": "focus_population_condition_distribution_gate",
            "role": "gate",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_enriched_with_substantial_control_presence",
              "covid_predominant",
              "covid_only",
              "covid_enriched_trend_not_significant"
            ]
          },
          {
            "name": "focus_population_location_gate",
            "role": "gate",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "ectopic_parenchyma_in_covid",
              "parenchyma_only",
              "bronchiolized_or_peribronchial_parenchyma",
              "airway_and_parenchyma"
            ]
          },
          {
            "name": "focus_population_recovery_mode_gate",
            "role": "gate",
            "field": "$.focus_population_recovery_mode",
            "kind": "value",
            "op": "in",
            "expected": [
              "continuum_not_discrete_cluster"
            ]
          },
          {
            "name": "focus_population_identity_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "ionocyte_like_beta",
              "neuroendocrine_like_gamma",
              "basal_stem_progenitor_like_delta",
              "aberrant_basaloid_transitional_like_epsilon",
              "neuronal_or_neural_gene_minor_population_zeta",
              "deuterosomal_ciliated_progenitor_like_eta",
              "hillock_transitional_like_theta",
              "low_quality_doublet_or_debris",
              "no_rare_cell_recovered",
              "not_assessed"
            ]
          },
          {
            "name": "cf_wrong_lineage_marker_called_positive",
            "role": "hard_fail",
            "field": "$.focus_population_marker_calls[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "focus_population_condition_distribution_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "equally_distributed",
              "control_predominant",
              "not_assessed"
            ]
          },
          {
            "name": "focus_population_location_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "airway_only",
              "not_assessed"
            ]
          },
          {
            "name": "cross_axis_canonical_identity_with_airway_only_location_incoherence",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "focus_population_identity": {
                "op": "in",
                "values": [
                  "chemosensory_marker_expressing_population_alpha"
                ]
              },
              "focus_population_location": {
                "op": "in",
                "values": [
                  "airway_only"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-glm-5p2-pi",
      "modelName": "glm-5p2",
      "harness": "pi",
      "task": "gse171524_ectopic_airway_rare_cell",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 8304,
      "cost": 2.4453504599999993,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "focus_population_identity_gate",
            "role": "gate",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "chemosensory_marker_expressing_population_alpha"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "focus_population_marker_calls",
            "kind": "list",
            "matchKey": "marker",
            "k": 3,
            "expected": [
              {
                "key": "POU2F3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "CHAT",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "TRPM5",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "LRMP",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNAT3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "AVIL",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNG13",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              }
            ]
          },
          {
            "name": "focus_population_condition_distribution_gate",
            "role": "gate",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_enriched_with_substantial_control_presence",
              "covid_predominant",
              "covid_only",
              "covid_enriched_trend_not_significant"
            ]
          },
          {
            "name": "focus_population_location_gate",
            "role": "gate",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "ectopic_parenchyma_in_covid",
              "parenchyma_only",
              "bronchiolized_or_peribronchial_parenchyma",
              "airway_and_parenchyma"
            ]
          },
          {
            "name": "focus_population_recovery_mode_gate",
            "role": "gate",
            "field": "$.focus_population_recovery_mode",
            "kind": "value",
            "op": "in",
            "expected": [
              "continuum_not_discrete_cluster"
            ]
          },
          {
            "name": "focus_population_identity_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "ionocyte_like_beta",
              "neuroendocrine_like_gamma",
              "basal_stem_progenitor_like_delta",
              "aberrant_basaloid_transitional_like_epsilon",
              "neuronal_or_neural_gene_minor_population_zeta",
              "deuterosomal_ciliated_progenitor_like_eta",
              "hillock_transitional_like_theta",
              "low_quality_doublet_or_debris",
              "no_rare_cell_recovered",
              "not_assessed"
            ]
          },
          {
            "name": "cf_wrong_lineage_marker_called_positive",
            "role": "hard_fail",
            "field": "$.focus_population_marker_calls[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "focus_population_condition_distribution_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "equally_distributed",
              "control_predominant",
              "not_assessed"
            ]
          },
          {
            "name": "focus_population_location_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "airway_only",
              "not_assessed"
            ]
          },
          {
            "name": "cross_axis_canonical_identity_with_airway_only_location_incoherence",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "focus_population_identity": {
                "op": "in",
                "values": [
                  "chemosensory_marker_expressing_population_alpha"
                ]
              },
              "focus_population_location": {
                "op": "in",
                "values": [
                  "airway_only"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-glm-5p2-pi",
      "modelName": "glm-5p2",
      "harness": "pi",
      "task": "gse171524_ectopic_airway_rare_cell",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 4542,
      "cost": 4.2636542,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "focus_population_identity_gate",
            "role": "gate",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "chemosensory_marker_expressing_population_alpha"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "focus_population_marker_calls",
            "kind": "list",
            "matchKey": "marker",
            "k": 3,
            "expected": [
              {
                "key": "POU2F3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "CHAT",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "TRPM5",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "LRMP",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNAT3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "AVIL",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNG13",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              }
            ]
          },
          {
            "name": "focus_population_condition_distribution_gate",
            "role": "gate",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_enriched_with_substantial_control_presence",
              "covid_predominant",
              "covid_only",
              "covid_enriched_trend_not_significant"
            ]
          },
          {
            "name": "focus_population_location_gate",
            "role": "gate",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "ectopic_parenchyma_in_covid",
              "parenchyma_only",
              "bronchiolized_or_peribronchial_parenchyma",
              "airway_and_parenchyma"
            ]
          },
          {
            "name": "focus_population_recovery_mode_gate",
            "role": "gate",
            "field": "$.focus_population_recovery_mode",
            "kind": "value",
            "op": "in",
            "expected": [
              "continuum_not_discrete_cluster"
            ]
          },
          {
            "name": "focus_population_identity_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "ionocyte_like_beta",
              "neuroendocrine_like_gamma",
              "basal_stem_progenitor_like_delta",
              "aberrant_basaloid_transitional_like_epsilon",
              "neuronal_or_neural_gene_minor_population_zeta",
              "deuterosomal_ciliated_progenitor_like_eta",
              "hillock_transitional_like_theta",
              "low_quality_doublet_or_debris",
              "no_rare_cell_recovered",
              "not_assessed"
            ]
          },
          {
            "name": "cf_wrong_lineage_marker_called_positive",
            "role": "hard_fail",
            "field": "$.focus_population_marker_calls[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "focus_population_condition_distribution_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "equally_distributed",
              "control_predominant",
              "not_assessed"
            ]
          },
          {
            "name": "focus_population_location_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "airway_only",
              "not_assessed"
            ]
          },
          {
            "name": "cross_axis_canonical_identity_with_airway_only_location_incoherence",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "focus_population_identity": {
                "op": "in",
                "values": [
                  "chemosensory_marker_expressing_population_alpha"
                ]
              },
              "focus_population_location": {
                "op": "in",
                "values": [
                  "airway_only"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-glm-5p2-pi",
      "modelName": "glm-5p2",
      "harness": "pi",
      "task": "gse171524_ectopic_airway_rare_cell",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 3294,
      "cost": 1.9462377999999998,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "focus_population_identity_gate",
            "role": "gate",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "chemosensory_marker_expressing_population_alpha"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "focus_population_marker_calls",
            "kind": "list",
            "matchKey": "marker",
            "k": 3,
            "expected": [
              {
                "key": "POU2F3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "CHAT",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "TRPM5",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "LRMP",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNAT3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "AVIL",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNG13",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              }
            ]
          },
          {
            "name": "focus_population_condition_distribution_gate",
            "role": "gate",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_enriched_with_substantial_control_presence",
              "covid_predominant",
              "covid_only",
              "covid_enriched_trend_not_significant"
            ]
          },
          {
            "name": "focus_population_location_gate",
            "role": "gate",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "ectopic_parenchyma_in_covid",
              "parenchyma_only",
              "bronchiolized_or_peribronchial_parenchyma",
              "airway_and_parenchyma"
            ]
          },
          {
            "name": "focus_population_recovery_mode_gate",
            "role": "gate",
            "field": "$.focus_population_recovery_mode",
            "kind": "value",
            "op": "in",
            "expected": [
              "continuum_not_discrete_cluster"
            ]
          },
          {
            "name": "focus_population_identity_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "ionocyte_like_beta",
              "neuroendocrine_like_gamma",
              "basal_stem_progenitor_like_delta",
              "aberrant_basaloid_transitional_like_epsilon",
              "neuronal_or_neural_gene_minor_population_zeta",
              "deuterosomal_ciliated_progenitor_like_eta",
              "hillock_transitional_like_theta",
              "low_quality_doublet_or_debris",
              "no_rare_cell_recovered",
              "not_assessed"
            ]
          },
          {
            "name": "cf_wrong_lineage_marker_called_positive",
            "role": "hard_fail",
            "field": "$.focus_population_marker_calls[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "focus_population_condition_distribution_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "equally_distributed",
              "control_predominant",
              "not_assessed"
            ]
          },
          {
            "name": "focus_population_location_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "airway_only",
              "not_assessed"
            ]
          },
          {
            "name": "cross_axis_canonical_identity_with_airway_only_location_incoherence",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "focus_population_identity": {
                "op": "in",
                "values": [
                  "chemosensory_marker_expressing_population_alpha"
                ]
              },
              "focus_population_location": {
                "op": "in",
                "values": [
                  "airway_only"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-4-codex",
      "modelName": "gpt-5.4",
      "harness": "openai-codex",
      "task": "gse171524_ectopic_airway_rare_cell",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 1251,
      "cost": 2.8959205,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "focus_population_identity_gate",
            "role": "gate",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "chemosensory_marker_expressing_population_alpha"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "focus_population_marker_calls",
            "kind": "list",
            "matchKey": "marker",
            "k": 3,
            "expected": [
              {
                "key": "POU2F3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "CHAT",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "TRPM5",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "LRMP",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNAT3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "AVIL",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNG13",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              }
            ]
          },
          {
            "name": "focus_population_condition_distribution_gate",
            "role": "gate",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_enriched_with_substantial_control_presence",
              "covid_predominant",
              "covid_only",
              "covid_enriched_trend_not_significant"
            ]
          },
          {
            "name": "focus_population_location_gate",
            "role": "gate",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "ectopic_parenchyma_in_covid",
              "parenchyma_only",
              "bronchiolized_or_peribronchial_parenchyma",
              "airway_and_parenchyma"
            ]
          },
          {
            "name": "focus_population_recovery_mode_gate",
            "role": "gate",
            "field": "$.focus_population_recovery_mode",
            "kind": "value",
            "op": "in",
            "expected": [
              "continuum_not_discrete_cluster"
            ]
          },
          {
            "name": "focus_population_identity_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "ionocyte_like_beta",
              "neuroendocrine_like_gamma",
              "basal_stem_progenitor_like_delta",
              "aberrant_basaloid_transitional_like_epsilon",
              "neuronal_or_neural_gene_minor_population_zeta",
              "deuterosomal_ciliated_progenitor_like_eta",
              "hillock_transitional_like_theta",
              "low_quality_doublet_or_debris",
              "no_rare_cell_recovered",
              "not_assessed"
            ]
          },
          {
            "name": "cf_wrong_lineage_marker_called_positive",
            "role": "hard_fail",
            "field": "$.focus_population_marker_calls[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "focus_population_condition_distribution_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "equally_distributed",
              "control_predominant",
              "not_assessed"
            ]
          },
          {
            "name": "focus_population_location_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "airway_only",
              "not_assessed"
            ]
          },
          {
            "name": "cross_axis_canonical_identity_with_airway_only_location_incoherence",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "focus_population_identity": {
                "op": "in",
                "values": [
                  "chemosensory_marker_expressing_population_alpha"
                ]
              },
              "focus_population_location": {
                "op": "in",
                "values": [
                  "airway_only"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-4-codex",
      "modelName": "gpt-5.4",
      "harness": "openai-codex",
      "task": "gse171524_ectopic_airway_rare_cell",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 655,
      "cost": 2.0362495000000003,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "focus_population_identity_gate",
            "role": "gate",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "chemosensory_marker_expressing_population_alpha"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "focus_population_marker_calls",
            "kind": "list",
            "matchKey": "marker",
            "k": 3,
            "expected": [
              {
                "key": "POU2F3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "CHAT",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "TRPM5",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "LRMP",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNAT3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "AVIL",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNG13",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              }
            ]
          },
          {
            "name": "focus_population_condition_distribution_gate",
            "role": "gate",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_enriched_with_substantial_control_presence",
              "covid_predominant",
              "covid_only",
              "covid_enriched_trend_not_significant"
            ]
          },
          {
            "name": "focus_population_location_gate",
            "role": "gate",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "ectopic_parenchyma_in_covid",
              "parenchyma_only",
              "bronchiolized_or_peribronchial_parenchyma",
              "airway_and_parenchyma"
            ]
          },
          {
            "name": "focus_population_recovery_mode_gate",
            "role": "gate",
            "field": "$.focus_population_recovery_mode",
            "kind": "value",
            "op": "in",
            "expected": [
              "continuum_not_discrete_cluster"
            ]
          },
          {
            "name": "focus_population_identity_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "ionocyte_like_beta",
              "neuroendocrine_like_gamma",
              "basal_stem_progenitor_like_delta",
              "aberrant_basaloid_transitional_like_epsilon",
              "neuronal_or_neural_gene_minor_population_zeta",
              "deuterosomal_ciliated_progenitor_like_eta",
              "hillock_transitional_like_theta",
              "low_quality_doublet_or_debris",
              "no_rare_cell_recovered",
              "not_assessed"
            ]
          },
          {
            "name": "cf_wrong_lineage_marker_called_positive",
            "role": "hard_fail",
            "field": "$.focus_population_marker_calls[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "focus_population_condition_distribution_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "equally_distributed",
              "control_predominant",
              "not_assessed"
            ]
          },
          {
            "name": "focus_population_location_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "airway_only",
              "not_assessed"
            ]
          },
          {
            "name": "cross_axis_canonical_identity_with_airway_only_location_incoherence",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "focus_population_identity": {
                "op": "in",
                "values": [
                  "chemosensory_marker_expressing_population_alpha"
                ]
              },
              "focus_population_location": {
                "op": "in",
                "values": [
                  "airway_only"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-4-codex",
      "modelName": "gpt-5.4",
      "harness": "openai-codex",
      "task": "gse171524_ectopic_airway_rare_cell",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1016,
      "cost": 2.598314,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "focus_population_identity_gate",
            "role": "gate",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "chemosensory_marker_expressing_population_alpha"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "focus_population_marker_calls",
            "kind": "list",
            "matchKey": "marker",
            "k": 3,
            "expected": [
              {
                "key": "POU2F3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "CHAT",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "TRPM5",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "LRMP",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNAT3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "AVIL",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNG13",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              }
            ]
          },
          {
            "name": "focus_population_condition_distribution_gate",
            "role": "gate",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_enriched_with_substantial_control_presence",
              "covid_predominant",
              "covid_only",
              "covid_enriched_trend_not_significant"
            ]
          },
          {
            "name": "focus_population_location_gate",
            "role": "gate",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "ectopic_parenchyma_in_covid",
              "parenchyma_only",
              "bronchiolized_or_peribronchial_parenchyma",
              "airway_and_parenchyma"
            ]
          },
          {
            "name": "focus_population_recovery_mode_gate",
            "role": "gate",
            "field": "$.focus_population_recovery_mode",
            "kind": "value",
            "op": "in",
            "expected": [
              "continuum_not_discrete_cluster"
            ]
          },
          {
            "name": "focus_population_identity_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "ionocyte_like_beta",
              "neuroendocrine_like_gamma",
              "basal_stem_progenitor_like_delta",
              "aberrant_basaloid_transitional_like_epsilon",
              "neuronal_or_neural_gene_minor_population_zeta",
              "deuterosomal_ciliated_progenitor_like_eta",
              "hillock_transitional_like_theta",
              "low_quality_doublet_or_debris",
              "no_rare_cell_recovered",
              "not_assessed"
            ]
          },
          {
            "name": "cf_wrong_lineage_marker_called_positive",
            "role": "hard_fail",
            "field": "$.focus_population_marker_calls[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "focus_population_condition_distribution_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "equally_distributed",
              "control_predominant",
              "not_assessed"
            ]
          },
          {
            "name": "focus_population_location_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "airway_only",
              "not_assessed"
            ]
          },
          {
            "name": "cross_axis_canonical_identity_with_airway_only_location_incoherence",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "focus_population_identity": {
                "op": "in",
                "values": [
                  "chemosensory_marker_expressing_population_alpha"
                ]
              },
              "focus_population_location": {
                "op": "in",
                "values": [
                  "airway_only"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-4-pi",
      "modelName": "gpt-5.4",
      "harness": "pi",
      "task": "gse171524_ectopic_airway_rare_cell",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 912,
      "cost": 0.979848,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "focus_population_identity_gate",
            "role": "gate",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "chemosensory_marker_expressing_population_alpha"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "focus_population_marker_calls",
            "kind": "list",
            "matchKey": "marker",
            "k": 3,
            "expected": [
              {
                "key": "POU2F3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "CHAT",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "TRPM5",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "LRMP",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNAT3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "AVIL",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNG13",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              }
            ]
          },
          {
            "name": "focus_population_condition_distribution_gate",
            "role": "gate",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_enriched_with_substantial_control_presence",
              "covid_predominant",
              "covid_only",
              "covid_enriched_trend_not_significant"
            ]
          },
          {
            "name": "focus_population_location_gate",
            "role": "gate",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "ectopic_parenchyma_in_covid",
              "parenchyma_only",
              "bronchiolized_or_peribronchial_parenchyma",
              "airway_and_parenchyma"
            ]
          },
          {
            "name": "focus_population_recovery_mode_gate",
            "role": "gate",
            "field": "$.focus_population_recovery_mode",
            "kind": "value",
            "op": "in",
            "expected": [
              "continuum_not_discrete_cluster"
            ]
          },
          {
            "name": "focus_population_identity_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "ionocyte_like_beta",
              "neuroendocrine_like_gamma",
              "basal_stem_progenitor_like_delta",
              "aberrant_basaloid_transitional_like_epsilon",
              "neuronal_or_neural_gene_minor_population_zeta",
              "deuterosomal_ciliated_progenitor_like_eta",
              "hillock_transitional_like_theta",
              "low_quality_doublet_or_debris",
              "no_rare_cell_recovered",
              "not_assessed"
            ]
          },
          {
            "name": "cf_wrong_lineage_marker_called_positive",
            "role": "hard_fail",
            "field": "$.focus_population_marker_calls[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "focus_population_condition_distribution_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "equally_distributed",
              "control_predominant",
              "not_assessed"
            ]
          },
          {
            "name": "focus_population_location_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "airway_only",
              "not_assessed"
            ]
          },
          {
            "name": "cross_axis_canonical_identity_with_airway_only_location_incoherence",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "focus_population_identity": {
                "op": "in",
                "values": [
                  "chemosensory_marker_expressing_population_alpha"
                ]
              },
              "focus_population_location": {
                "op": "in",
                "values": [
                  "airway_only"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-4-pi",
      "modelName": "gpt-5.4",
      "harness": "pi",
      "task": "gse171524_ectopic_airway_rare_cell",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 1165,
      "cost": 1.132043,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "focus_population_identity_gate",
            "role": "gate",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "chemosensory_marker_expressing_population_alpha"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "focus_population_marker_calls",
            "kind": "list",
            "matchKey": "marker",
            "k": 3,
            "expected": [
              {
                "key": "POU2F3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "CHAT",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "TRPM5",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "LRMP",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNAT3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "AVIL",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNG13",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              }
            ]
          },
          {
            "name": "focus_population_condition_distribution_gate",
            "role": "gate",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_enriched_with_substantial_control_presence",
              "covid_predominant",
              "covid_only",
              "covid_enriched_trend_not_significant"
            ]
          },
          {
            "name": "focus_population_location_gate",
            "role": "gate",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "ectopic_parenchyma_in_covid",
              "parenchyma_only",
              "bronchiolized_or_peribronchial_parenchyma",
              "airway_and_parenchyma"
            ]
          },
          {
            "name": "focus_population_recovery_mode_gate",
            "role": "gate",
            "field": "$.focus_population_recovery_mode",
            "kind": "value",
            "op": "in",
            "expected": [
              "continuum_not_discrete_cluster"
            ]
          },
          {
            "name": "focus_population_identity_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "ionocyte_like_beta",
              "neuroendocrine_like_gamma",
              "basal_stem_progenitor_like_delta",
              "aberrant_basaloid_transitional_like_epsilon",
              "neuronal_or_neural_gene_minor_population_zeta",
              "deuterosomal_ciliated_progenitor_like_eta",
              "hillock_transitional_like_theta",
              "low_quality_doublet_or_debris",
              "no_rare_cell_recovered",
              "not_assessed"
            ]
          },
          {
            "name": "cf_wrong_lineage_marker_called_positive",
            "role": "hard_fail",
            "field": "$.focus_population_marker_calls[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "focus_population_condition_distribution_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "equally_distributed",
              "control_predominant",
              "not_assessed"
            ]
          },
          {
            "name": "focus_population_location_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "airway_only",
              "not_assessed"
            ]
          },
          {
            "name": "cross_axis_canonical_identity_with_airway_only_location_incoherence",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "focus_population_identity": {
                "op": "in",
                "values": [
                  "chemosensory_marker_expressing_population_alpha"
                ]
              },
              "focus_population_location": {
                "op": "in",
                "values": [
                  "airway_only"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-4-pi",
      "modelName": "gpt-5.4",
      "harness": "pi",
      "task": "gse171524_ectopic_airway_rare_cell",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 3403,
      "cost": 1.6059785000000004,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "focus_population_identity_gate",
            "role": "gate",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "chemosensory_marker_expressing_population_alpha"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "focus_population_marker_calls",
            "kind": "list",
            "matchKey": "marker",
            "k": 3,
            "expected": [
              {
                "key": "POU2F3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "CHAT",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "TRPM5",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "LRMP",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNAT3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "AVIL",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNG13",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              }
            ]
          },
          {
            "name": "focus_population_condition_distribution_gate",
            "role": "gate",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_enriched_with_substantial_control_presence",
              "covid_predominant",
              "covid_only",
              "covid_enriched_trend_not_significant"
            ]
          },
          {
            "name": "focus_population_location_gate",
            "role": "gate",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "ectopic_parenchyma_in_covid",
              "parenchyma_only",
              "bronchiolized_or_peribronchial_parenchyma",
              "airway_and_parenchyma"
            ]
          },
          {
            "name": "focus_population_recovery_mode_gate",
            "role": "gate",
            "field": "$.focus_population_recovery_mode",
            "kind": "value",
            "op": "in",
            "expected": [
              "continuum_not_discrete_cluster"
            ]
          },
          {
            "name": "focus_population_identity_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "ionocyte_like_beta",
              "neuroendocrine_like_gamma",
              "basal_stem_progenitor_like_delta",
              "aberrant_basaloid_transitional_like_epsilon",
              "neuronal_or_neural_gene_minor_population_zeta",
              "deuterosomal_ciliated_progenitor_like_eta",
              "hillock_transitional_like_theta",
              "low_quality_doublet_or_debris",
              "no_rare_cell_recovered",
              "not_assessed"
            ]
          },
          {
            "name": "cf_wrong_lineage_marker_called_positive",
            "role": "hard_fail",
            "field": "$.focus_population_marker_calls[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "focus_population_condition_distribution_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "equally_distributed",
              "control_predominant",
              "not_assessed"
            ]
          },
          {
            "name": "focus_population_location_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "airway_only",
              "not_assessed"
            ]
          },
          {
            "name": "cross_axis_canonical_identity_with_airway_only_location_incoherence",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "focus_population_identity": {
                "op": "in",
                "values": [
                  "chemosensory_marker_expressing_population_alpha"
                ]
              },
              "focus_population_location": {
                "op": "in",
                "values": [
                  "airway_only"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-5-codex",
      "modelName": "gpt-5.5",
      "harness": "openai-codex",
      "task": "gse171524_ectopic_airway_rare_cell",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 1,
      "passed": true,
      "score": null,
      "durationS": 662,
      "cost": 2.832764,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "focus_population_identity_gate",
            "role": "gate",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "chemosensory_marker_expressing_population_alpha"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "focus_population_marker_calls",
            "kind": "list",
            "matchKey": "marker",
            "k": 3,
            "expected": [
              {
                "key": "POU2F3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "CHAT",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "TRPM5",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "LRMP",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNAT3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "AVIL",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNG13",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              }
            ]
          },
          {
            "name": "focus_population_condition_distribution_gate",
            "role": "gate",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_enriched_with_substantial_control_presence",
              "covid_predominant",
              "covid_only",
              "covid_enriched_trend_not_significant"
            ]
          },
          {
            "name": "focus_population_location_gate",
            "role": "gate",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "ectopic_parenchyma_in_covid",
              "parenchyma_only",
              "bronchiolized_or_peribronchial_parenchyma",
              "airway_and_parenchyma"
            ]
          },
          {
            "name": "focus_population_recovery_mode_gate",
            "role": "gate",
            "field": "$.focus_population_recovery_mode",
            "kind": "value",
            "op": "in",
            "expected": [
              "continuum_not_discrete_cluster"
            ]
          },
          {
            "name": "focus_population_identity_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "ionocyte_like_beta",
              "neuroendocrine_like_gamma",
              "basal_stem_progenitor_like_delta",
              "aberrant_basaloid_transitional_like_epsilon",
              "neuronal_or_neural_gene_minor_population_zeta",
              "deuterosomal_ciliated_progenitor_like_eta",
              "hillock_transitional_like_theta",
              "low_quality_doublet_or_debris",
              "no_rare_cell_recovered",
              "not_assessed"
            ]
          },
          {
            "name": "cf_wrong_lineage_marker_called_positive",
            "role": "hard_fail",
            "field": "$.focus_population_marker_calls[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "focus_population_condition_distribution_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "equally_distributed",
              "control_predominant",
              "not_assessed"
            ]
          },
          {
            "name": "focus_population_location_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "airway_only",
              "not_assessed"
            ]
          },
          {
            "name": "cross_axis_canonical_identity_with_airway_only_location_incoherence",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "focus_population_identity": {
                "op": "in",
                "values": [
                  "chemosensory_marker_expressing_population_alpha"
                ]
              },
              "focus_population_location": {
                "op": "in",
                "values": [
                  "airway_only"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-5-codex",
      "modelName": "gpt-5.5",
      "harness": "openai-codex",
      "task": "gse171524_ectopic_airway_rare_cell",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 1192,
      "cost": 2.687793,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "focus_population_identity_gate",
            "role": "gate",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "chemosensory_marker_expressing_population_alpha"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "focus_population_marker_calls",
            "kind": "list",
            "matchKey": "marker",
            "k": 3,
            "expected": [
              {
                "key": "POU2F3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "CHAT",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "TRPM5",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "LRMP",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNAT3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "AVIL",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNG13",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              }
            ]
          },
          {
            "name": "focus_population_condition_distribution_gate",
            "role": "gate",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_enriched_with_substantial_control_presence",
              "covid_predominant",
              "covid_only",
              "covid_enriched_trend_not_significant"
            ]
          },
          {
            "name": "focus_population_location_gate",
            "role": "gate",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "ectopic_parenchyma_in_covid",
              "parenchyma_only",
              "bronchiolized_or_peribronchial_parenchyma",
              "airway_and_parenchyma"
            ]
          },
          {
            "name": "focus_population_recovery_mode_gate",
            "role": "gate",
            "field": "$.focus_population_recovery_mode",
            "kind": "value",
            "op": "in",
            "expected": [
              "continuum_not_discrete_cluster"
            ]
          },
          {
            "name": "focus_population_identity_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "ionocyte_like_beta",
              "neuroendocrine_like_gamma",
              "basal_stem_progenitor_like_delta",
              "aberrant_basaloid_transitional_like_epsilon",
              "neuronal_or_neural_gene_minor_population_zeta",
              "deuterosomal_ciliated_progenitor_like_eta",
              "hillock_transitional_like_theta",
              "low_quality_doublet_or_debris",
              "no_rare_cell_recovered",
              "not_assessed"
            ]
          },
          {
            "name": "cf_wrong_lineage_marker_called_positive",
            "role": "hard_fail",
            "field": "$.focus_population_marker_calls[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "focus_population_condition_distribution_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "equally_distributed",
              "control_predominant",
              "not_assessed"
            ]
          },
          {
            "name": "focus_population_location_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "airway_only",
              "not_assessed"
            ]
          },
          {
            "name": "cross_axis_canonical_identity_with_airway_only_location_incoherence",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "focus_population_identity": {
                "op": "in",
                "values": [
                  "chemosensory_marker_expressing_population_alpha"
                ]
              },
              "focus_population_location": {
                "op": "in",
                "values": [
                  "airway_only"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-5-codex",
      "modelName": "gpt-5.5",
      "harness": "openai-codex",
      "task": "gse171524_ectopic_airway_rare_cell",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 704,
      "cost": 1.9617980000000004,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "focus_population_identity_gate",
            "role": "gate",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "chemosensory_marker_expressing_population_alpha"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "focus_population_marker_calls",
            "kind": "list",
            "matchKey": "marker",
            "k": 3,
            "expected": [
              {
                "key": "POU2F3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "CHAT",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "TRPM5",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "LRMP",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNAT3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "AVIL",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNG13",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              }
            ]
          },
          {
            "name": "focus_population_condition_distribution_gate",
            "role": "gate",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_enriched_with_substantial_control_presence",
              "covid_predominant",
              "covid_only",
              "covid_enriched_trend_not_significant"
            ]
          },
          {
            "name": "focus_population_location_gate",
            "role": "gate",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "ectopic_parenchyma_in_covid",
              "parenchyma_only",
              "bronchiolized_or_peribronchial_parenchyma",
              "airway_and_parenchyma"
            ]
          },
          {
            "name": "focus_population_recovery_mode_gate",
            "role": "gate",
            "field": "$.focus_population_recovery_mode",
            "kind": "value",
            "op": "in",
            "expected": [
              "continuum_not_discrete_cluster"
            ]
          },
          {
            "name": "focus_population_identity_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "ionocyte_like_beta",
              "neuroendocrine_like_gamma",
              "basal_stem_progenitor_like_delta",
              "aberrant_basaloid_transitional_like_epsilon",
              "neuronal_or_neural_gene_minor_population_zeta",
              "deuterosomal_ciliated_progenitor_like_eta",
              "hillock_transitional_like_theta",
              "low_quality_doublet_or_debris",
              "no_rare_cell_recovered",
              "not_assessed"
            ]
          },
          {
            "name": "cf_wrong_lineage_marker_called_positive",
            "role": "hard_fail",
            "field": "$.focus_population_marker_calls[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "focus_population_condition_distribution_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "equally_distributed",
              "control_predominant",
              "not_assessed"
            ]
          },
          {
            "name": "focus_population_location_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "airway_only",
              "not_assessed"
            ]
          },
          {
            "name": "cross_axis_canonical_identity_with_airway_only_location_incoherence",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "focus_population_identity": {
                "op": "in",
                "values": [
                  "chemosensory_marker_expressing_population_alpha"
                ]
              },
              "focus_population_location": {
                "op": "in",
                "values": [
                  "airway_only"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-5-pi",
      "modelName": "gpt-5.5",
      "harness": "pi",
      "task": "gse171524_ectopic_airway_rare_cell",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 2078,
      "cost": 3.1449300000000004,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "focus_population_identity_gate",
            "role": "gate",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "chemosensory_marker_expressing_population_alpha"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "focus_population_marker_calls",
            "kind": "list",
            "matchKey": "marker",
            "k": 3,
            "expected": [
              {
                "key": "POU2F3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "CHAT",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "TRPM5",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "LRMP",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNAT3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "AVIL",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNG13",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              }
            ]
          },
          {
            "name": "focus_population_condition_distribution_gate",
            "role": "gate",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_enriched_with_substantial_control_presence",
              "covid_predominant",
              "covid_only",
              "covid_enriched_trend_not_significant"
            ]
          },
          {
            "name": "focus_population_location_gate",
            "role": "gate",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "ectopic_parenchyma_in_covid",
              "parenchyma_only",
              "bronchiolized_or_peribronchial_parenchyma",
              "airway_and_parenchyma"
            ]
          },
          {
            "name": "focus_population_recovery_mode_gate",
            "role": "gate",
            "field": "$.focus_population_recovery_mode",
            "kind": "value",
            "op": "in",
            "expected": [
              "continuum_not_discrete_cluster"
            ]
          },
          {
            "name": "focus_population_identity_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "ionocyte_like_beta",
              "neuroendocrine_like_gamma",
              "basal_stem_progenitor_like_delta",
              "aberrant_basaloid_transitional_like_epsilon",
              "neuronal_or_neural_gene_minor_population_zeta",
              "deuterosomal_ciliated_progenitor_like_eta",
              "hillock_transitional_like_theta",
              "low_quality_doublet_or_debris",
              "no_rare_cell_recovered",
              "not_assessed"
            ]
          },
          {
            "name": "cf_wrong_lineage_marker_called_positive",
            "role": "hard_fail",
            "field": "$.focus_population_marker_calls[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "focus_population_condition_distribution_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "equally_distributed",
              "control_predominant",
              "not_assessed"
            ]
          },
          {
            "name": "focus_population_location_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "airway_only",
              "not_assessed"
            ]
          },
          {
            "name": "cross_axis_canonical_identity_with_airway_only_location_incoherence",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "focus_population_identity": {
                "op": "in",
                "values": [
                  "chemosensory_marker_expressing_population_alpha"
                ]
              },
              "focus_population_location": {
                "op": "in",
                "values": [
                  "airway_only"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-5-pi",
      "modelName": "gpt-5.5",
      "harness": "pi",
      "task": "gse171524_ectopic_airway_rare_cell",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 2262,
      "cost": 2.8362470000000006,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "focus_population_identity_gate",
            "role": "gate",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "chemosensory_marker_expressing_population_alpha"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "focus_population_marker_calls",
            "kind": "list",
            "matchKey": "marker",
            "k": 3,
            "expected": [
              {
                "key": "POU2F3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "CHAT",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "TRPM5",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "LRMP",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNAT3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "AVIL",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNG13",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              }
            ]
          },
          {
            "name": "focus_population_condition_distribution_gate",
            "role": "gate",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_enriched_with_substantial_control_presence",
              "covid_predominant",
              "covid_only",
              "covid_enriched_trend_not_significant"
            ]
          },
          {
            "name": "focus_population_location_gate",
            "role": "gate",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "ectopic_parenchyma_in_covid",
              "parenchyma_only",
              "bronchiolized_or_peribronchial_parenchyma",
              "airway_and_parenchyma"
            ]
          },
          {
            "name": "focus_population_recovery_mode_gate",
            "role": "gate",
            "field": "$.focus_population_recovery_mode",
            "kind": "value",
            "op": "in",
            "expected": [
              "continuum_not_discrete_cluster"
            ]
          },
          {
            "name": "focus_population_identity_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "ionocyte_like_beta",
              "neuroendocrine_like_gamma",
              "basal_stem_progenitor_like_delta",
              "aberrant_basaloid_transitional_like_epsilon",
              "neuronal_or_neural_gene_minor_population_zeta",
              "deuterosomal_ciliated_progenitor_like_eta",
              "hillock_transitional_like_theta",
              "low_quality_doublet_or_debris",
              "no_rare_cell_recovered",
              "not_assessed"
            ]
          },
          {
            "name": "cf_wrong_lineage_marker_called_positive",
            "role": "hard_fail",
            "field": "$.focus_population_marker_calls[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "focus_population_condition_distribution_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "equally_distributed",
              "control_predominant",
              "not_assessed"
            ]
          },
          {
            "name": "focus_population_location_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "airway_only",
              "not_assessed"
            ]
          },
          {
            "name": "cross_axis_canonical_identity_with_airway_only_location_incoherence",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "focus_population_identity": {
                "op": "in",
                "values": [
                  "chemosensory_marker_expressing_population_alpha"
                ]
              },
              "focus_population_location": {
                "op": "in",
                "values": [
                  "airway_only"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-5-pi",
      "modelName": "gpt-5.5",
      "harness": "pi",
      "task": "gse171524_ectopic_airway_rare_cell",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1942,
      "cost": 2.237872,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "focus_population_identity_gate",
            "role": "gate",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "chemosensory_marker_expressing_population_alpha"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "focus_population_marker_calls",
            "kind": "list",
            "matchKey": "marker",
            "k": 3,
            "expected": [
              {
                "key": "POU2F3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "CHAT",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "TRPM5",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "LRMP",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNAT3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "AVIL",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNG13",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              }
            ]
          },
          {
            "name": "focus_population_condition_distribution_gate",
            "role": "gate",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_enriched_with_substantial_control_presence",
              "covid_predominant",
              "covid_only",
              "covid_enriched_trend_not_significant"
            ]
          },
          {
            "name": "focus_population_location_gate",
            "role": "gate",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "ectopic_parenchyma_in_covid",
              "parenchyma_only",
              "bronchiolized_or_peribronchial_parenchyma",
              "airway_and_parenchyma"
            ]
          },
          {
            "name": "focus_population_recovery_mode_gate",
            "role": "gate",
            "field": "$.focus_population_recovery_mode",
            "kind": "value",
            "op": "in",
            "expected": [
              "continuum_not_discrete_cluster"
            ]
          },
          {
            "name": "focus_population_identity_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "ionocyte_like_beta",
              "neuroendocrine_like_gamma",
              "basal_stem_progenitor_like_delta",
              "aberrant_basaloid_transitional_like_epsilon",
              "neuronal_or_neural_gene_minor_population_zeta",
              "deuterosomal_ciliated_progenitor_like_eta",
              "hillock_transitional_like_theta",
              "low_quality_doublet_or_debris",
              "no_rare_cell_recovered",
              "not_assessed"
            ]
          },
          {
            "name": "cf_wrong_lineage_marker_called_positive",
            "role": "hard_fail",
            "field": "$.focus_population_marker_calls[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "focus_population_condition_distribution_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "equally_distributed",
              "control_predominant",
              "not_assessed"
            ]
          },
          {
            "name": "focus_population_location_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "airway_only",
              "not_assessed"
            ]
          },
          {
            "name": "cross_axis_canonical_identity_with_airway_only_location_incoherence",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "focus_population_identity": {
                "op": "in",
                "values": [
                  "chemosensory_marker_expressing_population_alpha"
                ]
              },
              "focus_population_location": {
                "op": "in",
                "values": [
                  "airway_only"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-20-beta-reasoning-pi",
      "modelName": "grok-4.20-beta-0309-reasoning",
      "harness": "pi",
      "task": "gse171524_ectopic_airway_rare_cell",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 517,
      "cost": 0.12412150000000001,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "focus_population_identity_gate",
            "role": "gate",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "chemosensory_marker_expressing_population_alpha"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "focus_population_marker_calls",
            "kind": "list",
            "matchKey": "marker",
            "k": 3,
            "expected": [
              {
                "key": "POU2F3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "CHAT",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "TRPM5",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "LRMP",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNAT3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "AVIL",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNG13",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              }
            ]
          },
          {
            "name": "focus_population_condition_distribution_gate",
            "role": "gate",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_enriched_with_substantial_control_presence",
              "covid_predominant",
              "covid_only",
              "covid_enriched_trend_not_significant"
            ]
          },
          {
            "name": "focus_population_location_gate",
            "role": "gate",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "ectopic_parenchyma_in_covid",
              "parenchyma_only",
              "bronchiolized_or_peribronchial_parenchyma",
              "airway_and_parenchyma"
            ]
          },
          {
            "name": "focus_population_recovery_mode_gate",
            "role": "gate",
            "field": "$.focus_population_recovery_mode",
            "kind": "value",
            "op": "in",
            "expected": [
              "continuum_not_discrete_cluster"
            ]
          },
          {
            "name": "focus_population_identity_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "ionocyte_like_beta",
              "neuroendocrine_like_gamma",
              "basal_stem_progenitor_like_delta",
              "aberrant_basaloid_transitional_like_epsilon",
              "neuronal_or_neural_gene_minor_population_zeta",
              "deuterosomal_ciliated_progenitor_like_eta",
              "hillock_transitional_like_theta",
              "low_quality_doublet_or_debris",
              "no_rare_cell_recovered",
              "not_assessed"
            ]
          },
          {
            "name": "cf_wrong_lineage_marker_called_positive",
            "role": "hard_fail",
            "field": "$.focus_population_marker_calls[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "focus_population_condition_distribution_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "equally_distributed",
              "control_predominant",
              "not_assessed"
            ]
          },
          {
            "name": "focus_population_location_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "airway_only",
              "not_assessed"
            ]
          },
          {
            "name": "cross_axis_canonical_identity_with_airway_only_location_incoherence",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "focus_population_identity": {
                "op": "in",
                "values": [
                  "chemosensory_marker_expressing_population_alpha"
                ]
              },
              "focus_population_location": {
                "op": "in",
                "values": [
                  "airway_only"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-20-beta-reasoning-pi",
      "modelName": "grok-4.20-beta-0309-reasoning",
      "harness": "pi",
      "task": "gse171524_ectopic_airway_rare_cell",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 479,
      "cost": 0.06510125,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "focus_population_identity_gate",
            "role": "gate",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "chemosensory_marker_expressing_population_alpha"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "focus_population_marker_calls",
            "kind": "list",
            "matchKey": "marker",
            "k": 3,
            "expected": [
              {
                "key": "POU2F3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "CHAT",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "TRPM5",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "LRMP",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNAT3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "AVIL",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNG13",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              }
            ]
          },
          {
            "name": "focus_population_condition_distribution_gate",
            "role": "gate",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_enriched_with_substantial_control_presence",
              "covid_predominant",
              "covid_only",
              "covid_enriched_trend_not_significant"
            ]
          },
          {
            "name": "focus_population_location_gate",
            "role": "gate",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "ectopic_parenchyma_in_covid",
              "parenchyma_only",
              "bronchiolized_or_peribronchial_parenchyma",
              "airway_and_parenchyma"
            ]
          },
          {
            "name": "focus_population_recovery_mode_gate",
            "role": "gate",
            "field": "$.focus_population_recovery_mode",
            "kind": "value",
            "op": "in",
            "expected": [
              "continuum_not_discrete_cluster"
            ]
          },
          {
            "name": "focus_population_identity_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "ionocyte_like_beta",
              "neuroendocrine_like_gamma",
              "basal_stem_progenitor_like_delta",
              "aberrant_basaloid_transitional_like_epsilon",
              "neuronal_or_neural_gene_minor_population_zeta",
              "deuterosomal_ciliated_progenitor_like_eta",
              "hillock_transitional_like_theta",
              "low_quality_doublet_or_debris",
              "no_rare_cell_recovered",
              "not_assessed"
            ]
          },
          {
            "name": "cf_wrong_lineage_marker_called_positive",
            "role": "hard_fail",
            "field": "$.focus_population_marker_calls[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "focus_population_condition_distribution_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "equally_distributed",
              "control_predominant",
              "not_assessed"
            ]
          },
          {
            "name": "focus_population_location_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "airway_only",
              "not_assessed"
            ]
          },
          {
            "name": "cross_axis_canonical_identity_with_airway_only_location_incoherence",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "focus_population_identity": {
                "op": "in",
                "values": [
                  "chemosensory_marker_expressing_population_alpha"
                ]
              },
              "focus_population_location": {
                "op": "in",
                "values": [
                  "airway_only"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-20-beta-reasoning-pi",
      "modelName": "grok-4.20-beta-0309-reasoning",
      "harness": "pi",
      "task": "gse171524_ectopic_airway_rare_cell",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1405,
      "cost": 0.04439785,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "focus_population_identity_gate",
            "role": "gate",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "chemosensory_marker_expressing_population_alpha"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "focus_population_marker_calls",
            "kind": "list",
            "matchKey": "marker",
            "k": 3,
            "expected": [
              {
                "key": "POU2F3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "CHAT",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "TRPM5",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "LRMP",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNAT3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "AVIL",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNG13",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              }
            ]
          },
          {
            "name": "focus_population_condition_distribution_gate",
            "role": "gate",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_enriched_with_substantial_control_presence",
              "covid_predominant",
              "covid_only",
              "covid_enriched_trend_not_significant"
            ]
          },
          {
            "name": "focus_population_location_gate",
            "role": "gate",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "ectopic_parenchyma_in_covid",
              "parenchyma_only",
              "bronchiolized_or_peribronchial_parenchyma",
              "airway_and_parenchyma"
            ]
          },
          {
            "name": "focus_population_recovery_mode_gate",
            "role": "gate",
            "field": "$.focus_population_recovery_mode",
            "kind": "value",
            "op": "in",
            "expected": [
              "continuum_not_discrete_cluster"
            ]
          },
          {
            "name": "focus_population_identity_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "ionocyte_like_beta",
              "neuroendocrine_like_gamma",
              "basal_stem_progenitor_like_delta",
              "aberrant_basaloid_transitional_like_epsilon",
              "neuronal_or_neural_gene_minor_population_zeta",
              "deuterosomal_ciliated_progenitor_like_eta",
              "hillock_transitional_like_theta",
              "low_quality_doublet_or_debris",
              "no_rare_cell_recovered",
              "not_assessed"
            ]
          },
          {
            "name": "cf_wrong_lineage_marker_called_positive",
            "role": "hard_fail",
            "field": "$.focus_population_marker_calls[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "focus_population_condition_distribution_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "equally_distributed",
              "control_predominant",
              "not_assessed"
            ]
          },
          {
            "name": "focus_population_location_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "airway_only",
              "not_assessed"
            ]
          },
          {
            "name": "cross_axis_canonical_identity_with_airway_only_location_incoherence",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "focus_population_identity": {
                "op": "in",
                "values": [
                  "chemosensory_marker_expressing_population_alpha"
                ]
              },
              "focus_population_location": {
                "op": "in",
                "values": [
                  "airway_only"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-3-pi",
      "modelName": "grok-4.3",
      "harness": "pi",
      "task": "gse171524_ectopic_airway_rare_cell",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 536,
      "cost": 0.024817399999999996,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "focus_population_identity_gate",
            "role": "gate",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "chemosensory_marker_expressing_population_alpha"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "focus_population_marker_calls",
            "kind": "list",
            "matchKey": "marker",
            "k": 3,
            "expected": [
              {
                "key": "POU2F3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "CHAT",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "TRPM5",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "LRMP",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNAT3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "AVIL",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNG13",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              }
            ]
          },
          {
            "name": "focus_population_condition_distribution_gate",
            "role": "gate",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_enriched_with_substantial_control_presence",
              "covid_predominant",
              "covid_only",
              "covid_enriched_trend_not_significant"
            ]
          },
          {
            "name": "focus_population_location_gate",
            "role": "gate",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "ectopic_parenchyma_in_covid",
              "parenchyma_only",
              "bronchiolized_or_peribronchial_parenchyma",
              "airway_and_parenchyma"
            ]
          },
          {
            "name": "focus_population_recovery_mode_gate",
            "role": "gate",
            "field": "$.focus_population_recovery_mode",
            "kind": "value",
            "op": "in",
            "expected": [
              "continuum_not_discrete_cluster"
            ]
          },
          {
            "name": "focus_population_identity_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "ionocyte_like_beta",
              "neuroendocrine_like_gamma",
              "basal_stem_progenitor_like_delta",
              "aberrant_basaloid_transitional_like_epsilon",
              "neuronal_or_neural_gene_minor_population_zeta",
              "deuterosomal_ciliated_progenitor_like_eta",
              "hillock_transitional_like_theta",
              "low_quality_doublet_or_debris",
              "no_rare_cell_recovered",
              "not_assessed"
            ]
          },
          {
            "name": "cf_wrong_lineage_marker_called_positive",
            "role": "hard_fail",
            "field": "$.focus_population_marker_calls[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "focus_population_condition_distribution_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "equally_distributed",
              "control_predominant",
              "not_assessed"
            ]
          },
          {
            "name": "focus_population_location_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "airway_only",
              "not_assessed"
            ]
          },
          {
            "name": "cross_axis_canonical_identity_with_airway_only_location_incoherence",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "focus_population_identity": {
                "op": "in",
                "values": [
                  "chemosensory_marker_expressing_population_alpha"
                ]
              },
              "focus_population_location": {
                "op": "in",
                "values": [
                  "airway_only"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-3-pi",
      "modelName": "grok-4.3",
      "harness": "pi",
      "task": "gse171524_ectopic_airway_rare_cell",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 642,
      "cost": 0.053312450000000004,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "focus_population_identity_gate",
            "role": "gate",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "chemosensory_marker_expressing_population_alpha"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "focus_population_marker_calls",
            "kind": "list",
            "matchKey": "marker",
            "k": 3,
            "expected": [
              {
                "key": "POU2F3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "CHAT",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "TRPM5",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "LRMP",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNAT3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "AVIL",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNG13",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              }
            ]
          },
          {
            "name": "focus_population_condition_distribution_gate",
            "role": "gate",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_enriched_with_substantial_control_presence",
              "covid_predominant",
              "covid_only",
              "covid_enriched_trend_not_significant"
            ]
          },
          {
            "name": "focus_population_location_gate",
            "role": "gate",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "ectopic_parenchyma_in_covid",
              "parenchyma_only",
              "bronchiolized_or_peribronchial_parenchyma",
              "airway_and_parenchyma"
            ]
          },
          {
            "name": "focus_population_recovery_mode_gate",
            "role": "gate",
            "field": "$.focus_population_recovery_mode",
            "kind": "value",
            "op": "in",
            "expected": [
              "continuum_not_discrete_cluster"
            ]
          },
          {
            "name": "focus_population_identity_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "ionocyte_like_beta",
              "neuroendocrine_like_gamma",
              "basal_stem_progenitor_like_delta",
              "aberrant_basaloid_transitional_like_epsilon",
              "neuronal_or_neural_gene_minor_population_zeta",
              "deuterosomal_ciliated_progenitor_like_eta",
              "hillock_transitional_like_theta",
              "low_quality_doublet_or_debris",
              "no_rare_cell_recovered",
              "not_assessed"
            ]
          },
          {
            "name": "cf_wrong_lineage_marker_called_positive",
            "role": "hard_fail",
            "field": "$.focus_population_marker_calls[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "focus_population_condition_distribution_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "equally_distributed",
              "control_predominant",
              "not_assessed"
            ]
          },
          {
            "name": "focus_population_location_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "airway_only",
              "not_assessed"
            ]
          },
          {
            "name": "cross_axis_canonical_identity_with_airway_only_location_incoherence",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "focus_population_identity": {
                "op": "in",
                "values": [
                  "chemosensory_marker_expressing_population_alpha"
                ]
              },
              "focus_population_location": {
                "op": "in",
                "values": [
                  "airway_only"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-3-pi",
      "modelName": "grok-4.3",
      "harness": "pi",
      "task": "gse171524_ectopic_airway_rare_cell",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 303,
      "cost": 0.023759100000000002,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "focus_population_identity_gate",
            "role": "gate",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "chemosensory_marker_expressing_population_alpha"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "focus_population_marker_calls",
            "kind": "list",
            "matchKey": "marker",
            "k": 3,
            "expected": [
              {
                "key": "POU2F3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "CHAT",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "TRPM5",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "LRMP",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNAT3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "AVIL",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNG13",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              }
            ]
          },
          {
            "name": "focus_population_condition_distribution_gate",
            "role": "gate",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_enriched_with_substantial_control_presence",
              "covid_predominant",
              "covid_only",
              "covid_enriched_trend_not_significant"
            ]
          },
          {
            "name": "focus_population_location_gate",
            "role": "gate",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "ectopic_parenchyma_in_covid",
              "parenchyma_only",
              "bronchiolized_or_peribronchial_parenchyma",
              "airway_and_parenchyma"
            ]
          },
          {
            "name": "focus_population_recovery_mode_gate",
            "role": "gate",
            "field": "$.focus_population_recovery_mode",
            "kind": "value",
            "op": "in",
            "expected": [
              "continuum_not_discrete_cluster"
            ]
          },
          {
            "name": "focus_population_identity_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "ionocyte_like_beta",
              "neuroendocrine_like_gamma",
              "basal_stem_progenitor_like_delta",
              "aberrant_basaloid_transitional_like_epsilon",
              "neuronal_or_neural_gene_minor_population_zeta",
              "deuterosomal_ciliated_progenitor_like_eta",
              "hillock_transitional_like_theta",
              "low_quality_doublet_or_debris",
              "no_rare_cell_recovered",
              "not_assessed"
            ]
          },
          {
            "name": "cf_wrong_lineage_marker_called_positive",
            "role": "hard_fail",
            "field": "$.focus_population_marker_calls[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "focus_population_condition_distribution_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "equally_distributed",
              "control_predominant",
              "not_assessed"
            ]
          },
          {
            "name": "focus_population_location_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "airway_only",
              "not_assessed"
            ]
          },
          {
            "name": "cross_axis_canonical_identity_with_airway_only_location_incoherence",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "focus_population_identity": {
                "op": "in",
                "values": [
                  "chemosensory_marker_expressing_population_alpha"
                ]
              },
              "focus_population_location": {
                "op": "in",
                "values": [
                  "airway_only"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-kimi-k2p6-pi",
      "modelName": "kimi-k2p6",
      "harness": "pi",
      "task": "gse171524_ectopic_airway_rare_cell",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 2260,
      "cost": 0.79769212,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "focus_population_identity_gate",
            "role": "gate",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "chemosensory_marker_expressing_population_alpha"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "focus_population_marker_calls",
            "kind": "list",
            "matchKey": "marker",
            "k": 3,
            "expected": [
              {
                "key": "POU2F3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "CHAT",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "TRPM5",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "LRMP",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNAT3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "AVIL",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNG13",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              }
            ]
          },
          {
            "name": "focus_population_condition_distribution_gate",
            "role": "gate",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_enriched_with_substantial_control_presence",
              "covid_predominant",
              "covid_only",
              "covid_enriched_trend_not_significant"
            ]
          },
          {
            "name": "focus_population_location_gate",
            "role": "gate",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "ectopic_parenchyma_in_covid",
              "parenchyma_only",
              "bronchiolized_or_peribronchial_parenchyma",
              "airway_and_parenchyma"
            ]
          },
          {
            "name": "focus_population_recovery_mode_gate",
            "role": "gate",
            "field": "$.focus_population_recovery_mode",
            "kind": "value",
            "op": "in",
            "expected": [
              "continuum_not_discrete_cluster"
            ]
          },
          {
            "name": "focus_population_identity_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "ionocyte_like_beta",
              "neuroendocrine_like_gamma",
              "basal_stem_progenitor_like_delta",
              "aberrant_basaloid_transitional_like_epsilon",
              "neuronal_or_neural_gene_minor_population_zeta",
              "deuterosomal_ciliated_progenitor_like_eta",
              "hillock_transitional_like_theta",
              "low_quality_doublet_or_debris",
              "no_rare_cell_recovered",
              "not_assessed"
            ]
          },
          {
            "name": "cf_wrong_lineage_marker_called_positive",
            "role": "hard_fail",
            "field": "$.focus_population_marker_calls[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "focus_population_condition_distribution_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "equally_distributed",
              "control_predominant",
              "not_assessed"
            ]
          },
          {
            "name": "focus_population_location_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "airway_only",
              "not_assessed"
            ]
          },
          {
            "name": "cross_axis_canonical_identity_with_airway_only_location_incoherence",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "focus_population_identity": {
                "op": "in",
                "values": [
                  "chemosensory_marker_expressing_population_alpha"
                ]
              },
              "focus_population_location": {
                "op": "in",
                "values": [
                  "airway_only"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-kimi-k2p6-pi",
      "modelName": "kimi-k2p6",
      "harness": "pi",
      "task": "gse171524_ectopic_airway_rare_cell",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 3958,
      "cost": 0.89348568,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "focus_population_identity_gate",
            "role": "gate",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "chemosensory_marker_expressing_population_alpha"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "focus_population_marker_calls",
            "kind": "list",
            "matchKey": "marker",
            "k": 3,
            "expected": [
              {
                "key": "POU2F3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "CHAT",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "TRPM5",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "LRMP",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNAT3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "AVIL",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNG13",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              }
            ]
          },
          {
            "name": "focus_population_condition_distribution_gate",
            "role": "gate",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_enriched_with_substantial_control_presence",
              "covid_predominant",
              "covid_only",
              "covid_enriched_trend_not_significant"
            ]
          },
          {
            "name": "focus_population_location_gate",
            "role": "gate",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "ectopic_parenchyma_in_covid",
              "parenchyma_only",
              "bronchiolized_or_peribronchial_parenchyma",
              "airway_and_parenchyma"
            ]
          },
          {
            "name": "focus_population_recovery_mode_gate",
            "role": "gate",
            "field": "$.focus_population_recovery_mode",
            "kind": "value",
            "op": "in",
            "expected": [
              "continuum_not_discrete_cluster"
            ]
          },
          {
            "name": "focus_population_identity_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "ionocyte_like_beta",
              "neuroendocrine_like_gamma",
              "basal_stem_progenitor_like_delta",
              "aberrant_basaloid_transitional_like_epsilon",
              "neuronal_or_neural_gene_minor_population_zeta",
              "deuterosomal_ciliated_progenitor_like_eta",
              "hillock_transitional_like_theta",
              "low_quality_doublet_or_debris",
              "no_rare_cell_recovered",
              "not_assessed"
            ]
          },
          {
            "name": "cf_wrong_lineage_marker_called_positive",
            "role": "hard_fail",
            "field": "$.focus_population_marker_calls[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "focus_population_condition_distribution_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "equally_distributed",
              "control_predominant",
              "not_assessed"
            ]
          },
          {
            "name": "focus_population_location_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "airway_only",
              "not_assessed"
            ]
          },
          {
            "name": "cross_axis_canonical_identity_with_airway_only_location_incoherence",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "focus_population_identity": {
                "op": "in",
                "values": [
                  "chemosensory_marker_expressing_population_alpha"
                ]
              },
              "focus_population_location": {
                "op": "in",
                "values": [
                  "airway_only"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-kimi-k2p6-pi",
      "modelName": "kimi-k2p6",
      "harness": "pi",
      "task": "gse171524_ectopic_airway_rare_cell",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1903,
      "cost": 0.66738595,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "focus_population_identity_gate",
            "role": "gate",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "chemosensory_marker_expressing_population_alpha"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "focus_population_marker_calls",
            "kind": "list",
            "matchKey": "marker",
            "k": 3,
            "expected": [
              {
                "key": "POU2F3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "CHAT",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "TRPM5",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "LRMP",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNAT3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "AVIL",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNG13",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              }
            ]
          },
          {
            "name": "focus_population_condition_distribution_gate",
            "role": "gate",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_enriched_with_substantial_control_presence",
              "covid_predominant",
              "covid_only",
              "covid_enriched_trend_not_significant"
            ]
          },
          {
            "name": "focus_population_location_gate",
            "role": "gate",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "ectopic_parenchyma_in_covid",
              "parenchyma_only",
              "bronchiolized_or_peribronchial_parenchyma",
              "airway_and_parenchyma"
            ]
          },
          {
            "name": "focus_population_recovery_mode_gate",
            "role": "gate",
            "field": "$.focus_population_recovery_mode",
            "kind": "value",
            "op": "in",
            "expected": [
              "continuum_not_discrete_cluster"
            ]
          },
          {
            "name": "focus_population_identity_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "ionocyte_like_beta",
              "neuroendocrine_like_gamma",
              "basal_stem_progenitor_like_delta",
              "aberrant_basaloid_transitional_like_epsilon",
              "neuronal_or_neural_gene_minor_population_zeta",
              "deuterosomal_ciliated_progenitor_like_eta",
              "hillock_transitional_like_theta",
              "low_quality_doublet_or_debris",
              "no_rare_cell_recovered",
              "not_assessed"
            ]
          },
          {
            "name": "cf_wrong_lineage_marker_called_positive",
            "role": "hard_fail",
            "field": "$.focus_population_marker_calls[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "focus_population_condition_distribution_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "equally_distributed",
              "control_predominant",
              "not_assessed"
            ]
          },
          {
            "name": "focus_population_location_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "airway_only",
              "not_assessed"
            ]
          },
          {
            "name": "cross_axis_canonical_identity_with_airway_only_location_incoherence",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "focus_population_identity": {
                "op": "in",
                "values": [
                  "chemosensory_marker_expressing_population_alpha"
                ]
              },
              "focus_population_location": {
                "op": "in",
                "values": [
                  "airway_only"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-6-claude-code",
      "modelName": "claude-opus-4-6",
      "harness": "claude-code",
      "task": "gse171524_pathological_fibroblast_master_regulator_druggable_targets",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 2502,
      "cost": 2.0570364999999997,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "disease_associated_fibroblast_subtype_weighted",
            "role": "additive",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "mr_tf_junb_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_alpha"
            ]
          },
          {
            "name": "mr_tf_jund_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_beta"
            ]
          },
          {
            "name": "drug_mmp14_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_alpha"
            ]
          },
          {
            "name": "drug_stat3_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_beta"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_weighted",
            "role": "additive",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "disease_associated_fibroblast_subtype_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "value",
            "op": "in",
            "expected": [
              "subtype_lambda",
              "subtype_epsilon",
              "subtype_iota",
              "subtype_beta",
              "subtype_eta",
              "subtype_zeta",
              "subtype_not_assessed"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "simple_de_ranking_by_log_fc",
              "marker_based_de_only",
              "not_computed"
            ]
          },
          {
            "name": "top_master_regulator_transcription_factors_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_not_computed"
            ]
          },
          {
            "name": "top_druggable_proteins_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_not_assessed"
            ]
          },
          {
            "name": "cross_axis_circular_marker_in_druggable_proteins",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_gamma",
              "drug_target_delta",
              "drug_target_eta"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-6-claude-code",
      "modelName": "claude-opus-4-6",
      "harness": "claude-code",
      "task": "gse171524_pathological_fibroblast_master_regulator_druggable_targets",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 1860,
      "cost": 1.3913302500000004,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "disease_associated_fibroblast_subtype_weighted",
            "role": "additive",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "mr_tf_junb_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_alpha"
            ]
          },
          {
            "name": "mr_tf_jund_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_beta"
            ]
          },
          {
            "name": "drug_mmp14_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_alpha"
            ]
          },
          {
            "name": "drug_stat3_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_beta"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_weighted",
            "role": "additive",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "disease_associated_fibroblast_subtype_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "value",
            "op": "in",
            "expected": [
              "subtype_lambda",
              "subtype_epsilon",
              "subtype_iota",
              "subtype_beta",
              "subtype_eta",
              "subtype_zeta",
              "subtype_not_assessed"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "simple_de_ranking_by_log_fc",
              "marker_based_de_only",
              "not_computed"
            ]
          },
          {
            "name": "top_master_regulator_transcription_factors_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_not_computed"
            ]
          },
          {
            "name": "top_druggable_proteins_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_not_assessed"
            ]
          },
          {
            "name": "cross_axis_circular_marker_in_druggable_proteins",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_gamma",
              "drug_target_delta",
              "drug_target_eta"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-6-claude-code",
      "modelName": "claude-opus-4-6",
      "harness": "claude-code",
      "task": "gse171524_pathological_fibroblast_master_regulator_druggable_targets",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 10555,
      "cost": 9.92197875,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "disease_associated_fibroblast_subtype_weighted",
            "role": "additive",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "mr_tf_junb_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_alpha"
            ]
          },
          {
            "name": "mr_tf_jund_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_beta"
            ]
          },
          {
            "name": "drug_mmp14_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_alpha"
            ]
          },
          {
            "name": "drug_stat3_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_beta"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_weighted",
            "role": "additive",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "disease_associated_fibroblast_subtype_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "value",
            "op": "in",
            "expected": [
              "subtype_lambda",
              "subtype_epsilon",
              "subtype_iota",
              "subtype_beta",
              "subtype_eta",
              "subtype_zeta",
              "subtype_not_assessed"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "simple_de_ranking_by_log_fc",
              "marker_based_de_only",
              "not_computed"
            ]
          },
          {
            "name": "top_master_regulator_transcription_factors_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_not_computed"
            ]
          },
          {
            "name": "top_druggable_proteins_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_not_assessed"
            ]
          },
          {
            "name": "cross_axis_circular_marker_in_druggable_proteins",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_gamma",
              "drug_target_delta",
              "drug_target_eta"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-6-pi",
      "modelName": "claude-opus-4-6",
      "harness": "pi",
      "task": "gse171524_pathological_fibroblast_master_regulator_druggable_targets",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 3127,
      "cost": 2.9313347499999995,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "disease_associated_fibroblast_subtype_weighted",
            "role": "additive",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "mr_tf_junb_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_alpha"
            ]
          },
          {
            "name": "mr_tf_jund_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_beta"
            ]
          },
          {
            "name": "drug_mmp14_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_alpha"
            ]
          },
          {
            "name": "drug_stat3_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_beta"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_weighted",
            "role": "additive",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "disease_associated_fibroblast_subtype_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "value",
            "op": "in",
            "expected": [
              "subtype_lambda",
              "subtype_epsilon",
              "subtype_iota",
              "subtype_beta",
              "subtype_eta",
              "subtype_zeta",
              "subtype_not_assessed"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "simple_de_ranking_by_log_fc",
              "marker_based_de_only",
              "not_computed"
            ]
          },
          {
            "name": "top_master_regulator_transcription_factors_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_not_computed"
            ]
          },
          {
            "name": "top_druggable_proteins_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_not_assessed"
            ]
          },
          {
            "name": "cross_axis_circular_marker_in_druggable_proteins",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_gamma",
              "drug_target_delta",
              "drug_target_eta"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-6-pi",
      "modelName": "claude-opus-4-6",
      "harness": "pi",
      "task": "gse171524_pathological_fibroblast_master_regulator_druggable_targets",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 2234,
      "cost": 2.1566642500000004,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "disease_associated_fibroblast_subtype_weighted",
            "role": "additive",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "mr_tf_junb_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_alpha"
            ]
          },
          {
            "name": "mr_tf_jund_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_beta"
            ]
          },
          {
            "name": "drug_mmp14_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_alpha"
            ]
          },
          {
            "name": "drug_stat3_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_beta"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_weighted",
            "role": "additive",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "disease_associated_fibroblast_subtype_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "value",
            "op": "in",
            "expected": [
              "subtype_lambda",
              "subtype_epsilon",
              "subtype_iota",
              "subtype_beta",
              "subtype_eta",
              "subtype_zeta",
              "subtype_not_assessed"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "simple_de_ranking_by_log_fc",
              "marker_based_de_only",
              "not_computed"
            ]
          },
          {
            "name": "top_master_regulator_transcription_factors_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_not_computed"
            ]
          },
          {
            "name": "top_druggable_proteins_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_not_assessed"
            ]
          },
          {
            "name": "cross_axis_circular_marker_in_druggable_proteins",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_gamma",
              "drug_target_delta",
              "drug_target_eta"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-6-pi",
      "modelName": "claude-opus-4-6",
      "harness": "pi",
      "task": "gse171524_pathological_fibroblast_master_regulator_druggable_targets",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 4214,
      "cost": 3.0365385000000003,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "disease_associated_fibroblast_subtype_weighted",
            "role": "additive",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "mr_tf_junb_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_alpha"
            ]
          },
          {
            "name": "mr_tf_jund_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_beta"
            ]
          },
          {
            "name": "drug_mmp14_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_alpha"
            ]
          },
          {
            "name": "drug_stat3_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_beta"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_weighted",
            "role": "additive",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "disease_associated_fibroblast_subtype_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "value",
            "op": "in",
            "expected": [
              "subtype_lambda",
              "subtype_epsilon",
              "subtype_iota",
              "subtype_beta",
              "subtype_eta",
              "subtype_zeta",
              "subtype_not_assessed"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "simple_de_ranking_by_log_fc",
              "marker_based_de_only",
              "not_computed"
            ]
          },
          {
            "name": "top_master_regulator_transcription_factors_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_not_computed"
            ]
          },
          {
            "name": "top_druggable_proteins_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_not_assessed"
            ]
          },
          {
            "name": "cross_axis_circular_marker_in_druggable_proteins",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_gamma",
              "drug_target_delta",
              "drug_target_eta"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-7-claude-code",
      "modelName": "claude-opus-4-7",
      "harness": "claude-code",
      "task": "gse171524_pathological_fibroblast_master_regulator_druggable_targets",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 1477,
      "cost": 6.544356499999999,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "disease_associated_fibroblast_subtype_weighted",
            "role": "additive",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "mr_tf_junb_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_alpha"
            ]
          },
          {
            "name": "mr_tf_jund_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_beta"
            ]
          },
          {
            "name": "drug_mmp14_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_alpha"
            ]
          },
          {
            "name": "drug_stat3_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_beta"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_weighted",
            "role": "additive",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "disease_associated_fibroblast_subtype_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "value",
            "op": "in",
            "expected": [
              "subtype_lambda",
              "subtype_epsilon",
              "subtype_iota",
              "subtype_beta",
              "subtype_eta",
              "subtype_zeta",
              "subtype_not_assessed"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "simple_de_ranking_by_log_fc",
              "marker_based_de_only",
              "not_computed"
            ]
          },
          {
            "name": "top_master_regulator_transcription_factors_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_not_computed"
            ]
          },
          {
            "name": "top_druggable_proteins_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_not_assessed"
            ]
          },
          {
            "name": "cross_axis_circular_marker_in_druggable_proteins",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_gamma",
              "drug_target_delta",
              "drug_target_eta"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-7-claude-code",
      "modelName": "claude-opus-4-7",
      "harness": "claude-code",
      "task": "gse171524_pathological_fibroblast_master_regulator_druggable_targets",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 1799,
      "cost": 8.040517250000002,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "disease_associated_fibroblast_subtype_weighted",
            "role": "additive",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "mr_tf_junb_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_alpha"
            ]
          },
          {
            "name": "mr_tf_jund_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_beta"
            ]
          },
          {
            "name": "drug_mmp14_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_alpha"
            ]
          },
          {
            "name": "drug_stat3_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_beta"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_weighted",
            "role": "additive",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "disease_associated_fibroblast_subtype_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "value",
            "op": "in",
            "expected": [
              "subtype_lambda",
              "subtype_epsilon",
              "subtype_iota",
              "subtype_beta",
              "subtype_eta",
              "subtype_zeta",
              "subtype_not_assessed"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "simple_de_ranking_by_log_fc",
              "marker_based_de_only",
              "not_computed"
            ]
          },
          {
            "name": "top_master_regulator_transcription_factors_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_not_computed"
            ]
          },
          {
            "name": "top_druggable_proteins_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_not_assessed"
            ]
          },
          {
            "name": "cross_axis_circular_marker_in_druggable_proteins",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_gamma",
              "drug_target_delta",
              "drug_target_eta"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-7-claude-code",
      "modelName": "claude-opus-4-7",
      "harness": "claude-code",
      "task": "gse171524_pathological_fibroblast_master_regulator_druggable_targets",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1624,
      "cost": 4.9431345,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "disease_associated_fibroblast_subtype_weighted",
            "role": "additive",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "mr_tf_junb_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_alpha"
            ]
          },
          {
            "name": "mr_tf_jund_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_beta"
            ]
          },
          {
            "name": "drug_mmp14_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_alpha"
            ]
          },
          {
            "name": "drug_stat3_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_beta"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_weighted",
            "role": "additive",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "disease_associated_fibroblast_subtype_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "value",
            "op": "in",
            "expected": [
              "subtype_lambda",
              "subtype_epsilon",
              "subtype_iota",
              "subtype_beta",
              "subtype_eta",
              "subtype_zeta",
              "subtype_not_assessed"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "simple_de_ranking_by_log_fc",
              "marker_based_de_only",
              "not_computed"
            ]
          },
          {
            "name": "top_master_regulator_transcription_factors_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_not_computed"
            ]
          },
          {
            "name": "top_druggable_proteins_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_not_assessed"
            ]
          },
          {
            "name": "cross_axis_circular_marker_in_druggable_proteins",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_gamma",
              "drug_target_delta",
              "drug_target_eta"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-7-pi",
      "modelName": "claude-opus-4-7",
      "harness": "pi",
      "task": "gse171524_pathological_fibroblast_master_regulator_druggable_targets",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 658,
      "cost": 1.11504825,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "disease_associated_fibroblast_subtype_weighted",
            "role": "additive",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "mr_tf_junb_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_alpha"
            ]
          },
          {
            "name": "mr_tf_jund_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_beta"
            ]
          },
          {
            "name": "drug_mmp14_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_alpha"
            ]
          },
          {
            "name": "drug_stat3_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_beta"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_weighted",
            "role": "additive",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "disease_associated_fibroblast_subtype_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "value",
            "op": "in",
            "expected": [
              "subtype_lambda",
              "subtype_epsilon",
              "subtype_iota",
              "subtype_beta",
              "subtype_eta",
              "subtype_zeta",
              "subtype_not_assessed"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "simple_de_ranking_by_log_fc",
              "marker_based_de_only",
              "not_computed"
            ]
          },
          {
            "name": "top_master_regulator_transcription_factors_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_not_computed"
            ]
          },
          {
            "name": "top_druggable_proteins_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_not_assessed"
            ]
          },
          {
            "name": "cross_axis_circular_marker_in_druggable_proteins",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_gamma",
              "drug_target_delta",
              "drug_target_eta"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-7-pi",
      "modelName": "claude-opus-4-7",
      "harness": "pi",
      "task": "gse171524_pathological_fibroblast_master_regulator_druggable_targets",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 1484,
      "cost": 2.4525727499999994,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "disease_associated_fibroblast_subtype_weighted",
            "role": "additive",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "mr_tf_junb_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_alpha"
            ]
          },
          {
            "name": "mr_tf_jund_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_beta"
            ]
          },
          {
            "name": "drug_mmp14_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_alpha"
            ]
          },
          {
            "name": "drug_stat3_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_beta"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_weighted",
            "role": "additive",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "disease_associated_fibroblast_subtype_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "value",
            "op": "in",
            "expected": [
              "subtype_lambda",
              "subtype_epsilon",
              "subtype_iota",
              "subtype_beta",
              "subtype_eta",
              "subtype_zeta",
              "subtype_not_assessed"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "simple_de_ranking_by_log_fc",
              "marker_based_de_only",
              "not_computed"
            ]
          },
          {
            "name": "top_master_regulator_transcription_factors_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_not_computed"
            ]
          },
          {
            "name": "top_druggable_proteins_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_not_assessed"
            ]
          },
          {
            "name": "cross_axis_circular_marker_in_druggable_proteins",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_gamma",
              "drug_target_delta",
              "drug_target_eta"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-7-pi",
      "modelName": "claude-opus-4-7",
      "harness": "pi",
      "task": "gse171524_pathological_fibroblast_master_regulator_druggable_targets",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1476,
      "cost": 1.8586667499999998,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "disease_associated_fibroblast_subtype_weighted",
            "role": "additive",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "mr_tf_junb_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_alpha"
            ]
          },
          {
            "name": "mr_tf_jund_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_beta"
            ]
          },
          {
            "name": "drug_mmp14_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_alpha"
            ]
          },
          {
            "name": "drug_stat3_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_beta"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_weighted",
            "role": "additive",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "disease_associated_fibroblast_subtype_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "value",
            "op": "in",
            "expected": [
              "subtype_lambda",
              "subtype_epsilon",
              "subtype_iota",
              "subtype_beta",
              "subtype_eta",
              "subtype_zeta",
              "subtype_not_assessed"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "simple_de_ranking_by_log_fc",
              "marker_based_de_only",
              "not_computed"
            ]
          },
          {
            "name": "top_master_regulator_transcription_factors_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_not_computed"
            ]
          },
          {
            "name": "top_druggable_proteins_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_not_assessed"
            ]
          },
          {
            "name": "cross_axis_circular_marker_in_druggable_proteins",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_gamma",
              "drug_target_delta",
              "drug_target_eta"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-8-claude-code",
      "modelName": "claude-opus-4-8",
      "harness": "claude-code",
      "task": "gse171524_pathological_fibroblast_master_regulator_druggable_targets",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 2584,
      "cost": 10.24723575,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "disease_associated_fibroblast_subtype_weighted",
            "role": "additive",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "mr_tf_junb_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_alpha"
            ]
          },
          {
            "name": "mr_tf_jund_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_beta"
            ]
          },
          {
            "name": "drug_mmp14_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_alpha"
            ]
          },
          {
            "name": "drug_stat3_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_beta"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_weighted",
            "role": "additive",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "disease_associated_fibroblast_subtype_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "value",
            "op": "in",
            "expected": [
              "subtype_lambda",
              "subtype_epsilon",
              "subtype_iota",
              "subtype_beta",
              "subtype_eta",
              "subtype_zeta",
              "subtype_not_assessed"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "simple_de_ranking_by_log_fc",
              "marker_based_de_only",
              "not_computed"
            ]
          },
          {
            "name": "top_master_regulator_transcription_factors_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_not_computed"
            ]
          },
          {
            "name": "top_druggable_proteins_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_not_assessed"
            ]
          },
          {
            "name": "cross_axis_circular_marker_in_druggable_proteins",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_gamma",
              "drug_target_delta",
              "drug_target_eta"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-8-claude-code",
      "modelName": "claude-opus-4-8",
      "harness": "claude-code",
      "task": "gse171524_pathological_fibroblast_master_regulator_druggable_targets",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 2726,
      "cost": 6.044113999999999,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "disease_associated_fibroblast_subtype_weighted",
            "role": "additive",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "mr_tf_junb_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_alpha"
            ]
          },
          {
            "name": "mr_tf_jund_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_beta"
            ]
          },
          {
            "name": "drug_mmp14_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_alpha"
            ]
          },
          {
            "name": "drug_stat3_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_beta"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_weighted",
            "role": "additive",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "disease_associated_fibroblast_subtype_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "value",
            "op": "in",
            "expected": [
              "subtype_lambda",
              "subtype_epsilon",
              "subtype_iota",
              "subtype_beta",
              "subtype_eta",
              "subtype_zeta",
              "subtype_not_assessed"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "simple_de_ranking_by_log_fc",
              "marker_based_de_only",
              "not_computed"
            ]
          },
          {
            "name": "top_master_regulator_transcription_factors_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_not_computed"
            ]
          },
          {
            "name": "top_druggable_proteins_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_not_assessed"
            ]
          },
          {
            "name": "cross_axis_circular_marker_in_druggable_proteins",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_gamma",
              "drug_target_delta",
              "drug_target_eta"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-8-claude-code",
      "modelName": "claude-opus-4-8",
      "harness": "claude-code",
      "task": "gse171524_pathological_fibroblast_master_regulator_druggable_targets",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 9155,
      "cost": 27.419283249999996,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "disease_associated_fibroblast_subtype_weighted",
            "role": "additive",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "mr_tf_junb_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_alpha"
            ]
          },
          {
            "name": "mr_tf_jund_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_beta"
            ]
          },
          {
            "name": "drug_mmp14_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_alpha"
            ]
          },
          {
            "name": "drug_stat3_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_beta"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_weighted",
            "role": "additive",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "disease_associated_fibroblast_subtype_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "value",
            "op": "in",
            "expected": [
              "subtype_lambda",
              "subtype_epsilon",
              "subtype_iota",
              "subtype_beta",
              "subtype_eta",
              "subtype_zeta",
              "subtype_not_assessed"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "simple_de_ranking_by_log_fc",
              "marker_based_de_only",
              "not_computed"
            ]
          },
          {
            "name": "top_master_regulator_transcription_factors_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_not_computed"
            ]
          },
          {
            "name": "top_druggable_proteins_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_not_assessed"
            ]
          },
          {
            "name": "cross_axis_circular_marker_in_druggable_proteins",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_gamma",
              "drug_target_delta",
              "drug_target_eta"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-8-pi",
      "modelName": "claude-opus-4-8",
      "harness": "pi",
      "task": "gse171524_pathological_fibroblast_master_regulator_druggable_targets",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 2594,
      "cost": 3.132208750000001,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "disease_associated_fibroblast_subtype_weighted",
            "role": "additive",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "mr_tf_junb_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_alpha"
            ]
          },
          {
            "name": "mr_tf_jund_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_beta"
            ]
          },
          {
            "name": "drug_mmp14_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_alpha"
            ]
          },
          {
            "name": "drug_stat3_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_beta"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_weighted",
            "role": "additive",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "disease_associated_fibroblast_subtype_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "value",
            "op": "in",
            "expected": [
              "subtype_lambda",
              "subtype_epsilon",
              "subtype_iota",
              "subtype_beta",
              "subtype_eta",
              "subtype_zeta",
              "subtype_not_assessed"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "simple_de_ranking_by_log_fc",
              "marker_based_de_only",
              "not_computed"
            ]
          },
          {
            "name": "top_master_regulator_transcription_factors_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_not_computed"
            ]
          },
          {
            "name": "top_druggable_proteins_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_not_assessed"
            ]
          },
          {
            "name": "cross_axis_circular_marker_in_druggable_proteins",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_gamma",
              "drug_target_delta",
              "drug_target_eta"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-8-pi",
      "modelName": "claude-opus-4-8",
      "harness": "pi",
      "task": "gse171524_pathological_fibroblast_master_regulator_druggable_targets",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 5987,
      "cost": 5.48386275,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "disease_associated_fibroblast_subtype_weighted",
            "role": "additive",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "mr_tf_junb_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_alpha"
            ]
          },
          {
            "name": "mr_tf_jund_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_beta"
            ]
          },
          {
            "name": "drug_mmp14_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_alpha"
            ]
          },
          {
            "name": "drug_stat3_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_beta"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_weighted",
            "role": "additive",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "disease_associated_fibroblast_subtype_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "value",
            "op": "in",
            "expected": [
              "subtype_lambda",
              "subtype_epsilon",
              "subtype_iota",
              "subtype_beta",
              "subtype_eta",
              "subtype_zeta",
              "subtype_not_assessed"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "simple_de_ranking_by_log_fc",
              "marker_based_de_only",
              "not_computed"
            ]
          },
          {
            "name": "top_master_regulator_transcription_factors_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_not_computed"
            ]
          },
          {
            "name": "top_druggable_proteins_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_not_assessed"
            ]
          },
          {
            "name": "cross_axis_circular_marker_in_druggable_proteins",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_gamma",
              "drug_target_delta",
              "drug_target_eta"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-8-pi",
      "modelName": "claude-opus-4-8",
      "harness": "pi",
      "task": "gse171524_pathological_fibroblast_master_regulator_druggable_targets",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 2404,
      "cost": 2.7706850000000003,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "disease_associated_fibroblast_subtype_weighted",
            "role": "additive",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "mr_tf_junb_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_alpha"
            ]
          },
          {
            "name": "mr_tf_jund_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_beta"
            ]
          },
          {
            "name": "drug_mmp14_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_alpha"
            ]
          },
          {
            "name": "drug_stat3_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_beta"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_weighted",
            "role": "additive",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "disease_associated_fibroblast_subtype_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "value",
            "op": "in",
            "expected": [
              "subtype_lambda",
              "subtype_epsilon",
              "subtype_iota",
              "subtype_beta",
              "subtype_eta",
              "subtype_zeta",
              "subtype_not_assessed"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "simple_de_ranking_by_log_fc",
              "marker_based_de_only",
              "not_computed"
            ]
          },
          {
            "name": "top_master_regulator_transcription_factors_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_not_computed"
            ]
          },
          {
            "name": "top_druggable_proteins_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_not_assessed"
            ]
          },
          {
            "name": "cross_axis_circular_marker_in_druggable_proteins",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_gamma",
              "drug_target_delta",
              "drug_target_eta"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-4-6-pi",
      "modelName": "claude-sonnet-4-6",
      "harness": "pi",
      "task": "gse171524_pathological_fibroblast_master_regulator_druggable_targets",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 4768,
      "cost": 2.0008621499999997,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "disease_associated_fibroblast_subtype_weighted",
            "role": "additive",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "mr_tf_junb_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_alpha"
            ]
          },
          {
            "name": "mr_tf_jund_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_beta"
            ]
          },
          {
            "name": "drug_mmp14_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_alpha"
            ]
          },
          {
            "name": "drug_stat3_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_beta"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_weighted",
            "role": "additive",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "disease_associated_fibroblast_subtype_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "value",
            "op": "in",
            "expected": [
              "subtype_lambda",
              "subtype_epsilon",
              "subtype_iota",
              "subtype_beta",
              "subtype_eta",
              "subtype_zeta",
              "subtype_not_assessed"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "simple_de_ranking_by_log_fc",
              "marker_based_de_only",
              "not_computed"
            ]
          },
          {
            "name": "top_master_regulator_transcription_factors_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_not_computed"
            ]
          },
          {
            "name": "top_druggable_proteins_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_not_assessed"
            ]
          },
          {
            "name": "cross_axis_circular_marker_in_druggable_proteins",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_gamma",
              "drug_target_delta",
              "drug_target_eta"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-4-6-pi",
      "modelName": "claude-sonnet-4-6",
      "harness": "pi",
      "task": "gse171524_pathological_fibroblast_master_regulator_druggable_targets",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 2568,
      "cost": 1.889064,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "disease_associated_fibroblast_subtype_weighted",
            "role": "additive",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "mr_tf_junb_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_alpha"
            ]
          },
          {
            "name": "mr_tf_jund_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_beta"
            ]
          },
          {
            "name": "drug_mmp14_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_alpha"
            ]
          },
          {
            "name": "drug_stat3_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_beta"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_weighted",
            "role": "additive",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "disease_associated_fibroblast_subtype_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "value",
            "op": "in",
            "expected": [
              "subtype_lambda",
              "subtype_epsilon",
              "subtype_iota",
              "subtype_beta",
              "subtype_eta",
              "subtype_zeta",
              "subtype_not_assessed"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "simple_de_ranking_by_log_fc",
              "marker_based_de_only",
              "not_computed"
            ]
          },
          {
            "name": "top_master_regulator_transcription_factors_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_not_computed"
            ]
          },
          {
            "name": "top_druggable_proteins_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_not_assessed"
            ]
          },
          {
            "name": "cross_axis_circular_marker_in_druggable_proteins",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_gamma",
              "drug_target_delta",
              "drug_target_eta"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-4-6-pi",
      "modelName": "claude-sonnet-4-6",
      "harness": "pi",
      "task": "gse171524_pathological_fibroblast_master_regulator_druggable_targets",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1606,
      "cost": 1.3330998,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "disease_associated_fibroblast_subtype_weighted",
            "role": "additive",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "mr_tf_junb_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_alpha"
            ]
          },
          {
            "name": "mr_tf_jund_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_beta"
            ]
          },
          {
            "name": "drug_mmp14_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_alpha"
            ]
          },
          {
            "name": "drug_stat3_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_beta"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_weighted",
            "role": "additive",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "disease_associated_fibroblast_subtype_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "value",
            "op": "in",
            "expected": [
              "subtype_lambda",
              "subtype_epsilon",
              "subtype_iota",
              "subtype_beta",
              "subtype_eta",
              "subtype_zeta",
              "subtype_not_assessed"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "simple_de_ranking_by_log_fc",
              "marker_based_de_only",
              "not_computed"
            ]
          },
          {
            "name": "top_master_regulator_transcription_factors_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_not_computed"
            ]
          },
          {
            "name": "top_druggable_proteins_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_not_assessed"
            ]
          },
          {
            "name": "cross_axis_circular_marker_in_druggable_proteins",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_gamma",
              "drug_target_delta",
              "drug_target_eta"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gemini-3-1-pro-preview-pi",
      "modelName": "gemini-3.1-pro-preview",
      "harness": "pi",
      "task": "gse171524_pathological_fibroblast_master_regulator_druggable_targets",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 1,
      "passed": true,
      "score": null,
      "durationS": 1358,
      "cost": 1.2807878000000001,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "disease_associated_fibroblast_subtype_weighted",
            "role": "additive",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "mr_tf_junb_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_alpha"
            ]
          },
          {
            "name": "mr_tf_jund_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_beta"
            ]
          },
          {
            "name": "drug_mmp14_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_alpha"
            ]
          },
          {
            "name": "drug_stat3_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_beta"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_weighted",
            "role": "additive",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "disease_associated_fibroblast_subtype_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "value",
            "op": "in",
            "expected": [
              "subtype_lambda",
              "subtype_epsilon",
              "subtype_iota",
              "subtype_beta",
              "subtype_eta",
              "subtype_zeta",
              "subtype_not_assessed"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "simple_de_ranking_by_log_fc",
              "marker_based_de_only",
              "not_computed"
            ]
          },
          {
            "name": "top_master_regulator_transcription_factors_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_not_computed"
            ]
          },
          {
            "name": "top_druggable_proteins_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_not_assessed"
            ]
          },
          {
            "name": "cross_axis_circular_marker_in_druggable_proteins",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_gamma",
              "drug_target_delta",
              "drug_target_eta"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gemini-3-1-pro-preview-pi",
      "modelName": "gemini-3.1-pro-preview",
      "harness": "pi",
      "task": "gse171524_pathological_fibroblast_master_regulator_druggable_targets",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 1532,
      "cost": 0.7759060000000001,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "disease_associated_fibroblast_subtype_weighted",
            "role": "additive",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "mr_tf_junb_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_alpha"
            ]
          },
          {
            "name": "mr_tf_jund_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_beta"
            ]
          },
          {
            "name": "drug_mmp14_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_alpha"
            ]
          },
          {
            "name": "drug_stat3_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_beta"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_weighted",
            "role": "additive",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "disease_associated_fibroblast_subtype_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "value",
            "op": "in",
            "expected": [
              "subtype_lambda",
              "subtype_epsilon",
              "subtype_iota",
              "subtype_beta",
              "subtype_eta",
              "subtype_zeta",
              "subtype_not_assessed"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "simple_de_ranking_by_log_fc",
              "marker_based_de_only",
              "not_computed"
            ]
          },
          {
            "name": "top_master_regulator_transcription_factors_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_not_computed"
            ]
          },
          {
            "name": "top_druggable_proteins_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_not_assessed"
            ]
          },
          {
            "name": "cross_axis_circular_marker_in_druggable_proteins",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_gamma",
              "drug_target_delta",
              "drug_target_eta"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gemini-3-1-pro-preview-pi",
      "modelName": "gemini-3.1-pro-preview",
      "harness": "pi",
      "task": "gse171524_pathological_fibroblast_master_regulator_druggable_targets",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1487,
      "cost": 1.0607937999999995,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "disease_associated_fibroblast_subtype_weighted",
            "role": "additive",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "mr_tf_junb_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_alpha"
            ]
          },
          {
            "name": "mr_tf_jund_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_beta"
            ]
          },
          {
            "name": "drug_mmp14_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_alpha"
            ]
          },
          {
            "name": "drug_stat3_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_beta"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_weighted",
            "role": "additive",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "disease_associated_fibroblast_subtype_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "value",
            "op": "in",
            "expected": [
              "subtype_lambda",
              "subtype_epsilon",
              "subtype_iota",
              "subtype_beta",
              "subtype_eta",
              "subtype_zeta",
              "subtype_not_assessed"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "simple_de_ranking_by_log_fc",
              "marker_based_de_only",
              "not_computed"
            ]
          },
          {
            "name": "top_master_regulator_transcription_factors_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_not_computed"
            ]
          },
          {
            "name": "top_druggable_proteins_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_not_assessed"
            ]
          },
          {
            "name": "cross_axis_circular_marker_in_druggable_proteins",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_gamma",
              "drug_target_delta",
              "drug_target_eta"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gemini-3-5-flash-pi",
      "modelName": "gemini-3.5-flash",
      "harness": "pi",
      "task": "gse171524_pathological_fibroblast_master_regulator_druggable_targets",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 1,
      "passed": true,
      "score": null,
      "durationS": 1709,
      "cost": 2.3232009,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "disease_associated_fibroblast_subtype_weighted",
            "role": "additive",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "mr_tf_junb_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_alpha"
            ]
          },
          {
            "name": "mr_tf_jund_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_beta"
            ]
          },
          {
            "name": "drug_mmp14_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_alpha"
            ]
          },
          {
            "name": "drug_stat3_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_beta"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_weighted",
            "role": "additive",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "disease_associated_fibroblast_subtype_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "value",
            "op": "in",
            "expected": [
              "subtype_lambda",
              "subtype_epsilon",
              "subtype_iota",
              "subtype_beta",
              "subtype_eta",
              "subtype_zeta",
              "subtype_not_assessed"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "simple_de_ranking_by_log_fc",
              "marker_based_de_only",
              "not_computed"
            ]
          },
          {
            "name": "top_master_regulator_transcription_factors_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_not_computed"
            ]
          },
          {
            "name": "top_druggable_proteins_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_not_assessed"
            ]
          },
          {
            "name": "cross_axis_circular_marker_in_druggable_proteins",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_gamma",
              "drug_target_delta",
              "drug_target_eta"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gemini-3-5-flash-pi",
      "modelName": "gemini-3.5-flash",
      "harness": "pi",
      "task": "gse171524_pathological_fibroblast_master_regulator_druggable_targets",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 907,
      "cost": 1.2945123,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "disease_associated_fibroblast_subtype_weighted",
            "role": "additive",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "mr_tf_junb_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_alpha"
            ]
          },
          {
            "name": "mr_tf_jund_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_beta"
            ]
          },
          {
            "name": "drug_mmp14_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_alpha"
            ]
          },
          {
            "name": "drug_stat3_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_beta"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_weighted",
            "role": "additive",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "disease_associated_fibroblast_subtype_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "value",
            "op": "in",
            "expected": [
              "subtype_lambda",
              "subtype_epsilon",
              "subtype_iota",
              "subtype_beta",
              "subtype_eta",
              "subtype_zeta",
              "subtype_not_assessed"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "simple_de_ranking_by_log_fc",
              "marker_based_de_only",
              "not_computed"
            ]
          },
          {
            "name": "top_master_regulator_transcription_factors_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_not_computed"
            ]
          },
          {
            "name": "top_druggable_proteins_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_not_assessed"
            ]
          },
          {
            "name": "cross_axis_circular_marker_in_druggable_proteins",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_gamma",
              "drug_target_delta",
              "drug_target_eta"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gemini-3-5-flash-pi",
      "modelName": "gemini-3.5-flash",
      "harness": "pi",
      "task": "gse171524_pathological_fibroblast_master_regulator_druggable_targets",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 3,
      "passed": true,
      "score": null,
      "durationS": 504,
      "cost": 1.3551693000000005,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "disease_associated_fibroblast_subtype_weighted",
            "role": "additive",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "mr_tf_junb_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_alpha"
            ]
          },
          {
            "name": "mr_tf_jund_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_beta"
            ]
          },
          {
            "name": "drug_mmp14_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_alpha"
            ]
          },
          {
            "name": "drug_stat3_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_beta"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_weighted",
            "role": "additive",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "disease_associated_fibroblast_subtype_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "value",
            "op": "in",
            "expected": [
              "subtype_lambda",
              "subtype_epsilon",
              "subtype_iota",
              "subtype_beta",
              "subtype_eta",
              "subtype_zeta",
              "subtype_not_assessed"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "simple_de_ranking_by_log_fc",
              "marker_based_de_only",
              "not_computed"
            ]
          },
          {
            "name": "top_master_regulator_transcription_factors_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_not_computed"
            ]
          },
          {
            "name": "top_druggable_proteins_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_not_assessed"
            ]
          },
          {
            "name": "cross_axis_circular_marker_in_druggable_proteins",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_gamma",
              "drug_target_delta",
              "drug_target_eta"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-glm-5p2-pi",
      "modelName": "glm-5p2",
      "harness": "pi",
      "task": "gse171524_pathological_fibroblast_master_regulator_druggable_targets",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 8020,
      "cost": 6.611188760000003,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "disease_associated_fibroblast_subtype_weighted",
            "role": "additive",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "mr_tf_junb_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_alpha"
            ]
          },
          {
            "name": "mr_tf_jund_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_beta"
            ]
          },
          {
            "name": "drug_mmp14_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_alpha"
            ]
          },
          {
            "name": "drug_stat3_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_beta"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_weighted",
            "role": "additive",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "disease_associated_fibroblast_subtype_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "value",
            "op": "in",
            "expected": [
              "subtype_lambda",
              "subtype_epsilon",
              "subtype_iota",
              "subtype_beta",
              "subtype_eta",
              "subtype_zeta",
              "subtype_not_assessed"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "simple_de_ranking_by_log_fc",
              "marker_based_de_only",
              "not_computed"
            ]
          },
          {
            "name": "top_master_regulator_transcription_factors_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_not_computed"
            ]
          },
          {
            "name": "top_druggable_proteins_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_not_assessed"
            ]
          },
          {
            "name": "cross_axis_circular_marker_in_druggable_proteins",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_gamma",
              "drug_target_delta",
              "drug_target_eta"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-glm-5p2-pi",
      "modelName": "glm-5p2",
      "harness": "pi",
      "task": "gse171524_pathological_fibroblast_master_regulator_druggable_targets",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 4648,
      "cost": 3.7677674199999993,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "disease_associated_fibroblast_subtype_weighted",
            "role": "additive",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "mr_tf_junb_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_alpha"
            ]
          },
          {
            "name": "mr_tf_jund_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_beta"
            ]
          },
          {
            "name": "drug_mmp14_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_alpha"
            ]
          },
          {
            "name": "drug_stat3_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_beta"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_weighted",
            "role": "additive",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "disease_associated_fibroblast_subtype_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "value",
            "op": "in",
            "expected": [
              "subtype_lambda",
              "subtype_epsilon",
              "subtype_iota",
              "subtype_beta",
              "subtype_eta",
              "subtype_zeta",
              "subtype_not_assessed"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "simple_de_ranking_by_log_fc",
              "marker_based_de_only",
              "not_computed"
            ]
          },
          {
            "name": "top_master_regulator_transcription_factors_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_not_computed"
            ]
          },
          {
            "name": "top_druggable_proteins_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_not_assessed"
            ]
          },
          {
            "name": "cross_axis_circular_marker_in_druggable_proteins",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_gamma",
              "drug_target_delta",
              "drug_target_eta"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-glm-5p2-pi",
      "modelName": "glm-5p2",
      "harness": "pi",
      "task": "gse171524_pathological_fibroblast_master_regulator_druggable_targets",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 5395,
      "cost": 3.9112554200000003,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "disease_associated_fibroblast_subtype_weighted",
            "role": "additive",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "mr_tf_junb_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_alpha"
            ]
          },
          {
            "name": "mr_tf_jund_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_beta"
            ]
          },
          {
            "name": "drug_mmp14_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_alpha"
            ]
          },
          {
            "name": "drug_stat3_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_beta"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_weighted",
            "role": "additive",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "disease_associated_fibroblast_subtype_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "value",
            "op": "in",
            "expected": [
              "subtype_lambda",
              "subtype_epsilon",
              "subtype_iota",
              "subtype_beta",
              "subtype_eta",
              "subtype_zeta",
              "subtype_not_assessed"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "simple_de_ranking_by_log_fc",
              "marker_based_de_only",
              "not_computed"
            ]
          },
          {
            "name": "top_master_regulator_transcription_factors_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_not_computed"
            ]
          },
          {
            "name": "top_druggable_proteins_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_not_assessed"
            ]
          },
          {
            "name": "cross_axis_circular_marker_in_druggable_proteins",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_gamma",
              "drug_target_delta",
              "drug_target_eta"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-4-codex",
      "modelName": "gpt-5.4",
      "harness": "openai-codex",
      "task": "gse171524_pathological_fibroblast_master_regulator_druggable_targets",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 1058,
      "cost": 1.5248465,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "disease_associated_fibroblast_subtype_weighted",
            "role": "additive",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "mr_tf_junb_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_alpha"
            ]
          },
          {
            "name": "mr_tf_jund_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_beta"
            ]
          },
          {
            "name": "drug_mmp14_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_alpha"
            ]
          },
          {
            "name": "drug_stat3_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_beta"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_weighted",
            "role": "additive",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "disease_associated_fibroblast_subtype_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "value",
            "op": "in",
            "expected": [
              "subtype_lambda",
              "subtype_epsilon",
              "subtype_iota",
              "subtype_beta",
              "subtype_eta",
              "subtype_zeta",
              "subtype_not_assessed"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "simple_de_ranking_by_log_fc",
              "marker_based_de_only",
              "not_computed"
            ]
          },
          {
            "name": "top_master_regulator_transcription_factors_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_not_computed"
            ]
          },
          {
            "name": "top_druggable_proteins_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_not_assessed"
            ]
          },
          {
            "name": "cross_axis_circular_marker_in_druggable_proteins",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_gamma",
              "drug_target_delta",
              "drug_target_eta"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-4-codex",
      "modelName": "gpt-5.4",
      "harness": "openai-codex",
      "task": "gse171524_pathological_fibroblast_master_regulator_druggable_targets",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 1537,
      "cost": 2.267563,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "disease_associated_fibroblast_subtype_weighted",
            "role": "additive",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "mr_tf_junb_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_alpha"
            ]
          },
          {
            "name": "mr_tf_jund_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_beta"
            ]
          },
          {
            "name": "drug_mmp14_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_alpha"
            ]
          },
          {
            "name": "drug_stat3_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_beta"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_weighted",
            "role": "additive",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "disease_associated_fibroblast_subtype_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "value",
            "op": "in",
            "expected": [
              "subtype_lambda",
              "subtype_epsilon",
              "subtype_iota",
              "subtype_beta",
              "subtype_eta",
              "subtype_zeta",
              "subtype_not_assessed"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "simple_de_ranking_by_log_fc",
              "marker_based_de_only",
              "not_computed"
            ]
          },
          {
            "name": "top_master_regulator_transcription_factors_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_not_computed"
            ]
          },
          {
            "name": "top_druggable_proteins_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_not_assessed"
            ]
          },
          {
            "name": "cross_axis_circular_marker_in_druggable_proteins",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_gamma",
              "drug_target_delta",
              "drug_target_eta"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-4-codex",
      "modelName": "gpt-5.4",
      "harness": "openai-codex",
      "task": "gse171524_pathological_fibroblast_master_regulator_druggable_targets",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1208,
      "cost": 2.9872905,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "disease_associated_fibroblast_subtype_weighted",
            "role": "additive",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "mr_tf_junb_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_alpha"
            ]
          },
          {
            "name": "mr_tf_jund_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_beta"
            ]
          },
          {
            "name": "drug_mmp14_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_alpha"
            ]
          },
          {
            "name": "drug_stat3_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_beta"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_weighted",
            "role": "additive",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "disease_associated_fibroblast_subtype_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "value",
            "op": "in",
            "expected": [
              "subtype_lambda",
              "subtype_epsilon",
              "subtype_iota",
              "subtype_beta",
              "subtype_eta",
              "subtype_zeta",
              "subtype_not_assessed"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "simple_de_ranking_by_log_fc",
              "marker_based_de_only",
              "not_computed"
            ]
          },
          {
            "name": "top_master_regulator_transcription_factors_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_not_computed"
            ]
          },
          {
            "name": "top_druggable_proteins_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_not_assessed"
            ]
          },
          {
            "name": "cross_axis_circular_marker_in_druggable_proteins",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_gamma",
              "drug_target_delta",
              "drug_target_eta"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-4-pi",
      "modelName": "gpt-5.4",
      "harness": "pi",
      "task": "gse171524_pathological_fibroblast_master_regulator_druggable_targets",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 2872,
      "cost": 1.8472834999999999,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "disease_associated_fibroblast_subtype_weighted",
            "role": "additive",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "mr_tf_junb_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_alpha"
            ]
          },
          {
            "name": "mr_tf_jund_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_beta"
            ]
          },
          {
            "name": "drug_mmp14_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_alpha"
            ]
          },
          {
            "name": "drug_stat3_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_beta"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_weighted",
            "role": "additive",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "disease_associated_fibroblast_subtype_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "value",
            "op": "in",
            "expected": [
              "subtype_lambda",
              "subtype_epsilon",
              "subtype_iota",
              "subtype_beta",
              "subtype_eta",
              "subtype_zeta",
              "subtype_not_assessed"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "simple_de_ranking_by_log_fc",
              "marker_based_de_only",
              "not_computed"
            ]
          },
          {
            "name": "top_master_regulator_transcription_factors_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_not_computed"
            ]
          },
          {
            "name": "top_druggable_proteins_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_not_assessed"
            ]
          },
          {
            "name": "cross_axis_circular_marker_in_druggable_proteins",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_gamma",
              "drug_target_delta",
              "drug_target_eta"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-4-pi",
      "modelName": "gpt-5.4",
      "harness": "pi",
      "task": "gse171524_pathological_fibroblast_master_regulator_druggable_targets",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 4857,
      "cost": 1.2731519999999998,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "disease_associated_fibroblast_subtype_weighted",
            "role": "additive",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "mr_tf_junb_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_alpha"
            ]
          },
          {
            "name": "mr_tf_jund_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_beta"
            ]
          },
          {
            "name": "drug_mmp14_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_alpha"
            ]
          },
          {
            "name": "drug_stat3_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_beta"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_weighted",
            "role": "additive",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "disease_associated_fibroblast_subtype_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "value",
            "op": "in",
            "expected": [
              "subtype_lambda",
              "subtype_epsilon",
              "subtype_iota",
              "subtype_beta",
              "subtype_eta",
              "subtype_zeta",
              "subtype_not_assessed"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "simple_de_ranking_by_log_fc",
              "marker_based_de_only",
              "not_computed"
            ]
          },
          {
            "name": "top_master_regulator_transcription_factors_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_not_computed"
            ]
          },
          {
            "name": "top_druggable_proteins_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_not_assessed"
            ]
          },
          {
            "name": "cross_axis_circular_marker_in_druggable_proteins",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_gamma",
              "drug_target_delta",
              "drug_target_eta"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-4-pi",
      "modelName": "gpt-5.4",
      "harness": "pi",
      "task": "gse171524_pathological_fibroblast_master_regulator_druggable_targets",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 3262,
      "cost": 1.504473,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "disease_associated_fibroblast_subtype_weighted",
            "role": "additive",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "mr_tf_junb_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_alpha"
            ]
          },
          {
            "name": "mr_tf_jund_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_beta"
            ]
          },
          {
            "name": "drug_mmp14_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_alpha"
            ]
          },
          {
            "name": "drug_stat3_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_beta"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_weighted",
            "role": "additive",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "disease_associated_fibroblast_subtype_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "value",
            "op": "in",
            "expected": [
              "subtype_lambda",
              "subtype_epsilon",
              "subtype_iota",
              "subtype_beta",
              "subtype_eta",
              "subtype_zeta",
              "subtype_not_assessed"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "simple_de_ranking_by_log_fc",
              "marker_based_de_only",
              "not_computed"
            ]
          },
          {
            "name": "top_master_regulator_transcription_factors_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_not_computed"
            ]
          },
          {
            "name": "top_druggable_proteins_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_not_assessed"
            ]
          },
          {
            "name": "cross_axis_circular_marker_in_druggable_proteins",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_gamma",
              "drug_target_delta",
              "drug_target_eta"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-5-codex",
      "modelName": "gpt-5.5",
      "harness": "openai-codex",
      "task": "gse171524_pathological_fibroblast_master_regulator_druggable_targets",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 1,
      "passed": true,
      "score": null,
      "durationS": 735,
      "cost": 2.106758,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "disease_associated_fibroblast_subtype_weighted",
            "role": "additive",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "mr_tf_junb_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_alpha"
            ]
          },
          {
            "name": "mr_tf_jund_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_beta"
            ]
          },
          {
            "name": "drug_mmp14_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_alpha"
            ]
          },
          {
            "name": "drug_stat3_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_beta"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_weighted",
            "role": "additive",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "disease_associated_fibroblast_subtype_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "value",
            "op": "in",
            "expected": [
              "subtype_lambda",
              "subtype_epsilon",
              "subtype_iota",
              "subtype_beta",
              "subtype_eta",
              "subtype_zeta",
              "subtype_not_assessed"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "simple_de_ranking_by_log_fc",
              "marker_based_de_only",
              "not_computed"
            ]
          },
          {
            "name": "top_master_regulator_transcription_factors_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_not_computed"
            ]
          },
          {
            "name": "top_druggable_proteins_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_not_assessed"
            ]
          },
          {
            "name": "cross_axis_circular_marker_in_druggable_proteins",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_gamma",
              "drug_target_delta",
              "drug_target_eta"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-5-codex",
      "modelName": "gpt-5.5",
      "harness": "openai-codex",
      "task": "gse171524_pathological_fibroblast_master_regulator_druggable_targets",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 2,
      "passed": true,
      "score": null,
      "durationS": 1375,
      "cost": 2.6001390000000004,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "disease_associated_fibroblast_subtype_weighted",
            "role": "additive",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "mr_tf_junb_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_alpha"
            ]
          },
          {
            "name": "mr_tf_jund_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_beta"
            ]
          },
          {
            "name": "drug_mmp14_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_alpha"
            ]
          },
          {
            "name": "drug_stat3_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_beta"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_weighted",
            "role": "additive",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "disease_associated_fibroblast_subtype_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "value",
            "op": "in",
            "expected": [
              "subtype_lambda",
              "subtype_epsilon",
              "subtype_iota",
              "subtype_beta",
              "subtype_eta",
              "subtype_zeta",
              "subtype_not_assessed"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "simple_de_ranking_by_log_fc",
              "marker_based_de_only",
              "not_computed"
            ]
          },
          {
            "name": "top_master_regulator_transcription_factors_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_not_computed"
            ]
          },
          {
            "name": "top_druggable_proteins_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_not_assessed"
            ]
          },
          {
            "name": "cross_axis_circular_marker_in_druggable_proteins",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_gamma",
              "drug_target_delta",
              "drug_target_eta"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-5-codex",
      "modelName": "gpt-5.5",
      "harness": "openai-codex",
      "task": "gse171524_pathological_fibroblast_master_regulator_druggable_targets",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 3,
      "passed": true,
      "score": null,
      "durationS": 800,
      "cost": 1.8783690000000002,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "disease_associated_fibroblast_subtype_weighted",
            "role": "additive",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "mr_tf_junb_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_alpha"
            ]
          },
          {
            "name": "mr_tf_jund_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_beta"
            ]
          },
          {
            "name": "drug_mmp14_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_alpha"
            ]
          },
          {
            "name": "drug_stat3_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_beta"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_weighted",
            "role": "additive",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "disease_associated_fibroblast_subtype_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "value",
            "op": "in",
            "expected": [
              "subtype_lambda",
              "subtype_epsilon",
              "subtype_iota",
              "subtype_beta",
              "subtype_eta",
              "subtype_zeta",
              "subtype_not_assessed"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "simple_de_ranking_by_log_fc",
              "marker_based_de_only",
              "not_computed"
            ]
          },
          {
            "name": "top_master_regulator_transcription_factors_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_not_computed"
            ]
          },
          {
            "name": "top_druggable_proteins_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_not_assessed"
            ]
          },
          {
            "name": "cross_axis_circular_marker_in_druggable_proteins",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_gamma",
              "drug_target_delta",
              "drug_target_eta"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-5-pi",
      "modelName": "gpt-5.5",
      "harness": "pi",
      "task": "gse171524_pathological_fibroblast_master_regulator_druggable_targets",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 1926,
      "cost": 2.5609030000000006,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "disease_associated_fibroblast_subtype_weighted",
            "role": "additive",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "mr_tf_junb_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_alpha"
            ]
          },
          {
            "name": "mr_tf_jund_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_beta"
            ]
          },
          {
            "name": "drug_mmp14_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_alpha"
            ]
          },
          {
            "name": "drug_stat3_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_beta"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_weighted",
            "role": "additive",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "disease_associated_fibroblast_subtype_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "value",
            "op": "in",
            "expected": [
              "subtype_lambda",
              "subtype_epsilon",
              "subtype_iota",
              "subtype_beta",
              "subtype_eta",
              "subtype_zeta",
              "subtype_not_assessed"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "simple_de_ranking_by_log_fc",
              "marker_based_de_only",
              "not_computed"
            ]
          },
          {
            "name": "top_master_regulator_transcription_factors_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_not_computed"
            ]
          },
          {
            "name": "top_druggable_proteins_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_not_assessed"
            ]
          },
          {
            "name": "cross_axis_circular_marker_in_druggable_proteins",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_gamma",
              "drug_target_delta",
              "drug_target_eta"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-5-pi",
      "modelName": "gpt-5.5",
      "harness": "pi",
      "task": "gse171524_pathological_fibroblast_master_regulator_druggable_targets",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 2,
      "passed": true,
      "score": null,
      "durationS": 3731,
      "cost": 2.0601959999999995,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "disease_associated_fibroblast_subtype_weighted",
            "role": "additive",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "mr_tf_junb_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_alpha"
            ]
          },
          {
            "name": "mr_tf_jund_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_beta"
            ]
          },
          {
            "name": "drug_mmp14_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_alpha"
            ]
          },
          {
            "name": "drug_stat3_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_beta"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_weighted",
            "role": "additive",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "disease_associated_fibroblast_subtype_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "value",
            "op": "in",
            "expected": [
              "subtype_lambda",
              "subtype_epsilon",
              "subtype_iota",
              "subtype_beta",
              "subtype_eta",
              "subtype_zeta",
              "subtype_not_assessed"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "simple_de_ranking_by_log_fc",
              "marker_based_de_only",
              "not_computed"
            ]
          },
          {
            "name": "top_master_regulator_transcription_factors_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_not_computed"
            ]
          },
          {
            "name": "top_druggable_proteins_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_not_assessed"
            ]
          },
          {
            "name": "cross_axis_circular_marker_in_druggable_proteins",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_gamma",
              "drug_target_delta",
              "drug_target_eta"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-5-pi",
      "modelName": "gpt-5.5",
      "harness": "pi",
      "task": "gse171524_pathological_fibroblast_master_regulator_druggable_targets",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 3,
      "passed": true,
      "score": null,
      "durationS": 5145,
      "cost": 2.472323,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "disease_associated_fibroblast_subtype_weighted",
            "role": "additive",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "mr_tf_junb_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_alpha"
            ]
          },
          {
            "name": "mr_tf_jund_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_beta"
            ]
          },
          {
            "name": "drug_mmp14_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_alpha"
            ]
          },
          {
            "name": "drug_stat3_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_beta"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_weighted",
            "role": "additive",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "disease_associated_fibroblast_subtype_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "value",
            "op": "in",
            "expected": [
              "subtype_lambda",
              "subtype_epsilon",
              "subtype_iota",
              "subtype_beta",
              "subtype_eta",
              "subtype_zeta",
              "subtype_not_assessed"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "simple_de_ranking_by_log_fc",
              "marker_based_de_only",
              "not_computed"
            ]
          },
          {
            "name": "top_master_regulator_transcription_factors_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_not_computed"
            ]
          },
          {
            "name": "top_druggable_proteins_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_not_assessed"
            ]
          },
          {
            "name": "cross_axis_circular_marker_in_druggable_proteins",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_gamma",
              "drug_target_delta",
              "drug_target_eta"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-20-beta-reasoning-pi",
      "modelName": "grok-4.20-beta-0309-reasoning",
      "harness": "pi",
      "task": "gse171524_pathological_fibroblast_master_regulator_druggable_targets",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 427,
      "cost": 0.07006115000000002,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "disease_associated_fibroblast_subtype_weighted",
            "role": "additive",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "mr_tf_junb_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_alpha"
            ]
          },
          {
            "name": "mr_tf_jund_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_beta"
            ]
          },
          {
            "name": "drug_mmp14_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_alpha"
            ]
          },
          {
            "name": "drug_stat3_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_beta"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_weighted",
            "role": "additive",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "disease_associated_fibroblast_subtype_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "value",
            "op": "in",
            "expected": [
              "subtype_lambda",
              "subtype_epsilon",
              "subtype_iota",
              "subtype_beta",
              "subtype_eta",
              "subtype_zeta",
              "subtype_not_assessed"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "simple_de_ranking_by_log_fc",
              "marker_based_de_only",
              "not_computed"
            ]
          },
          {
            "name": "top_master_regulator_transcription_factors_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_not_computed"
            ]
          },
          {
            "name": "top_druggable_proteins_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_not_assessed"
            ]
          },
          {
            "name": "cross_axis_circular_marker_in_druggable_proteins",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_gamma",
              "drug_target_delta",
              "drug_target_eta"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-20-beta-reasoning-pi",
      "modelName": "grok-4.20-beta-0309-reasoning",
      "harness": "pi",
      "task": "gse171524_pathological_fibroblast_master_regulator_druggable_targets",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 2,
      "passed": true,
      "score": null,
      "durationS": 879,
      "cost": 0.0364192,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "disease_associated_fibroblast_subtype_weighted",
            "role": "additive",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "mr_tf_junb_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_alpha"
            ]
          },
          {
            "name": "mr_tf_jund_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_beta"
            ]
          },
          {
            "name": "drug_mmp14_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_alpha"
            ]
          },
          {
            "name": "drug_stat3_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_beta"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_weighted",
            "role": "additive",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "disease_associated_fibroblast_subtype_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "value",
            "op": "in",
            "expected": [
              "subtype_lambda",
              "subtype_epsilon",
              "subtype_iota",
              "subtype_beta",
              "subtype_eta",
              "subtype_zeta",
              "subtype_not_assessed"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "simple_de_ranking_by_log_fc",
              "marker_based_de_only",
              "not_computed"
            ]
          },
          {
            "name": "top_master_regulator_transcription_factors_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_not_computed"
            ]
          },
          {
            "name": "top_druggable_proteins_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_not_assessed"
            ]
          },
          {
            "name": "cross_axis_circular_marker_in_druggable_proteins",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_gamma",
              "drug_target_delta",
              "drug_target_eta"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-20-beta-reasoning-pi",
      "modelName": "grok-4.20-beta-0309-reasoning",
      "harness": "pi",
      "task": "gse171524_pathological_fibroblast_master_regulator_druggable_targets",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 3,
      "passed": true,
      "score": null,
      "durationS": 1502,
      "cost": 0.11526995000000002,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "disease_associated_fibroblast_subtype_weighted",
            "role": "additive",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "mr_tf_junb_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_alpha"
            ]
          },
          {
            "name": "mr_tf_jund_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_beta"
            ]
          },
          {
            "name": "drug_mmp14_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_alpha"
            ]
          },
          {
            "name": "drug_stat3_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_beta"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_weighted",
            "role": "additive",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "disease_associated_fibroblast_subtype_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "value",
            "op": "in",
            "expected": [
              "subtype_lambda",
              "subtype_epsilon",
              "subtype_iota",
              "subtype_beta",
              "subtype_eta",
              "subtype_zeta",
              "subtype_not_assessed"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "simple_de_ranking_by_log_fc",
              "marker_based_de_only",
              "not_computed"
            ]
          },
          {
            "name": "top_master_regulator_transcription_factors_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_not_computed"
            ]
          },
          {
            "name": "top_druggable_proteins_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_not_assessed"
            ]
          },
          {
            "name": "cross_axis_circular_marker_in_druggable_proteins",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_gamma",
              "drug_target_delta",
              "drug_target_eta"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-3-pi",
      "modelName": "grok-4.3",
      "harness": "pi",
      "task": "gse171524_pathological_fibroblast_master_regulator_druggable_targets",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 48,
      "cost": 0.01753725,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "disease_associated_fibroblast_subtype_weighted",
            "role": "additive",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "mr_tf_junb_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_alpha"
            ]
          },
          {
            "name": "mr_tf_jund_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_beta"
            ]
          },
          {
            "name": "drug_mmp14_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_alpha"
            ]
          },
          {
            "name": "drug_stat3_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_beta"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_weighted",
            "role": "additive",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "disease_associated_fibroblast_subtype_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "value",
            "op": "in",
            "expected": [
              "subtype_lambda",
              "subtype_epsilon",
              "subtype_iota",
              "subtype_beta",
              "subtype_eta",
              "subtype_zeta",
              "subtype_not_assessed"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "simple_de_ranking_by_log_fc",
              "marker_based_de_only",
              "not_computed"
            ]
          },
          {
            "name": "top_master_regulator_transcription_factors_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_not_computed"
            ]
          },
          {
            "name": "top_druggable_proteins_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_not_assessed"
            ]
          },
          {
            "name": "cross_axis_circular_marker_in_druggable_proteins",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_gamma",
              "drug_target_delta",
              "drug_target_eta"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-3-pi",
      "modelName": "grok-4.3",
      "harness": "pi",
      "task": "gse171524_pathological_fibroblast_master_regulator_druggable_targets",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 2,
      "passed": true,
      "score": null,
      "durationS": 31,
      "cost": 0.0098044,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "disease_associated_fibroblast_subtype_weighted",
            "role": "additive",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "mr_tf_junb_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_alpha"
            ]
          },
          {
            "name": "mr_tf_jund_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_beta"
            ]
          },
          {
            "name": "drug_mmp14_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_alpha"
            ]
          },
          {
            "name": "drug_stat3_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_beta"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_weighted",
            "role": "additive",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "disease_associated_fibroblast_subtype_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "value",
            "op": "in",
            "expected": [
              "subtype_lambda",
              "subtype_epsilon",
              "subtype_iota",
              "subtype_beta",
              "subtype_eta",
              "subtype_zeta",
              "subtype_not_assessed"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "simple_de_ranking_by_log_fc",
              "marker_based_de_only",
              "not_computed"
            ]
          },
          {
            "name": "top_master_regulator_transcription_factors_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_not_computed"
            ]
          },
          {
            "name": "top_druggable_proteins_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_not_assessed"
            ]
          },
          {
            "name": "cross_axis_circular_marker_in_druggable_proteins",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_gamma",
              "drug_target_delta",
              "drug_target_eta"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-3-pi",
      "modelName": "grok-4.3",
      "harness": "pi",
      "task": "gse171524_pathological_fibroblast_master_regulator_druggable_targets",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 64,
      "cost": 0.01977985,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "disease_associated_fibroblast_subtype_weighted",
            "role": "additive",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "mr_tf_junb_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_alpha"
            ]
          },
          {
            "name": "mr_tf_jund_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_beta"
            ]
          },
          {
            "name": "drug_mmp14_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_alpha"
            ]
          },
          {
            "name": "drug_stat3_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_beta"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_weighted",
            "role": "additive",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "disease_associated_fibroblast_subtype_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "value",
            "op": "in",
            "expected": [
              "subtype_lambda",
              "subtype_epsilon",
              "subtype_iota",
              "subtype_beta",
              "subtype_eta",
              "subtype_zeta",
              "subtype_not_assessed"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "simple_de_ranking_by_log_fc",
              "marker_based_de_only",
              "not_computed"
            ]
          },
          {
            "name": "top_master_regulator_transcription_factors_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_not_computed"
            ]
          },
          {
            "name": "top_druggable_proteins_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_not_assessed"
            ]
          },
          {
            "name": "cross_axis_circular_marker_in_druggable_proteins",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_gamma",
              "drug_target_delta",
              "drug_target_eta"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-kimi-k2p6-pi",
      "modelName": "kimi-k2p6",
      "harness": "pi",
      "task": "gse171524_pathological_fibroblast_master_regulator_druggable_targets",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 894,
      "cost": 0.19059751000000003,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "disease_associated_fibroblast_subtype_weighted",
            "role": "additive",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "mr_tf_junb_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_alpha"
            ]
          },
          {
            "name": "mr_tf_jund_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_beta"
            ]
          },
          {
            "name": "drug_mmp14_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_alpha"
            ]
          },
          {
            "name": "drug_stat3_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_beta"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_weighted",
            "role": "additive",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "disease_associated_fibroblast_subtype_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "value",
            "op": "in",
            "expected": [
              "subtype_lambda",
              "subtype_epsilon",
              "subtype_iota",
              "subtype_beta",
              "subtype_eta",
              "subtype_zeta",
              "subtype_not_assessed"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "simple_de_ranking_by_log_fc",
              "marker_based_de_only",
              "not_computed"
            ]
          },
          {
            "name": "top_master_regulator_transcription_factors_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_not_computed"
            ]
          },
          {
            "name": "top_druggable_proteins_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_not_assessed"
            ]
          },
          {
            "name": "cross_axis_circular_marker_in_druggable_proteins",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_gamma",
              "drug_target_delta",
              "drug_target_eta"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-kimi-k2p6-pi",
      "modelName": "kimi-k2p6",
      "harness": "pi",
      "task": "gse171524_pathological_fibroblast_master_regulator_druggable_targets",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 4566,
      "cost": 0.9794080300000001,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "disease_associated_fibroblast_subtype_weighted",
            "role": "additive",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "mr_tf_junb_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_alpha"
            ]
          },
          {
            "name": "mr_tf_jund_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_beta"
            ]
          },
          {
            "name": "drug_mmp14_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_alpha"
            ]
          },
          {
            "name": "drug_stat3_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_beta"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_weighted",
            "role": "additive",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "disease_associated_fibroblast_subtype_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "value",
            "op": "in",
            "expected": [
              "subtype_lambda",
              "subtype_epsilon",
              "subtype_iota",
              "subtype_beta",
              "subtype_eta",
              "subtype_zeta",
              "subtype_not_assessed"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "simple_de_ranking_by_log_fc",
              "marker_based_de_only",
              "not_computed"
            ]
          },
          {
            "name": "top_master_regulator_transcription_factors_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_not_computed"
            ]
          },
          {
            "name": "top_druggable_proteins_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_not_assessed"
            ]
          },
          {
            "name": "cross_axis_circular_marker_in_druggable_proteins",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_gamma",
              "drug_target_delta",
              "drug_target_eta"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-kimi-k2p6-pi",
      "modelName": "kimi-k2p6",
      "harness": "pi",
      "task": "gse171524_pathological_fibroblast_master_regulator_druggable_targets",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 2321,
      "cost": 0.8116164499999999,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "disease_associated_fibroblast_subtype_weighted",
            "role": "additive",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "mr_tf_junb_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_alpha"
            ]
          },
          {
            "name": "mr_tf_jund_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_beta"
            ]
          },
          {
            "name": "drug_mmp14_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_alpha"
            ]
          },
          {
            "name": "drug_stat3_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_beta"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_weighted",
            "role": "additive",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "disease_associated_fibroblast_subtype_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "value",
            "op": "in",
            "expected": [
              "subtype_lambda",
              "subtype_epsilon",
              "subtype_iota",
              "subtype_beta",
              "subtype_eta",
              "subtype_zeta",
              "subtype_not_assessed"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "simple_de_ranking_by_log_fc",
              "marker_based_de_only",
              "not_computed"
            ]
          },
          {
            "name": "top_master_regulator_transcription_factors_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_not_computed"
            ]
          },
          {
            "name": "top_druggable_proteins_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_not_assessed"
            ]
          },
          {
            "name": "cross_axis_circular_marker_in_druggable_proteins",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_gamma",
              "drug_target_delta",
              "drug_target_eta"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-6-claude-code",
      "modelName": "claude-opus-4-6",
      "harness": "claude-code",
      "task": "gse171524_plasma_bcr_neutralizing_clonotype",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 297,
      "cost": 0.736701,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dominant_pairing_heavy_gate",
            "role": "gate",
            "field": "$.dominant_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV4-34"
            ]
          },
          {
            "name": "dominant_pairing_light_gate",
            "role": "gate",
            "field": "$.dominant_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGLV3-19"
            ]
          },
          {
            "name": "dominant_neutralizer_match_gate",
            "role": "gate",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_known_neutralizer_match"
            ]
          },
          {
            "name": "neutralizer_pairing_heavy_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV1-18"
            ]
          },
          {
            "name": "neutralizer_pairing_light_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGKV3-20"
            ]
          },
          {
            "name": "neutralizer_recurrence_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "observed_in_2_to_5_donors"
            ]
          },
          {
            "name": "neutralizer_rank_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "top_5_pairing",
              "top_6_to_20_pairing"
            ]
          },
          {
            "name": "dominant_clone_isotype_gate",
            "role": "gate",
            "field": "$.dominant_clone_isotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "IgG"
            ]
          },
          {
            "name": "blineage_class_switch_contrast_gate",
            "role": "gate",
            "field": "$.blineage_class_switch_contrast",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_class_switched_control_higher_unswitched",
              "class_switched_in_both_cohorts"
            ]
          },
          {
            "name": "dominant_match_narrative_trap_or_null",
            "role": "hard_fail",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_rbd_neutralizer",
              "matches_known_autoreactive_clone",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_heavy",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_light",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_recurrence_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_rank_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "cross_axis_found_dominant_but_failed_to_scan_for_embedded_neutralizer",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_pairing_neutralizer_match": {
                "op": "in",
                "values": [
                  "no_known_neutralizer_match"
                ]
              },
              "neutralizer_pairing_heavy": {
                "op": "in",
                "values": [
                  "none_detected"
                ]
              }
            }
          },
          {
            "name": "neutralizer_pairing_not_exactly_the_s309_pair",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "op": "not",
            "expected": {
              "op": "and",
              "args": [
                {
                  "op": "field",
                  "name": "neutralizer_pairing_heavy",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGHV1-18"
                    ]
                  }
                },
                {
                  "op": "field",
                  "name": "neutralizer_pairing_light",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGKV3-20"
                    ]
                  }
                }
              ]
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-6-claude-code",
      "modelName": "claude-opus-4-6",
      "harness": "claude-code",
      "task": "gse171524_plasma_bcr_neutralizing_clonotype",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 266,
      "cost": 0.6950230000000001,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dominant_pairing_heavy_gate",
            "role": "gate",
            "field": "$.dominant_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV4-34"
            ]
          },
          {
            "name": "dominant_pairing_light_gate",
            "role": "gate",
            "field": "$.dominant_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGLV3-19"
            ]
          },
          {
            "name": "dominant_neutralizer_match_gate",
            "role": "gate",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_known_neutralizer_match"
            ]
          },
          {
            "name": "neutralizer_pairing_heavy_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV1-18"
            ]
          },
          {
            "name": "neutralizer_pairing_light_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGKV3-20"
            ]
          },
          {
            "name": "neutralizer_recurrence_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "observed_in_2_to_5_donors"
            ]
          },
          {
            "name": "neutralizer_rank_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "top_5_pairing",
              "top_6_to_20_pairing"
            ]
          },
          {
            "name": "dominant_clone_isotype_gate",
            "role": "gate",
            "field": "$.dominant_clone_isotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "IgG"
            ]
          },
          {
            "name": "blineage_class_switch_contrast_gate",
            "role": "gate",
            "field": "$.blineage_class_switch_contrast",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_class_switched_control_higher_unswitched",
              "class_switched_in_both_cohorts"
            ]
          },
          {
            "name": "dominant_match_narrative_trap_or_null",
            "role": "hard_fail",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_rbd_neutralizer",
              "matches_known_autoreactive_clone",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_heavy",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_light",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_recurrence_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_rank_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "cross_axis_found_dominant_but_failed_to_scan_for_embedded_neutralizer",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_pairing_neutralizer_match": {
                "op": "in",
                "values": [
                  "no_known_neutralizer_match"
                ]
              },
              "neutralizer_pairing_heavy": {
                "op": "in",
                "values": [
                  "none_detected"
                ]
              }
            }
          },
          {
            "name": "neutralizer_pairing_not_exactly_the_s309_pair",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "op": "not",
            "expected": {
              "op": "and",
              "args": [
                {
                  "op": "field",
                  "name": "neutralizer_pairing_heavy",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGHV1-18"
                    ]
                  }
                },
                {
                  "op": "field",
                  "name": "neutralizer_pairing_light",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGKV3-20"
                    ]
                  }
                }
              ]
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-6-claude-code",
      "modelName": "claude-opus-4-6",
      "harness": "claude-code",
      "task": "gse171524_plasma_bcr_neutralizing_clonotype",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 296,
      "cost": 0.67309675,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dominant_pairing_heavy_gate",
            "role": "gate",
            "field": "$.dominant_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV4-34"
            ]
          },
          {
            "name": "dominant_pairing_light_gate",
            "role": "gate",
            "field": "$.dominant_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGLV3-19"
            ]
          },
          {
            "name": "dominant_neutralizer_match_gate",
            "role": "gate",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_known_neutralizer_match"
            ]
          },
          {
            "name": "neutralizer_pairing_heavy_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV1-18"
            ]
          },
          {
            "name": "neutralizer_pairing_light_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGKV3-20"
            ]
          },
          {
            "name": "neutralizer_recurrence_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "observed_in_2_to_5_donors"
            ]
          },
          {
            "name": "neutralizer_rank_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "top_5_pairing",
              "top_6_to_20_pairing"
            ]
          },
          {
            "name": "dominant_clone_isotype_gate",
            "role": "gate",
            "field": "$.dominant_clone_isotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "IgG"
            ]
          },
          {
            "name": "blineage_class_switch_contrast_gate",
            "role": "gate",
            "field": "$.blineage_class_switch_contrast",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_class_switched_control_higher_unswitched",
              "class_switched_in_both_cohorts"
            ]
          },
          {
            "name": "dominant_match_narrative_trap_or_null",
            "role": "hard_fail",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_rbd_neutralizer",
              "matches_known_autoreactive_clone",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_heavy",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_light",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_recurrence_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_rank_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "cross_axis_found_dominant_but_failed_to_scan_for_embedded_neutralizer",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_pairing_neutralizer_match": {
                "op": "in",
                "values": [
                  "no_known_neutralizer_match"
                ]
              },
              "neutralizer_pairing_heavy": {
                "op": "in",
                "values": [
                  "none_detected"
                ]
              }
            }
          },
          {
            "name": "neutralizer_pairing_not_exactly_the_s309_pair",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "op": "not",
            "expected": {
              "op": "and",
              "args": [
                {
                  "op": "field",
                  "name": "neutralizer_pairing_heavy",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGHV1-18"
                    ]
                  }
                },
                {
                  "op": "field",
                  "name": "neutralizer_pairing_light",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGKV3-20"
                    ]
                  }
                }
              ]
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-6-pi",
      "modelName": "claude-opus-4-6",
      "harness": "pi",
      "task": "gse171524_plasma_bcr_neutralizing_clonotype",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 561,
      "cost": 1.2024375,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dominant_pairing_heavy_gate",
            "role": "gate",
            "field": "$.dominant_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV4-34"
            ]
          },
          {
            "name": "dominant_pairing_light_gate",
            "role": "gate",
            "field": "$.dominant_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGLV3-19"
            ]
          },
          {
            "name": "dominant_neutralizer_match_gate",
            "role": "gate",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_known_neutralizer_match"
            ]
          },
          {
            "name": "neutralizer_pairing_heavy_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV1-18"
            ]
          },
          {
            "name": "neutralizer_pairing_light_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGKV3-20"
            ]
          },
          {
            "name": "neutralizer_recurrence_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "observed_in_2_to_5_donors"
            ]
          },
          {
            "name": "neutralizer_rank_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "top_5_pairing",
              "top_6_to_20_pairing"
            ]
          },
          {
            "name": "dominant_clone_isotype_gate",
            "role": "gate",
            "field": "$.dominant_clone_isotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "IgG"
            ]
          },
          {
            "name": "blineage_class_switch_contrast_gate",
            "role": "gate",
            "field": "$.blineage_class_switch_contrast",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_class_switched_control_higher_unswitched",
              "class_switched_in_both_cohorts"
            ]
          },
          {
            "name": "dominant_match_narrative_trap_or_null",
            "role": "hard_fail",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_rbd_neutralizer",
              "matches_known_autoreactive_clone",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_heavy",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_light",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_recurrence_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_rank_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "cross_axis_found_dominant_but_failed_to_scan_for_embedded_neutralizer",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_pairing_neutralizer_match": {
                "op": "in",
                "values": [
                  "no_known_neutralizer_match"
                ]
              },
              "neutralizer_pairing_heavy": {
                "op": "in",
                "values": [
                  "none_detected"
                ]
              }
            }
          },
          {
            "name": "neutralizer_pairing_not_exactly_the_s309_pair",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "op": "not",
            "expected": {
              "op": "and",
              "args": [
                {
                  "op": "field",
                  "name": "neutralizer_pairing_heavy",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGHV1-18"
                    ]
                  }
                },
                {
                  "op": "field",
                  "name": "neutralizer_pairing_light",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGKV3-20"
                    ]
                  }
                }
              ]
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-6-pi",
      "modelName": "claude-opus-4-6",
      "harness": "pi",
      "task": "gse171524_plasma_bcr_neutralizing_clonotype",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 393,
      "cost": 0.7831677500000002,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dominant_pairing_heavy_gate",
            "role": "gate",
            "field": "$.dominant_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV4-34"
            ]
          },
          {
            "name": "dominant_pairing_light_gate",
            "role": "gate",
            "field": "$.dominant_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGLV3-19"
            ]
          },
          {
            "name": "dominant_neutralizer_match_gate",
            "role": "gate",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_known_neutralizer_match"
            ]
          },
          {
            "name": "neutralizer_pairing_heavy_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV1-18"
            ]
          },
          {
            "name": "neutralizer_pairing_light_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGKV3-20"
            ]
          },
          {
            "name": "neutralizer_recurrence_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "observed_in_2_to_5_donors"
            ]
          },
          {
            "name": "neutralizer_rank_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "top_5_pairing",
              "top_6_to_20_pairing"
            ]
          },
          {
            "name": "dominant_clone_isotype_gate",
            "role": "gate",
            "field": "$.dominant_clone_isotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "IgG"
            ]
          },
          {
            "name": "blineage_class_switch_contrast_gate",
            "role": "gate",
            "field": "$.blineage_class_switch_contrast",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_class_switched_control_higher_unswitched",
              "class_switched_in_both_cohorts"
            ]
          },
          {
            "name": "dominant_match_narrative_trap_or_null",
            "role": "hard_fail",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_rbd_neutralizer",
              "matches_known_autoreactive_clone",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_heavy",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_light",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_recurrence_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_rank_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "cross_axis_found_dominant_but_failed_to_scan_for_embedded_neutralizer",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_pairing_neutralizer_match": {
                "op": "in",
                "values": [
                  "no_known_neutralizer_match"
                ]
              },
              "neutralizer_pairing_heavy": {
                "op": "in",
                "values": [
                  "none_detected"
                ]
              }
            }
          },
          {
            "name": "neutralizer_pairing_not_exactly_the_s309_pair",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "op": "not",
            "expected": {
              "op": "and",
              "args": [
                {
                  "op": "field",
                  "name": "neutralizer_pairing_heavy",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGHV1-18"
                    ]
                  }
                },
                {
                  "op": "field",
                  "name": "neutralizer_pairing_light",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGKV3-20"
                    ]
                  }
                }
              ]
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-6-pi",
      "modelName": "claude-opus-4-6",
      "harness": "pi",
      "task": "gse171524_plasma_bcr_neutralizing_clonotype",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 321,
      "cost": 0.72795475,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dominant_pairing_heavy_gate",
            "role": "gate",
            "field": "$.dominant_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV4-34"
            ]
          },
          {
            "name": "dominant_pairing_light_gate",
            "role": "gate",
            "field": "$.dominant_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGLV3-19"
            ]
          },
          {
            "name": "dominant_neutralizer_match_gate",
            "role": "gate",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_known_neutralizer_match"
            ]
          },
          {
            "name": "neutralizer_pairing_heavy_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV1-18"
            ]
          },
          {
            "name": "neutralizer_pairing_light_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGKV3-20"
            ]
          },
          {
            "name": "neutralizer_recurrence_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "observed_in_2_to_5_donors"
            ]
          },
          {
            "name": "neutralizer_rank_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "top_5_pairing",
              "top_6_to_20_pairing"
            ]
          },
          {
            "name": "dominant_clone_isotype_gate",
            "role": "gate",
            "field": "$.dominant_clone_isotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "IgG"
            ]
          },
          {
            "name": "blineage_class_switch_contrast_gate",
            "role": "gate",
            "field": "$.blineage_class_switch_contrast",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_class_switched_control_higher_unswitched",
              "class_switched_in_both_cohorts"
            ]
          },
          {
            "name": "dominant_match_narrative_trap_or_null",
            "role": "hard_fail",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_rbd_neutralizer",
              "matches_known_autoreactive_clone",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_heavy",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_light",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_recurrence_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_rank_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "cross_axis_found_dominant_but_failed_to_scan_for_embedded_neutralizer",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_pairing_neutralizer_match": {
                "op": "in",
                "values": [
                  "no_known_neutralizer_match"
                ]
              },
              "neutralizer_pairing_heavy": {
                "op": "in",
                "values": [
                  "none_detected"
                ]
              }
            }
          },
          {
            "name": "neutralizer_pairing_not_exactly_the_s309_pair",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "op": "not",
            "expected": {
              "op": "and",
              "args": [
                {
                  "op": "field",
                  "name": "neutralizer_pairing_heavy",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGHV1-18"
                    ]
                  }
                },
                {
                  "op": "field",
                  "name": "neutralizer_pairing_light",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGKV3-20"
                    ]
                  }
                }
              ]
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-7-claude-code",
      "modelName": "claude-opus-4-7",
      "harness": "claude-code",
      "task": "gse171524_plasma_bcr_neutralizing_clonotype",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 674,
      "cost": 2.2266065000000004,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dominant_pairing_heavy_gate",
            "role": "gate",
            "field": "$.dominant_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV4-34"
            ]
          },
          {
            "name": "dominant_pairing_light_gate",
            "role": "gate",
            "field": "$.dominant_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGLV3-19"
            ]
          },
          {
            "name": "dominant_neutralizer_match_gate",
            "role": "gate",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_known_neutralizer_match"
            ]
          },
          {
            "name": "neutralizer_pairing_heavy_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV1-18"
            ]
          },
          {
            "name": "neutralizer_pairing_light_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGKV3-20"
            ]
          },
          {
            "name": "neutralizer_recurrence_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "observed_in_2_to_5_donors"
            ]
          },
          {
            "name": "neutralizer_rank_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "top_5_pairing",
              "top_6_to_20_pairing"
            ]
          },
          {
            "name": "dominant_clone_isotype_gate",
            "role": "gate",
            "field": "$.dominant_clone_isotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "IgG"
            ]
          },
          {
            "name": "blineage_class_switch_contrast_gate",
            "role": "gate",
            "field": "$.blineage_class_switch_contrast",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_class_switched_control_higher_unswitched",
              "class_switched_in_both_cohorts"
            ]
          },
          {
            "name": "dominant_match_narrative_trap_or_null",
            "role": "hard_fail",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_rbd_neutralizer",
              "matches_known_autoreactive_clone",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_heavy",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_light",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_recurrence_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_rank_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "cross_axis_found_dominant_but_failed_to_scan_for_embedded_neutralizer",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_pairing_neutralizer_match": {
                "op": "in",
                "values": [
                  "no_known_neutralizer_match"
                ]
              },
              "neutralizer_pairing_heavy": {
                "op": "in",
                "values": [
                  "none_detected"
                ]
              }
            }
          },
          {
            "name": "neutralizer_pairing_not_exactly_the_s309_pair",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "op": "not",
            "expected": {
              "op": "and",
              "args": [
                {
                  "op": "field",
                  "name": "neutralizer_pairing_heavy",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGHV1-18"
                    ]
                  }
                },
                {
                  "op": "field",
                  "name": "neutralizer_pairing_light",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGKV3-20"
                    ]
                  }
                }
              ]
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-7-claude-code",
      "modelName": "claude-opus-4-7",
      "harness": "claude-code",
      "task": "gse171524_plasma_bcr_neutralizing_clonotype",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 326,
      "cost": 1.4783577500000002,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dominant_pairing_heavy_gate",
            "role": "gate",
            "field": "$.dominant_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV4-34"
            ]
          },
          {
            "name": "dominant_pairing_light_gate",
            "role": "gate",
            "field": "$.dominant_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGLV3-19"
            ]
          },
          {
            "name": "dominant_neutralizer_match_gate",
            "role": "gate",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_known_neutralizer_match"
            ]
          },
          {
            "name": "neutralizer_pairing_heavy_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV1-18"
            ]
          },
          {
            "name": "neutralizer_pairing_light_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGKV3-20"
            ]
          },
          {
            "name": "neutralizer_recurrence_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "observed_in_2_to_5_donors"
            ]
          },
          {
            "name": "neutralizer_rank_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "top_5_pairing",
              "top_6_to_20_pairing"
            ]
          },
          {
            "name": "dominant_clone_isotype_gate",
            "role": "gate",
            "field": "$.dominant_clone_isotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "IgG"
            ]
          },
          {
            "name": "blineage_class_switch_contrast_gate",
            "role": "gate",
            "field": "$.blineage_class_switch_contrast",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_class_switched_control_higher_unswitched",
              "class_switched_in_both_cohorts"
            ]
          },
          {
            "name": "dominant_match_narrative_trap_or_null",
            "role": "hard_fail",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_rbd_neutralizer",
              "matches_known_autoreactive_clone",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_heavy",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_light",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_recurrence_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_rank_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "cross_axis_found_dominant_but_failed_to_scan_for_embedded_neutralizer",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_pairing_neutralizer_match": {
                "op": "in",
                "values": [
                  "no_known_neutralizer_match"
                ]
              },
              "neutralizer_pairing_heavy": {
                "op": "in",
                "values": [
                  "none_detected"
                ]
              }
            }
          },
          {
            "name": "neutralizer_pairing_not_exactly_the_s309_pair",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "op": "not",
            "expected": {
              "op": "and",
              "args": [
                {
                  "op": "field",
                  "name": "neutralizer_pairing_heavy",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGHV1-18"
                    ]
                  }
                },
                {
                  "op": "field",
                  "name": "neutralizer_pairing_light",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGKV3-20"
                    ]
                  }
                }
              ]
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-7-claude-code",
      "modelName": "claude-opus-4-7",
      "harness": "claude-code",
      "task": "gse171524_plasma_bcr_neutralizing_clonotype",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 476,
      "cost": 1.6692607499999998,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dominant_pairing_heavy_gate",
            "role": "gate",
            "field": "$.dominant_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV4-34"
            ]
          },
          {
            "name": "dominant_pairing_light_gate",
            "role": "gate",
            "field": "$.dominant_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGLV3-19"
            ]
          },
          {
            "name": "dominant_neutralizer_match_gate",
            "role": "gate",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_known_neutralizer_match"
            ]
          },
          {
            "name": "neutralizer_pairing_heavy_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV1-18"
            ]
          },
          {
            "name": "neutralizer_pairing_light_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGKV3-20"
            ]
          },
          {
            "name": "neutralizer_recurrence_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "observed_in_2_to_5_donors"
            ]
          },
          {
            "name": "neutralizer_rank_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "top_5_pairing",
              "top_6_to_20_pairing"
            ]
          },
          {
            "name": "dominant_clone_isotype_gate",
            "role": "gate",
            "field": "$.dominant_clone_isotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "IgG"
            ]
          },
          {
            "name": "blineage_class_switch_contrast_gate",
            "role": "gate",
            "field": "$.blineage_class_switch_contrast",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_class_switched_control_higher_unswitched",
              "class_switched_in_both_cohorts"
            ]
          },
          {
            "name": "dominant_match_narrative_trap_or_null",
            "role": "hard_fail",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_rbd_neutralizer",
              "matches_known_autoreactive_clone",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_heavy",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_light",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_recurrence_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_rank_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "cross_axis_found_dominant_but_failed_to_scan_for_embedded_neutralizer",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_pairing_neutralizer_match": {
                "op": "in",
                "values": [
                  "no_known_neutralizer_match"
                ]
              },
              "neutralizer_pairing_heavy": {
                "op": "in",
                "values": [
                  "none_detected"
                ]
              }
            }
          },
          {
            "name": "neutralizer_pairing_not_exactly_the_s309_pair",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "op": "not",
            "expected": {
              "op": "and",
              "args": [
                {
                  "op": "field",
                  "name": "neutralizer_pairing_heavy",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGHV1-18"
                    ]
                  }
                },
                {
                  "op": "field",
                  "name": "neutralizer_pairing_light",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGKV3-20"
                    ]
                  }
                }
              ]
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-7-pi",
      "modelName": "claude-opus-4-7",
      "harness": "pi",
      "task": "gse171524_plasma_bcr_neutralizing_clonotype",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 322,
      "cost": 0.8153122500000001,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dominant_pairing_heavy_gate",
            "role": "gate",
            "field": "$.dominant_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV4-34"
            ]
          },
          {
            "name": "dominant_pairing_light_gate",
            "role": "gate",
            "field": "$.dominant_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGLV3-19"
            ]
          },
          {
            "name": "dominant_neutralizer_match_gate",
            "role": "gate",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_known_neutralizer_match"
            ]
          },
          {
            "name": "neutralizer_pairing_heavy_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV1-18"
            ]
          },
          {
            "name": "neutralizer_pairing_light_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGKV3-20"
            ]
          },
          {
            "name": "neutralizer_recurrence_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "observed_in_2_to_5_donors"
            ]
          },
          {
            "name": "neutralizer_rank_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "top_5_pairing",
              "top_6_to_20_pairing"
            ]
          },
          {
            "name": "dominant_clone_isotype_gate",
            "role": "gate",
            "field": "$.dominant_clone_isotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "IgG"
            ]
          },
          {
            "name": "blineage_class_switch_contrast_gate",
            "role": "gate",
            "field": "$.blineage_class_switch_contrast",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_class_switched_control_higher_unswitched",
              "class_switched_in_both_cohorts"
            ]
          },
          {
            "name": "dominant_match_narrative_trap_or_null",
            "role": "hard_fail",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_rbd_neutralizer",
              "matches_known_autoreactive_clone",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_heavy",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_light",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_recurrence_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_rank_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "cross_axis_found_dominant_but_failed_to_scan_for_embedded_neutralizer",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_pairing_neutralizer_match": {
                "op": "in",
                "values": [
                  "no_known_neutralizer_match"
                ]
              },
              "neutralizer_pairing_heavy": {
                "op": "in",
                "values": [
                  "none_detected"
                ]
              }
            }
          },
          {
            "name": "neutralizer_pairing_not_exactly_the_s309_pair",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "op": "not",
            "expected": {
              "op": "and",
              "args": [
                {
                  "op": "field",
                  "name": "neutralizer_pairing_heavy",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGHV1-18"
                    ]
                  }
                },
                {
                  "op": "field",
                  "name": "neutralizer_pairing_light",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGKV3-20"
                    ]
                  }
                }
              ]
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-7-pi",
      "modelName": "claude-opus-4-7",
      "harness": "pi",
      "task": "gse171524_plasma_bcr_neutralizing_clonotype",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 280,
      "cost": 0.58231025,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dominant_pairing_heavy_gate",
            "role": "gate",
            "field": "$.dominant_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV4-34"
            ]
          },
          {
            "name": "dominant_pairing_light_gate",
            "role": "gate",
            "field": "$.dominant_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGLV3-19"
            ]
          },
          {
            "name": "dominant_neutralizer_match_gate",
            "role": "gate",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_known_neutralizer_match"
            ]
          },
          {
            "name": "neutralizer_pairing_heavy_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV1-18"
            ]
          },
          {
            "name": "neutralizer_pairing_light_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGKV3-20"
            ]
          },
          {
            "name": "neutralizer_recurrence_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "observed_in_2_to_5_donors"
            ]
          },
          {
            "name": "neutralizer_rank_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "top_5_pairing",
              "top_6_to_20_pairing"
            ]
          },
          {
            "name": "dominant_clone_isotype_gate",
            "role": "gate",
            "field": "$.dominant_clone_isotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "IgG"
            ]
          },
          {
            "name": "blineage_class_switch_contrast_gate",
            "role": "gate",
            "field": "$.blineage_class_switch_contrast",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_class_switched_control_higher_unswitched",
              "class_switched_in_both_cohorts"
            ]
          },
          {
            "name": "dominant_match_narrative_trap_or_null",
            "role": "hard_fail",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_rbd_neutralizer",
              "matches_known_autoreactive_clone",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_heavy",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_light",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_recurrence_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_rank_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "cross_axis_found_dominant_but_failed_to_scan_for_embedded_neutralizer",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_pairing_neutralizer_match": {
                "op": "in",
                "values": [
                  "no_known_neutralizer_match"
                ]
              },
              "neutralizer_pairing_heavy": {
                "op": "in",
                "values": [
                  "none_detected"
                ]
              }
            }
          },
          {
            "name": "neutralizer_pairing_not_exactly_the_s309_pair",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "op": "not",
            "expected": {
              "op": "and",
              "args": [
                {
                  "op": "field",
                  "name": "neutralizer_pairing_heavy",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGHV1-18"
                    ]
                  }
                },
                {
                  "op": "field",
                  "name": "neutralizer_pairing_light",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGKV3-20"
                    ]
                  }
                }
              ]
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-7-pi",
      "modelName": "claude-opus-4-7",
      "harness": "pi",
      "task": "gse171524_plasma_bcr_neutralizing_clonotype",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 280,
      "cost": 0.69703375,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dominant_pairing_heavy_gate",
            "role": "gate",
            "field": "$.dominant_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV4-34"
            ]
          },
          {
            "name": "dominant_pairing_light_gate",
            "role": "gate",
            "field": "$.dominant_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGLV3-19"
            ]
          },
          {
            "name": "dominant_neutralizer_match_gate",
            "role": "gate",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_known_neutralizer_match"
            ]
          },
          {
            "name": "neutralizer_pairing_heavy_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV1-18"
            ]
          },
          {
            "name": "neutralizer_pairing_light_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGKV3-20"
            ]
          },
          {
            "name": "neutralizer_recurrence_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "observed_in_2_to_5_donors"
            ]
          },
          {
            "name": "neutralizer_rank_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "top_5_pairing",
              "top_6_to_20_pairing"
            ]
          },
          {
            "name": "dominant_clone_isotype_gate",
            "role": "gate",
            "field": "$.dominant_clone_isotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "IgG"
            ]
          },
          {
            "name": "blineage_class_switch_contrast_gate",
            "role": "gate",
            "field": "$.blineage_class_switch_contrast",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_class_switched_control_higher_unswitched",
              "class_switched_in_both_cohorts"
            ]
          },
          {
            "name": "dominant_match_narrative_trap_or_null",
            "role": "hard_fail",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_rbd_neutralizer",
              "matches_known_autoreactive_clone",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_heavy",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_light",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_recurrence_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_rank_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "cross_axis_found_dominant_but_failed_to_scan_for_embedded_neutralizer",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_pairing_neutralizer_match": {
                "op": "in",
                "values": [
                  "no_known_neutralizer_match"
                ]
              },
              "neutralizer_pairing_heavy": {
                "op": "in",
                "values": [
                  "none_detected"
                ]
              }
            }
          },
          {
            "name": "neutralizer_pairing_not_exactly_the_s309_pair",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "op": "not",
            "expected": {
              "op": "and",
              "args": [
                {
                  "op": "field",
                  "name": "neutralizer_pairing_heavy",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGHV1-18"
                    ]
                  }
                },
                {
                  "op": "field",
                  "name": "neutralizer_pairing_light",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGKV3-20"
                    ]
                  }
                }
              ]
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-8-claude-code",
      "modelName": "claude-opus-4-8",
      "harness": "claude-code",
      "task": "gse171524_plasma_bcr_neutralizing_clonotype",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 1556,
      "cost": 4.391825250000001,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dominant_pairing_heavy_gate",
            "role": "gate",
            "field": "$.dominant_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV4-34"
            ]
          },
          {
            "name": "dominant_pairing_light_gate",
            "role": "gate",
            "field": "$.dominant_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGLV3-19"
            ]
          },
          {
            "name": "dominant_neutralizer_match_gate",
            "role": "gate",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_known_neutralizer_match"
            ]
          },
          {
            "name": "neutralizer_pairing_heavy_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV1-18"
            ]
          },
          {
            "name": "neutralizer_pairing_light_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGKV3-20"
            ]
          },
          {
            "name": "neutralizer_recurrence_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "observed_in_2_to_5_donors"
            ]
          },
          {
            "name": "neutralizer_rank_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "top_5_pairing",
              "top_6_to_20_pairing"
            ]
          },
          {
            "name": "dominant_clone_isotype_gate",
            "role": "gate",
            "field": "$.dominant_clone_isotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "IgG"
            ]
          },
          {
            "name": "blineage_class_switch_contrast_gate",
            "role": "gate",
            "field": "$.blineage_class_switch_contrast",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_class_switched_control_higher_unswitched",
              "class_switched_in_both_cohorts"
            ]
          },
          {
            "name": "dominant_match_narrative_trap_or_null",
            "role": "hard_fail",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_rbd_neutralizer",
              "matches_known_autoreactive_clone",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_heavy",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_light",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_recurrence_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_rank_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "cross_axis_found_dominant_but_failed_to_scan_for_embedded_neutralizer",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_pairing_neutralizer_match": {
                "op": "in",
                "values": [
                  "no_known_neutralizer_match"
                ]
              },
              "neutralizer_pairing_heavy": {
                "op": "in",
                "values": [
                  "none_detected"
                ]
              }
            }
          },
          {
            "name": "neutralizer_pairing_not_exactly_the_s309_pair",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "op": "not",
            "expected": {
              "op": "and",
              "args": [
                {
                  "op": "field",
                  "name": "neutralizer_pairing_heavy",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGHV1-18"
                    ]
                  }
                },
                {
                  "op": "field",
                  "name": "neutralizer_pairing_light",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGKV3-20"
                    ]
                  }
                }
              ]
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-8-claude-code",
      "modelName": "claude-opus-4-8",
      "harness": "claude-code",
      "task": "gse171524_plasma_bcr_neutralizing_clonotype",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 968,
      "cost": 3.44366525,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dominant_pairing_heavy_gate",
            "role": "gate",
            "field": "$.dominant_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV4-34"
            ]
          },
          {
            "name": "dominant_pairing_light_gate",
            "role": "gate",
            "field": "$.dominant_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGLV3-19"
            ]
          },
          {
            "name": "dominant_neutralizer_match_gate",
            "role": "gate",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_known_neutralizer_match"
            ]
          },
          {
            "name": "neutralizer_pairing_heavy_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV1-18"
            ]
          },
          {
            "name": "neutralizer_pairing_light_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGKV3-20"
            ]
          },
          {
            "name": "neutralizer_recurrence_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "observed_in_2_to_5_donors"
            ]
          },
          {
            "name": "neutralizer_rank_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "top_5_pairing",
              "top_6_to_20_pairing"
            ]
          },
          {
            "name": "dominant_clone_isotype_gate",
            "role": "gate",
            "field": "$.dominant_clone_isotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "IgG"
            ]
          },
          {
            "name": "blineage_class_switch_contrast_gate",
            "role": "gate",
            "field": "$.blineage_class_switch_contrast",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_class_switched_control_higher_unswitched",
              "class_switched_in_both_cohorts"
            ]
          },
          {
            "name": "dominant_match_narrative_trap_or_null",
            "role": "hard_fail",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_rbd_neutralizer",
              "matches_known_autoreactive_clone",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_heavy",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_light",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_recurrence_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_rank_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "cross_axis_found_dominant_but_failed_to_scan_for_embedded_neutralizer",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_pairing_neutralizer_match": {
                "op": "in",
                "values": [
                  "no_known_neutralizer_match"
                ]
              },
              "neutralizer_pairing_heavy": {
                "op": "in",
                "values": [
                  "none_detected"
                ]
              }
            }
          },
          {
            "name": "neutralizer_pairing_not_exactly_the_s309_pair",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "op": "not",
            "expected": {
              "op": "and",
              "args": [
                {
                  "op": "field",
                  "name": "neutralizer_pairing_heavy",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGHV1-18"
                    ]
                  }
                },
                {
                  "op": "field",
                  "name": "neutralizer_pairing_light",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGKV3-20"
                    ]
                  }
                }
              ]
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-8-claude-code",
      "modelName": "claude-opus-4-8",
      "harness": "claude-code",
      "task": "gse171524_plasma_bcr_neutralizing_clonotype",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1424,
      "cost": 4.874309999999999,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dominant_pairing_heavy_gate",
            "role": "gate",
            "field": "$.dominant_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV4-34"
            ]
          },
          {
            "name": "dominant_pairing_light_gate",
            "role": "gate",
            "field": "$.dominant_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGLV3-19"
            ]
          },
          {
            "name": "dominant_neutralizer_match_gate",
            "role": "gate",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_known_neutralizer_match"
            ]
          },
          {
            "name": "neutralizer_pairing_heavy_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV1-18"
            ]
          },
          {
            "name": "neutralizer_pairing_light_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGKV3-20"
            ]
          },
          {
            "name": "neutralizer_recurrence_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "observed_in_2_to_5_donors"
            ]
          },
          {
            "name": "neutralizer_rank_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "top_5_pairing",
              "top_6_to_20_pairing"
            ]
          },
          {
            "name": "dominant_clone_isotype_gate",
            "role": "gate",
            "field": "$.dominant_clone_isotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "IgG"
            ]
          },
          {
            "name": "blineage_class_switch_contrast_gate",
            "role": "gate",
            "field": "$.blineage_class_switch_contrast",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_class_switched_control_higher_unswitched",
              "class_switched_in_both_cohorts"
            ]
          },
          {
            "name": "dominant_match_narrative_trap_or_null",
            "role": "hard_fail",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_rbd_neutralizer",
              "matches_known_autoreactive_clone",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_heavy",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_light",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_recurrence_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_rank_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "cross_axis_found_dominant_but_failed_to_scan_for_embedded_neutralizer",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_pairing_neutralizer_match": {
                "op": "in",
                "values": [
                  "no_known_neutralizer_match"
                ]
              },
              "neutralizer_pairing_heavy": {
                "op": "in",
                "values": [
                  "none_detected"
                ]
              }
            }
          },
          {
            "name": "neutralizer_pairing_not_exactly_the_s309_pair",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "op": "not",
            "expected": {
              "op": "and",
              "args": [
                {
                  "op": "field",
                  "name": "neutralizer_pairing_heavy",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGHV1-18"
                    ]
                  }
                },
                {
                  "op": "field",
                  "name": "neutralizer_pairing_light",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGKV3-20"
                    ]
                  }
                }
              ]
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-8-pi",
      "modelName": "claude-opus-4-8",
      "harness": "pi",
      "task": "gse171524_plasma_bcr_neutralizing_clonotype",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 1423,
      "cost": 1.8523827500000005,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dominant_pairing_heavy_gate",
            "role": "gate",
            "field": "$.dominant_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV4-34"
            ]
          },
          {
            "name": "dominant_pairing_light_gate",
            "role": "gate",
            "field": "$.dominant_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGLV3-19"
            ]
          },
          {
            "name": "dominant_neutralizer_match_gate",
            "role": "gate",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_known_neutralizer_match"
            ]
          },
          {
            "name": "neutralizer_pairing_heavy_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV1-18"
            ]
          },
          {
            "name": "neutralizer_pairing_light_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGKV3-20"
            ]
          },
          {
            "name": "neutralizer_recurrence_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "observed_in_2_to_5_donors"
            ]
          },
          {
            "name": "neutralizer_rank_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "top_5_pairing",
              "top_6_to_20_pairing"
            ]
          },
          {
            "name": "dominant_clone_isotype_gate",
            "role": "gate",
            "field": "$.dominant_clone_isotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "IgG"
            ]
          },
          {
            "name": "blineage_class_switch_contrast_gate",
            "role": "gate",
            "field": "$.blineage_class_switch_contrast",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_class_switched_control_higher_unswitched",
              "class_switched_in_both_cohorts"
            ]
          },
          {
            "name": "dominant_match_narrative_trap_or_null",
            "role": "hard_fail",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_rbd_neutralizer",
              "matches_known_autoreactive_clone",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_heavy",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_light",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_recurrence_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_rank_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "cross_axis_found_dominant_but_failed_to_scan_for_embedded_neutralizer",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_pairing_neutralizer_match": {
                "op": "in",
                "values": [
                  "no_known_neutralizer_match"
                ]
              },
              "neutralizer_pairing_heavy": {
                "op": "in",
                "values": [
                  "none_detected"
                ]
              }
            }
          },
          {
            "name": "neutralizer_pairing_not_exactly_the_s309_pair",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "op": "not",
            "expected": {
              "op": "and",
              "args": [
                {
                  "op": "field",
                  "name": "neutralizer_pairing_heavy",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGHV1-18"
                    ]
                  }
                },
                {
                  "op": "field",
                  "name": "neutralizer_pairing_light",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGKV3-20"
                    ]
                  }
                }
              ]
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-8-pi",
      "modelName": "claude-opus-4-8",
      "harness": "pi",
      "task": "gse171524_plasma_bcr_neutralizing_clonotype",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 748,
      "cost": 1.57524275,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dominant_pairing_heavy_gate",
            "role": "gate",
            "field": "$.dominant_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV4-34"
            ]
          },
          {
            "name": "dominant_pairing_light_gate",
            "role": "gate",
            "field": "$.dominant_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGLV3-19"
            ]
          },
          {
            "name": "dominant_neutralizer_match_gate",
            "role": "gate",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_known_neutralizer_match"
            ]
          },
          {
            "name": "neutralizer_pairing_heavy_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV1-18"
            ]
          },
          {
            "name": "neutralizer_pairing_light_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGKV3-20"
            ]
          },
          {
            "name": "neutralizer_recurrence_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "observed_in_2_to_5_donors"
            ]
          },
          {
            "name": "neutralizer_rank_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "top_5_pairing",
              "top_6_to_20_pairing"
            ]
          },
          {
            "name": "dominant_clone_isotype_gate",
            "role": "gate",
            "field": "$.dominant_clone_isotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "IgG"
            ]
          },
          {
            "name": "blineage_class_switch_contrast_gate",
            "role": "gate",
            "field": "$.blineage_class_switch_contrast",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_class_switched_control_higher_unswitched",
              "class_switched_in_both_cohorts"
            ]
          },
          {
            "name": "dominant_match_narrative_trap_or_null",
            "role": "hard_fail",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_rbd_neutralizer",
              "matches_known_autoreactive_clone",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_heavy",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_light",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_recurrence_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_rank_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "cross_axis_found_dominant_but_failed_to_scan_for_embedded_neutralizer",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_pairing_neutralizer_match": {
                "op": "in",
                "values": [
                  "no_known_neutralizer_match"
                ]
              },
              "neutralizer_pairing_heavy": {
                "op": "in",
                "values": [
                  "none_detected"
                ]
              }
            }
          },
          {
            "name": "neutralizer_pairing_not_exactly_the_s309_pair",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "op": "not",
            "expected": {
              "op": "and",
              "args": [
                {
                  "op": "field",
                  "name": "neutralizer_pairing_heavy",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGHV1-18"
                    ]
                  }
                },
                {
                  "op": "field",
                  "name": "neutralizer_pairing_light",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGKV3-20"
                    ]
                  }
                }
              ]
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-8-pi",
      "modelName": "claude-opus-4-8",
      "harness": "pi",
      "task": "gse171524_plasma_bcr_neutralizing_clonotype",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 520,
      "cost": 1.0459525,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dominant_pairing_heavy_gate",
            "role": "gate",
            "field": "$.dominant_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV4-34"
            ]
          },
          {
            "name": "dominant_pairing_light_gate",
            "role": "gate",
            "field": "$.dominant_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGLV3-19"
            ]
          },
          {
            "name": "dominant_neutralizer_match_gate",
            "role": "gate",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_known_neutralizer_match"
            ]
          },
          {
            "name": "neutralizer_pairing_heavy_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV1-18"
            ]
          },
          {
            "name": "neutralizer_pairing_light_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGKV3-20"
            ]
          },
          {
            "name": "neutralizer_recurrence_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "observed_in_2_to_5_donors"
            ]
          },
          {
            "name": "neutralizer_rank_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "top_5_pairing",
              "top_6_to_20_pairing"
            ]
          },
          {
            "name": "dominant_clone_isotype_gate",
            "role": "gate",
            "field": "$.dominant_clone_isotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "IgG"
            ]
          },
          {
            "name": "blineage_class_switch_contrast_gate",
            "role": "gate",
            "field": "$.blineage_class_switch_contrast",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_class_switched_control_higher_unswitched",
              "class_switched_in_both_cohorts"
            ]
          },
          {
            "name": "dominant_match_narrative_trap_or_null",
            "role": "hard_fail",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_rbd_neutralizer",
              "matches_known_autoreactive_clone",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_heavy",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_light",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_recurrence_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_rank_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "cross_axis_found_dominant_but_failed_to_scan_for_embedded_neutralizer",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_pairing_neutralizer_match": {
                "op": "in",
                "values": [
                  "no_known_neutralizer_match"
                ]
              },
              "neutralizer_pairing_heavy": {
                "op": "in",
                "values": [
                  "none_detected"
                ]
              }
            }
          },
          {
            "name": "neutralizer_pairing_not_exactly_the_s309_pair",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "op": "not",
            "expected": {
              "op": "and",
              "args": [
                {
                  "op": "field",
                  "name": "neutralizer_pairing_heavy",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGHV1-18"
                    ]
                  }
                },
                {
                  "op": "field",
                  "name": "neutralizer_pairing_light",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGKV3-20"
                    ]
                  }
                }
              ]
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-4-6-pi",
      "modelName": "claude-sonnet-4-6",
      "harness": "pi",
      "task": "gse171524_plasma_bcr_neutralizing_clonotype",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 849,
      "cost": 1.0861564500000003,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dominant_pairing_heavy_gate",
            "role": "gate",
            "field": "$.dominant_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV4-34"
            ]
          },
          {
            "name": "dominant_pairing_light_gate",
            "role": "gate",
            "field": "$.dominant_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGLV3-19"
            ]
          },
          {
            "name": "dominant_neutralizer_match_gate",
            "role": "gate",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_known_neutralizer_match"
            ]
          },
          {
            "name": "neutralizer_pairing_heavy_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV1-18"
            ]
          },
          {
            "name": "neutralizer_pairing_light_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGKV3-20"
            ]
          },
          {
            "name": "neutralizer_recurrence_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "observed_in_2_to_5_donors"
            ]
          },
          {
            "name": "neutralizer_rank_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "top_5_pairing",
              "top_6_to_20_pairing"
            ]
          },
          {
            "name": "dominant_clone_isotype_gate",
            "role": "gate",
            "field": "$.dominant_clone_isotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "IgG"
            ]
          },
          {
            "name": "blineage_class_switch_contrast_gate",
            "role": "gate",
            "field": "$.blineage_class_switch_contrast",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_class_switched_control_higher_unswitched",
              "class_switched_in_both_cohorts"
            ]
          },
          {
            "name": "dominant_match_narrative_trap_or_null",
            "role": "hard_fail",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_rbd_neutralizer",
              "matches_known_autoreactive_clone",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_heavy",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_light",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_recurrence_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_rank_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "cross_axis_found_dominant_but_failed_to_scan_for_embedded_neutralizer",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_pairing_neutralizer_match": {
                "op": "in",
                "values": [
                  "no_known_neutralizer_match"
                ]
              },
              "neutralizer_pairing_heavy": {
                "op": "in",
                "values": [
                  "none_detected"
                ]
              }
            }
          },
          {
            "name": "neutralizer_pairing_not_exactly_the_s309_pair",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "op": "not",
            "expected": {
              "op": "and",
              "args": [
                {
                  "op": "field",
                  "name": "neutralizer_pairing_heavy",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGHV1-18"
                    ]
                  }
                },
                {
                  "op": "field",
                  "name": "neutralizer_pairing_light",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGKV3-20"
                    ]
                  }
                }
              ]
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-4-6-pi",
      "modelName": "claude-sonnet-4-6",
      "harness": "pi",
      "task": "gse171524_plasma_bcr_neutralizing_clonotype",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 672,
      "cost": 1.02503085,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dominant_pairing_heavy_gate",
            "role": "gate",
            "field": "$.dominant_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV4-34"
            ]
          },
          {
            "name": "dominant_pairing_light_gate",
            "role": "gate",
            "field": "$.dominant_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGLV3-19"
            ]
          },
          {
            "name": "dominant_neutralizer_match_gate",
            "role": "gate",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_known_neutralizer_match"
            ]
          },
          {
            "name": "neutralizer_pairing_heavy_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV1-18"
            ]
          },
          {
            "name": "neutralizer_pairing_light_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGKV3-20"
            ]
          },
          {
            "name": "neutralizer_recurrence_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "observed_in_2_to_5_donors"
            ]
          },
          {
            "name": "neutralizer_rank_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "top_5_pairing",
              "top_6_to_20_pairing"
            ]
          },
          {
            "name": "dominant_clone_isotype_gate",
            "role": "gate",
            "field": "$.dominant_clone_isotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "IgG"
            ]
          },
          {
            "name": "blineage_class_switch_contrast_gate",
            "role": "gate",
            "field": "$.blineage_class_switch_contrast",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_class_switched_control_higher_unswitched",
              "class_switched_in_both_cohorts"
            ]
          },
          {
            "name": "dominant_match_narrative_trap_or_null",
            "role": "hard_fail",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_rbd_neutralizer",
              "matches_known_autoreactive_clone",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_heavy",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_light",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_recurrence_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_rank_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "cross_axis_found_dominant_but_failed_to_scan_for_embedded_neutralizer",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_pairing_neutralizer_match": {
                "op": "in",
                "values": [
                  "no_known_neutralizer_match"
                ]
              },
              "neutralizer_pairing_heavy": {
                "op": "in",
                "values": [
                  "none_detected"
                ]
              }
            }
          },
          {
            "name": "neutralizer_pairing_not_exactly_the_s309_pair",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "op": "not",
            "expected": {
              "op": "and",
              "args": [
                {
                  "op": "field",
                  "name": "neutralizer_pairing_heavy",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGHV1-18"
                    ]
                  }
                },
                {
                  "op": "field",
                  "name": "neutralizer_pairing_light",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGKV3-20"
                    ]
                  }
                }
              ]
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-4-6-pi",
      "modelName": "claude-sonnet-4-6",
      "harness": "pi",
      "task": "gse171524_plasma_bcr_neutralizing_clonotype",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 909,
      "cost": 1.1607336000000004,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dominant_pairing_heavy_gate",
            "role": "gate",
            "field": "$.dominant_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV4-34"
            ]
          },
          {
            "name": "dominant_pairing_light_gate",
            "role": "gate",
            "field": "$.dominant_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGLV3-19"
            ]
          },
          {
            "name": "dominant_neutralizer_match_gate",
            "role": "gate",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_known_neutralizer_match"
            ]
          },
          {
            "name": "neutralizer_pairing_heavy_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV1-18"
            ]
          },
          {
            "name": "neutralizer_pairing_light_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGKV3-20"
            ]
          },
          {
            "name": "neutralizer_recurrence_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "observed_in_2_to_5_donors"
            ]
          },
          {
            "name": "neutralizer_rank_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "top_5_pairing",
              "top_6_to_20_pairing"
            ]
          },
          {
            "name": "dominant_clone_isotype_gate",
            "role": "gate",
            "field": "$.dominant_clone_isotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "IgG"
            ]
          },
          {
            "name": "blineage_class_switch_contrast_gate",
            "role": "gate",
            "field": "$.blineage_class_switch_contrast",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_class_switched_control_higher_unswitched",
              "class_switched_in_both_cohorts"
            ]
          },
          {
            "name": "dominant_match_narrative_trap_or_null",
            "role": "hard_fail",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_rbd_neutralizer",
              "matches_known_autoreactive_clone",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_heavy",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_light",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_recurrence_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_rank_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "cross_axis_found_dominant_but_failed_to_scan_for_embedded_neutralizer",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_pairing_neutralizer_match": {
                "op": "in",
                "values": [
                  "no_known_neutralizer_match"
                ]
              },
              "neutralizer_pairing_heavy": {
                "op": "in",
                "values": [
                  "none_detected"
                ]
              }
            }
          },
          {
            "name": "neutralizer_pairing_not_exactly_the_s309_pair",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "op": "not",
            "expected": {
              "op": "and",
              "args": [
                {
                  "op": "field",
                  "name": "neutralizer_pairing_heavy",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGHV1-18"
                    ]
                  }
                },
                {
                  "op": "field",
                  "name": "neutralizer_pairing_light",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGKV3-20"
                    ]
                  }
                }
              ]
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-gemini-3-1-pro-preview-pi",
      "modelName": "gemini-3.1-pro-preview",
      "harness": "pi",
      "task": "gse171524_plasma_bcr_neutralizing_clonotype",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 803,
      "cost": 1.0336687999999998,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dominant_pairing_heavy_gate",
            "role": "gate",
            "field": "$.dominant_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV4-34"
            ]
          },
          {
            "name": "dominant_pairing_light_gate",
            "role": "gate",
            "field": "$.dominant_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGLV3-19"
            ]
          },
          {
            "name": "dominant_neutralizer_match_gate",
            "role": "gate",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_known_neutralizer_match"
            ]
          },
          {
            "name": "neutralizer_pairing_heavy_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV1-18"
            ]
          },
          {
            "name": "neutralizer_pairing_light_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGKV3-20"
            ]
          },
          {
            "name": "neutralizer_recurrence_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "observed_in_2_to_5_donors"
            ]
          },
          {
            "name": "neutralizer_rank_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "top_5_pairing",
              "top_6_to_20_pairing"
            ]
          },
          {
            "name": "dominant_clone_isotype_gate",
            "role": "gate",
            "field": "$.dominant_clone_isotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "IgG"
            ]
          },
          {
            "name": "blineage_class_switch_contrast_gate",
            "role": "gate",
            "field": "$.blineage_class_switch_contrast",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_class_switched_control_higher_unswitched",
              "class_switched_in_both_cohorts"
            ]
          },
          {
            "name": "dominant_match_narrative_trap_or_null",
            "role": "hard_fail",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_rbd_neutralizer",
              "matches_known_autoreactive_clone",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_heavy",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_light",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_recurrence_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_rank_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "cross_axis_found_dominant_but_failed_to_scan_for_embedded_neutralizer",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_pairing_neutralizer_match": {
                "op": "in",
                "values": [
                  "no_known_neutralizer_match"
                ]
              },
              "neutralizer_pairing_heavy": {
                "op": "in",
                "values": [
                  "none_detected"
                ]
              }
            }
          },
          {
            "name": "neutralizer_pairing_not_exactly_the_s309_pair",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "op": "not",
            "expected": {
              "op": "and",
              "args": [
                {
                  "op": "field",
                  "name": "neutralizer_pairing_heavy",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGHV1-18"
                    ]
                  }
                },
                {
                  "op": "field",
                  "name": "neutralizer_pairing_light",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGKV3-20"
                    ]
                  }
                }
              ]
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-gemini-3-1-pro-preview-pi",
      "modelName": "gemini-3.1-pro-preview",
      "harness": "pi",
      "task": "gse171524_plasma_bcr_neutralizing_clonotype",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 779,
      "cost": 0.9328546000000001,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dominant_pairing_heavy_gate",
            "role": "gate",
            "field": "$.dominant_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV4-34"
            ]
          },
          {
            "name": "dominant_pairing_light_gate",
            "role": "gate",
            "field": "$.dominant_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGLV3-19"
            ]
          },
          {
            "name": "dominant_neutralizer_match_gate",
            "role": "gate",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_known_neutralizer_match"
            ]
          },
          {
            "name": "neutralizer_pairing_heavy_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV1-18"
            ]
          },
          {
            "name": "neutralizer_pairing_light_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGKV3-20"
            ]
          },
          {
            "name": "neutralizer_recurrence_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "observed_in_2_to_5_donors"
            ]
          },
          {
            "name": "neutralizer_rank_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "top_5_pairing",
              "top_6_to_20_pairing"
            ]
          },
          {
            "name": "dominant_clone_isotype_gate",
            "role": "gate",
            "field": "$.dominant_clone_isotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "IgG"
            ]
          },
          {
            "name": "blineage_class_switch_contrast_gate",
            "role": "gate",
            "field": "$.blineage_class_switch_contrast",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_class_switched_control_higher_unswitched",
              "class_switched_in_both_cohorts"
            ]
          },
          {
            "name": "dominant_match_narrative_trap_or_null",
            "role": "hard_fail",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_rbd_neutralizer",
              "matches_known_autoreactive_clone",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_heavy",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_light",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_recurrence_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_rank_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "cross_axis_found_dominant_but_failed_to_scan_for_embedded_neutralizer",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_pairing_neutralizer_match": {
                "op": "in",
                "values": [
                  "no_known_neutralizer_match"
                ]
              },
              "neutralizer_pairing_heavy": {
                "op": "in",
                "values": [
                  "none_detected"
                ]
              }
            }
          },
          {
            "name": "neutralizer_pairing_not_exactly_the_s309_pair",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "op": "not",
            "expected": {
              "op": "and",
              "args": [
                {
                  "op": "field",
                  "name": "neutralizer_pairing_heavy",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGHV1-18"
                    ]
                  }
                },
                {
                  "op": "field",
                  "name": "neutralizer_pairing_light",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGKV3-20"
                    ]
                  }
                }
              ]
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-gemini-3-1-pro-preview-pi",
      "modelName": "gemini-3.1-pro-preview",
      "harness": "pi",
      "task": "gse171524_plasma_bcr_neutralizing_clonotype",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 241,
      "cost": 0.4595154,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dominant_pairing_heavy_gate",
            "role": "gate",
            "field": "$.dominant_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV4-34"
            ]
          },
          {
            "name": "dominant_pairing_light_gate",
            "role": "gate",
            "field": "$.dominant_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGLV3-19"
            ]
          },
          {
            "name": "dominant_neutralizer_match_gate",
            "role": "gate",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_known_neutralizer_match"
            ]
          },
          {
            "name": "neutralizer_pairing_heavy_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV1-18"
            ]
          },
          {
            "name": "neutralizer_pairing_light_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGKV3-20"
            ]
          },
          {
            "name": "neutralizer_recurrence_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "observed_in_2_to_5_donors"
            ]
          },
          {
            "name": "neutralizer_rank_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "top_5_pairing",
              "top_6_to_20_pairing"
            ]
          },
          {
            "name": "dominant_clone_isotype_gate",
            "role": "gate",
            "field": "$.dominant_clone_isotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "IgG"
            ]
          },
          {
            "name": "blineage_class_switch_contrast_gate",
            "role": "gate",
            "field": "$.blineage_class_switch_contrast",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_class_switched_control_higher_unswitched",
              "class_switched_in_both_cohorts"
            ]
          },
          {
            "name": "dominant_match_narrative_trap_or_null",
            "role": "hard_fail",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_rbd_neutralizer",
              "matches_known_autoreactive_clone",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_heavy",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_light",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_recurrence_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_rank_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "cross_axis_found_dominant_but_failed_to_scan_for_embedded_neutralizer",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_pairing_neutralizer_match": {
                "op": "in",
                "values": [
                  "no_known_neutralizer_match"
                ]
              },
              "neutralizer_pairing_heavy": {
                "op": "in",
                "values": [
                  "none_detected"
                ]
              }
            }
          },
          {
            "name": "neutralizer_pairing_not_exactly_the_s309_pair",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "op": "not",
            "expected": {
              "op": "and",
              "args": [
                {
                  "op": "field",
                  "name": "neutralizer_pairing_heavy",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGHV1-18"
                    ]
                  }
                },
                {
                  "op": "field",
                  "name": "neutralizer_pairing_light",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGKV3-20"
                    ]
                  }
                }
              ]
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-gemini-3-5-flash-pi",
      "modelName": "gemini-3.5-flash",
      "harness": "pi",
      "task": "gse171524_plasma_bcr_neutralizing_clonotype",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 672,
      "cost": 0.9842737499999998,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dominant_pairing_heavy_gate",
            "role": "gate",
            "field": "$.dominant_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV4-34"
            ]
          },
          {
            "name": "dominant_pairing_light_gate",
            "role": "gate",
            "field": "$.dominant_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGLV3-19"
            ]
          },
          {
            "name": "dominant_neutralizer_match_gate",
            "role": "gate",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_known_neutralizer_match"
            ]
          },
          {
            "name": "neutralizer_pairing_heavy_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV1-18"
            ]
          },
          {
            "name": "neutralizer_pairing_light_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGKV3-20"
            ]
          },
          {
            "name": "neutralizer_recurrence_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "observed_in_2_to_5_donors"
            ]
          },
          {
            "name": "neutralizer_rank_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "top_5_pairing",
              "top_6_to_20_pairing"
            ]
          },
          {
            "name": "dominant_clone_isotype_gate",
            "role": "gate",
            "field": "$.dominant_clone_isotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "IgG"
            ]
          },
          {
            "name": "blineage_class_switch_contrast_gate",
            "role": "gate",
            "field": "$.blineage_class_switch_contrast",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_class_switched_control_higher_unswitched",
              "class_switched_in_both_cohorts"
            ]
          },
          {
            "name": "dominant_match_narrative_trap_or_null",
            "role": "hard_fail",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_rbd_neutralizer",
              "matches_known_autoreactive_clone",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_heavy",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_light",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_recurrence_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_rank_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "cross_axis_found_dominant_but_failed_to_scan_for_embedded_neutralizer",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_pairing_neutralizer_match": {
                "op": "in",
                "values": [
                  "no_known_neutralizer_match"
                ]
              },
              "neutralizer_pairing_heavy": {
                "op": "in",
                "values": [
                  "none_detected"
                ]
              }
            }
          },
          {
            "name": "neutralizer_pairing_not_exactly_the_s309_pair",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "op": "not",
            "expected": {
              "op": "and",
              "args": [
                {
                  "op": "field",
                  "name": "neutralizer_pairing_heavy",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGHV1-18"
                    ]
                  }
                },
                {
                  "op": "field",
                  "name": "neutralizer_pairing_light",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGKV3-20"
                    ]
                  }
                }
              ]
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-gemini-3-5-flash-pi",
      "modelName": "gemini-3.5-flash",
      "harness": "pi",
      "task": "gse171524_plasma_bcr_neutralizing_clonotype",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 1419,
      "cost": 1.0974306000000005,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dominant_pairing_heavy_gate",
            "role": "gate",
            "field": "$.dominant_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV4-34"
            ]
          },
          {
            "name": "dominant_pairing_light_gate",
            "role": "gate",
            "field": "$.dominant_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGLV3-19"
            ]
          },
          {
            "name": "dominant_neutralizer_match_gate",
            "role": "gate",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_known_neutralizer_match"
            ]
          },
          {
            "name": "neutralizer_pairing_heavy_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV1-18"
            ]
          },
          {
            "name": "neutralizer_pairing_light_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGKV3-20"
            ]
          },
          {
            "name": "neutralizer_recurrence_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "observed_in_2_to_5_donors"
            ]
          },
          {
            "name": "neutralizer_rank_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "top_5_pairing",
              "top_6_to_20_pairing"
            ]
          },
          {
            "name": "dominant_clone_isotype_gate",
            "role": "gate",
            "field": "$.dominant_clone_isotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "IgG"
            ]
          },
          {
            "name": "blineage_class_switch_contrast_gate",
            "role": "gate",
            "field": "$.blineage_class_switch_contrast",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_class_switched_control_higher_unswitched",
              "class_switched_in_both_cohorts"
            ]
          },
          {
            "name": "dominant_match_narrative_trap_or_null",
            "role": "hard_fail",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_rbd_neutralizer",
              "matches_known_autoreactive_clone",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_heavy",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_light",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_recurrence_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_rank_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "cross_axis_found_dominant_but_failed_to_scan_for_embedded_neutralizer",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_pairing_neutralizer_match": {
                "op": "in",
                "values": [
                  "no_known_neutralizer_match"
                ]
              },
              "neutralizer_pairing_heavy": {
                "op": "in",
                "values": [
                  "none_detected"
                ]
              }
            }
          },
          {
            "name": "neutralizer_pairing_not_exactly_the_s309_pair",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "op": "not",
            "expected": {
              "op": "and",
              "args": [
                {
                  "op": "field",
                  "name": "neutralizer_pairing_heavy",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGHV1-18"
                    ]
                  }
                },
                {
                  "op": "field",
                  "name": "neutralizer_pairing_light",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGKV3-20"
                    ]
                  }
                }
              ]
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-gemini-3-5-flash-pi",
      "modelName": "gemini-3.5-flash",
      "harness": "pi",
      "task": "gse171524_plasma_bcr_neutralizing_clonotype",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 407,
      "cost": 0.7006708500000001,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dominant_pairing_heavy_gate",
            "role": "gate",
            "field": "$.dominant_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV4-34"
            ]
          },
          {
            "name": "dominant_pairing_light_gate",
            "role": "gate",
            "field": "$.dominant_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGLV3-19"
            ]
          },
          {
            "name": "dominant_neutralizer_match_gate",
            "role": "gate",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_known_neutralizer_match"
            ]
          },
          {
            "name": "neutralizer_pairing_heavy_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV1-18"
            ]
          },
          {
            "name": "neutralizer_pairing_light_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGKV3-20"
            ]
          },
          {
            "name": "neutralizer_recurrence_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "observed_in_2_to_5_donors"
            ]
          },
          {
            "name": "neutralizer_rank_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "top_5_pairing",
              "top_6_to_20_pairing"
            ]
          },
          {
            "name": "dominant_clone_isotype_gate",
            "role": "gate",
            "field": "$.dominant_clone_isotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "IgG"
            ]
          },
          {
            "name": "blineage_class_switch_contrast_gate",
            "role": "gate",
            "field": "$.blineage_class_switch_contrast",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_class_switched_control_higher_unswitched",
              "class_switched_in_both_cohorts"
            ]
          },
          {
            "name": "dominant_match_narrative_trap_or_null",
            "role": "hard_fail",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_rbd_neutralizer",
              "matches_known_autoreactive_clone",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_heavy",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_light",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_recurrence_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_rank_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "cross_axis_found_dominant_but_failed_to_scan_for_embedded_neutralizer",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_pairing_neutralizer_match": {
                "op": "in",
                "values": [
                  "no_known_neutralizer_match"
                ]
              },
              "neutralizer_pairing_heavy": {
                "op": "in",
                "values": [
                  "none_detected"
                ]
              }
            }
          },
          {
            "name": "neutralizer_pairing_not_exactly_the_s309_pair",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "op": "not",
            "expected": {
              "op": "and",
              "args": [
                {
                  "op": "field",
                  "name": "neutralizer_pairing_heavy",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGHV1-18"
                    ]
                  }
                },
                {
                  "op": "field",
                  "name": "neutralizer_pairing_light",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGKV3-20"
                    ]
                  }
                }
              ]
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-glm-5p2-pi",
      "modelName": "glm-5p2",
      "harness": "pi",
      "task": "gse171524_plasma_bcr_neutralizing_clonotype",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 1317,
      "cost": 0.8552673599999999,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dominant_pairing_heavy_gate",
            "role": "gate",
            "field": "$.dominant_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV4-34"
            ]
          },
          {
            "name": "dominant_pairing_light_gate",
            "role": "gate",
            "field": "$.dominant_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGLV3-19"
            ]
          },
          {
            "name": "dominant_neutralizer_match_gate",
            "role": "gate",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_known_neutralizer_match"
            ]
          },
          {
            "name": "neutralizer_pairing_heavy_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV1-18"
            ]
          },
          {
            "name": "neutralizer_pairing_light_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGKV3-20"
            ]
          },
          {
            "name": "neutralizer_recurrence_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "observed_in_2_to_5_donors"
            ]
          },
          {
            "name": "neutralizer_rank_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "top_5_pairing",
              "top_6_to_20_pairing"
            ]
          },
          {
            "name": "dominant_clone_isotype_gate",
            "role": "gate",
            "field": "$.dominant_clone_isotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "IgG"
            ]
          },
          {
            "name": "blineage_class_switch_contrast_gate",
            "role": "gate",
            "field": "$.blineage_class_switch_contrast",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_class_switched_control_higher_unswitched",
              "class_switched_in_both_cohorts"
            ]
          },
          {
            "name": "dominant_match_narrative_trap_or_null",
            "role": "hard_fail",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_rbd_neutralizer",
              "matches_known_autoreactive_clone",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_heavy",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_light",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_recurrence_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_rank_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "cross_axis_found_dominant_but_failed_to_scan_for_embedded_neutralizer",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_pairing_neutralizer_match": {
                "op": "in",
                "values": [
                  "no_known_neutralizer_match"
                ]
              },
              "neutralizer_pairing_heavy": {
                "op": "in",
                "values": [
                  "none_detected"
                ]
              }
            }
          },
          {
            "name": "neutralizer_pairing_not_exactly_the_s309_pair",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "op": "not",
            "expected": {
              "op": "and",
              "args": [
                {
                  "op": "field",
                  "name": "neutralizer_pairing_heavy",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGHV1-18"
                    ]
                  }
                },
                {
                  "op": "field",
                  "name": "neutralizer_pairing_light",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGKV3-20"
                    ]
                  }
                }
              ]
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-glm-5p2-pi",
      "modelName": "glm-5p2",
      "harness": "pi",
      "task": "gse171524_plasma_bcr_neutralizing_clonotype",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 2801,
      "cost": 2.1127816800000003,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dominant_pairing_heavy_gate",
            "role": "gate",
            "field": "$.dominant_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV4-34"
            ]
          },
          {
            "name": "dominant_pairing_light_gate",
            "role": "gate",
            "field": "$.dominant_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGLV3-19"
            ]
          },
          {
            "name": "dominant_neutralizer_match_gate",
            "role": "gate",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_known_neutralizer_match"
            ]
          },
          {
            "name": "neutralizer_pairing_heavy_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV1-18"
            ]
          },
          {
            "name": "neutralizer_pairing_light_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGKV3-20"
            ]
          },
          {
            "name": "neutralizer_recurrence_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "observed_in_2_to_5_donors"
            ]
          },
          {
            "name": "neutralizer_rank_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "top_5_pairing",
              "top_6_to_20_pairing"
            ]
          },
          {
            "name": "dominant_clone_isotype_gate",
            "role": "gate",
            "field": "$.dominant_clone_isotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "IgG"
            ]
          },
          {
            "name": "blineage_class_switch_contrast_gate",
            "role": "gate",
            "field": "$.blineage_class_switch_contrast",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_class_switched_control_higher_unswitched",
              "class_switched_in_both_cohorts"
            ]
          },
          {
            "name": "dominant_match_narrative_trap_or_null",
            "role": "hard_fail",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_rbd_neutralizer",
              "matches_known_autoreactive_clone",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_heavy",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_light",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_recurrence_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_rank_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "cross_axis_found_dominant_but_failed_to_scan_for_embedded_neutralizer",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_pairing_neutralizer_match": {
                "op": "in",
                "values": [
                  "no_known_neutralizer_match"
                ]
              },
              "neutralizer_pairing_heavy": {
                "op": "in",
                "values": [
                  "none_detected"
                ]
              }
            }
          },
          {
            "name": "neutralizer_pairing_not_exactly_the_s309_pair",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "op": "not",
            "expected": {
              "op": "and",
              "args": [
                {
                  "op": "field",
                  "name": "neutralizer_pairing_heavy",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGHV1-18"
                    ]
                  }
                },
                {
                  "op": "field",
                  "name": "neutralizer_pairing_light",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGKV3-20"
                    ]
                  }
                }
              ]
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-glm-5p2-pi",
      "modelName": "glm-5p2",
      "harness": "pi",
      "task": "gse171524_plasma_bcr_neutralizing_clonotype",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1101,
      "cost": 0.74896706,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dominant_pairing_heavy_gate",
            "role": "gate",
            "field": "$.dominant_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV4-34"
            ]
          },
          {
            "name": "dominant_pairing_light_gate",
            "role": "gate",
            "field": "$.dominant_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGLV3-19"
            ]
          },
          {
            "name": "dominant_neutralizer_match_gate",
            "role": "gate",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_known_neutralizer_match"
            ]
          },
          {
            "name": "neutralizer_pairing_heavy_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV1-18"
            ]
          },
          {
            "name": "neutralizer_pairing_light_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGKV3-20"
            ]
          },
          {
            "name": "neutralizer_recurrence_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "observed_in_2_to_5_donors"
            ]
          },
          {
            "name": "neutralizer_rank_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "top_5_pairing",
              "top_6_to_20_pairing"
            ]
          },
          {
            "name": "dominant_clone_isotype_gate",
            "role": "gate",
            "field": "$.dominant_clone_isotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "IgG"
            ]
          },
          {
            "name": "blineage_class_switch_contrast_gate",
            "role": "gate",
            "field": "$.blineage_class_switch_contrast",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_class_switched_control_higher_unswitched",
              "class_switched_in_both_cohorts"
            ]
          },
          {
            "name": "dominant_match_narrative_trap_or_null",
            "role": "hard_fail",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_rbd_neutralizer",
              "matches_known_autoreactive_clone",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_heavy",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_light",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_recurrence_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_rank_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "cross_axis_found_dominant_but_failed_to_scan_for_embedded_neutralizer",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_pairing_neutralizer_match": {
                "op": "in",
                "values": [
                  "no_known_neutralizer_match"
                ]
              },
              "neutralizer_pairing_heavy": {
                "op": "in",
                "values": [
                  "none_detected"
                ]
              }
            }
          },
          {
            "name": "neutralizer_pairing_not_exactly_the_s309_pair",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "op": "not",
            "expected": {
              "op": "and",
              "args": [
                {
                  "op": "field",
                  "name": "neutralizer_pairing_heavy",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGHV1-18"
                    ]
                  }
                },
                {
                  "op": "field",
                  "name": "neutralizer_pairing_light",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGKV3-20"
                    ]
                  }
                }
              ]
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-4-codex",
      "modelName": "gpt-5.4",
      "harness": "openai-codex",
      "task": "gse171524_plasma_bcr_neutralizing_clonotype",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 672,
      "cost": 1.674747,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dominant_pairing_heavy_gate",
            "role": "gate",
            "field": "$.dominant_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV4-34"
            ]
          },
          {
            "name": "dominant_pairing_light_gate",
            "role": "gate",
            "field": "$.dominant_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGLV3-19"
            ]
          },
          {
            "name": "dominant_neutralizer_match_gate",
            "role": "gate",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_known_neutralizer_match"
            ]
          },
          {
            "name": "neutralizer_pairing_heavy_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV1-18"
            ]
          },
          {
            "name": "neutralizer_pairing_light_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGKV3-20"
            ]
          },
          {
            "name": "neutralizer_recurrence_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "observed_in_2_to_5_donors"
            ]
          },
          {
            "name": "neutralizer_rank_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "top_5_pairing",
              "top_6_to_20_pairing"
            ]
          },
          {
            "name": "dominant_clone_isotype_gate",
            "role": "gate",
            "field": "$.dominant_clone_isotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "IgG"
            ]
          },
          {
            "name": "blineage_class_switch_contrast_gate",
            "role": "gate",
            "field": "$.blineage_class_switch_contrast",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_class_switched_control_higher_unswitched",
              "class_switched_in_both_cohorts"
            ]
          },
          {
            "name": "dominant_match_narrative_trap_or_null",
            "role": "hard_fail",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_rbd_neutralizer",
              "matches_known_autoreactive_clone",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_heavy",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_light",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_recurrence_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_rank_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "cross_axis_found_dominant_but_failed_to_scan_for_embedded_neutralizer",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_pairing_neutralizer_match": {
                "op": "in",
                "values": [
                  "no_known_neutralizer_match"
                ]
              },
              "neutralizer_pairing_heavy": {
                "op": "in",
                "values": [
                  "none_detected"
                ]
              }
            }
          },
          {
            "name": "neutralizer_pairing_not_exactly_the_s309_pair",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "op": "not",
            "expected": {
              "op": "and",
              "args": [
                {
                  "op": "field",
                  "name": "neutralizer_pairing_heavy",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGHV1-18"
                    ]
                  }
                },
                {
                  "op": "field",
                  "name": "neutralizer_pairing_light",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGKV3-20"
                    ]
                  }
                }
              ]
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-4-codex",
      "modelName": "gpt-5.4",
      "harness": "openai-codex",
      "task": "gse171524_plasma_bcr_neutralizing_clonotype",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 358,
      "cost": 1.073042,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dominant_pairing_heavy_gate",
            "role": "gate",
            "field": "$.dominant_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV4-34"
            ]
          },
          {
            "name": "dominant_pairing_light_gate",
            "role": "gate",
            "field": "$.dominant_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGLV3-19"
            ]
          },
          {
            "name": "dominant_neutralizer_match_gate",
            "role": "gate",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_known_neutralizer_match"
            ]
          },
          {
            "name": "neutralizer_pairing_heavy_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV1-18"
            ]
          },
          {
            "name": "neutralizer_pairing_light_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGKV3-20"
            ]
          },
          {
            "name": "neutralizer_recurrence_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "observed_in_2_to_5_donors"
            ]
          },
          {
            "name": "neutralizer_rank_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "top_5_pairing",
              "top_6_to_20_pairing"
            ]
          },
          {
            "name": "dominant_clone_isotype_gate",
            "role": "gate",
            "field": "$.dominant_clone_isotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "IgG"
            ]
          },
          {
            "name": "blineage_class_switch_contrast_gate",
            "role": "gate",
            "field": "$.blineage_class_switch_contrast",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_class_switched_control_higher_unswitched",
              "class_switched_in_both_cohorts"
            ]
          },
          {
            "name": "dominant_match_narrative_trap_or_null",
            "role": "hard_fail",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_rbd_neutralizer",
              "matches_known_autoreactive_clone",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_heavy",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_light",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_recurrence_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_rank_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "cross_axis_found_dominant_but_failed_to_scan_for_embedded_neutralizer",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_pairing_neutralizer_match": {
                "op": "in",
                "values": [
                  "no_known_neutralizer_match"
                ]
              },
              "neutralizer_pairing_heavy": {
                "op": "in",
                "values": [
                  "none_detected"
                ]
              }
            }
          },
          {
            "name": "neutralizer_pairing_not_exactly_the_s309_pair",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "op": "not",
            "expected": {
              "op": "and",
              "args": [
                {
                  "op": "field",
                  "name": "neutralizer_pairing_heavy",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGHV1-18"
                    ]
                  }
                },
                {
                  "op": "field",
                  "name": "neutralizer_pairing_light",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGKV3-20"
                    ]
                  }
                }
              ]
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-4-codex",
      "modelName": "gpt-5.4",
      "harness": "openai-codex",
      "task": "gse171524_plasma_bcr_neutralizing_clonotype",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 464,
      "cost": 1.3284345,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dominant_pairing_heavy_gate",
            "role": "gate",
            "field": "$.dominant_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV4-34"
            ]
          },
          {
            "name": "dominant_pairing_light_gate",
            "role": "gate",
            "field": "$.dominant_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGLV3-19"
            ]
          },
          {
            "name": "dominant_neutralizer_match_gate",
            "role": "gate",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_known_neutralizer_match"
            ]
          },
          {
            "name": "neutralizer_pairing_heavy_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV1-18"
            ]
          },
          {
            "name": "neutralizer_pairing_light_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGKV3-20"
            ]
          },
          {
            "name": "neutralizer_recurrence_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "observed_in_2_to_5_donors"
            ]
          },
          {
            "name": "neutralizer_rank_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "top_5_pairing",
              "top_6_to_20_pairing"
            ]
          },
          {
            "name": "dominant_clone_isotype_gate",
            "role": "gate",
            "field": "$.dominant_clone_isotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "IgG"
            ]
          },
          {
            "name": "blineage_class_switch_contrast_gate",
            "role": "gate",
            "field": "$.blineage_class_switch_contrast",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_class_switched_control_higher_unswitched",
              "class_switched_in_both_cohorts"
            ]
          },
          {
            "name": "dominant_match_narrative_trap_or_null",
            "role": "hard_fail",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_rbd_neutralizer",
              "matches_known_autoreactive_clone",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_heavy",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_light",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_recurrence_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_rank_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "cross_axis_found_dominant_but_failed_to_scan_for_embedded_neutralizer",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_pairing_neutralizer_match": {
                "op": "in",
                "values": [
                  "no_known_neutralizer_match"
                ]
              },
              "neutralizer_pairing_heavy": {
                "op": "in",
                "values": [
                  "none_detected"
                ]
              }
            }
          },
          {
            "name": "neutralizer_pairing_not_exactly_the_s309_pair",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "op": "not",
            "expected": {
              "op": "and",
              "args": [
                {
                  "op": "field",
                  "name": "neutralizer_pairing_heavy",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGHV1-18"
                    ]
                  }
                },
                {
                  "op": "field",
                  "name": "neutralizer_pairing_light",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGKV3-20"
                    ]
                  }
                }
              ]
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-4-pi",
      "modelName": "gpt-5.4",
      "harness": "pi",
      "task": "gse171524_plasma_bcr_neutralizing_clonotype",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 1,
      "passed": true,
      "score": null,
      "durationS": 1363,
      "cost": 0.8580505,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dominant_pairing_heavy_gate",
            "role": "gate",
            "field": "$.dominant_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV4-34"
            ]
          },
          {
            "name": "dominant_pairing_light_gate",
            "role": "gate",
            "field": "$.dominant_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGLV3-19"
            ]
          },
          {
            "name": "dominant_neutralizer_match_gate",
            "role": "gate",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_known_neutralizer_match"
            ]
          },
          {
            "name": "neutralizer_pairing_heavy_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV1-18"
            ]
          },
          {
            "name": "neutralizer_pairing_light_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGKV3-20"
            ]
          },
          {
            "name": "neutralizer_recurrence_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "observed_in_2_to_5_donors"
            ]
          },
          {
            "name": "neutralizer_rank_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "top_5_pairing",
              "top_6_to_20_pairing"
            ]
          },
          {
            "name": "dominant_clone_isotype_gate",
            "role": "gate",
            "field": "$.dominant_clone_isotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "IgG"
            ]
          },
          {
            "name": "blineage_class_switch_contrast_gate",
            "role": "gate",
            "field": "$.blineage_class_switch_contrast",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_class_switched_control_higher_unswitched",
              "class_switched_in_both_cohorts"
            ]
          },
          {
            "name": "dominant_match_narrative_trap_or_null",
            "role": "hard_fail",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_rbd_neutralizer",
              "matches_known_autoreactive_clone",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_heavy",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_light",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_recurrence_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_rank_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "cross_axis_found_dominant_but_failed_to_scan_for_embedded_neutralizer",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_pairing_neutralizer_match": {
                "op": "in",
                "values": [
                  "no_known_neutralizer_match"
                ]
              },
              "neutralizer_pairing_heavy": {
                "op": "in",
                "values": [
                  "none_detected"
                ]
              }
            }
          },
          {
            "name": "neutralizer_pairing_not_exactly_the_s309_pair",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "op": "not",
            "expected": {
              "op": "and",
              "args": [
                {
                  "op": "field",
                  "name": "neutralizer_pairing_heavy",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGHV1-18"
                    ]
                  }
                },
                {
                  "op": "field",
                  "name": "neutralizer_pairing_light",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGKV3-20"
                    ]
                  }
                }
              ]
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-4-pi",
      "modelName": "gpt-5.4",
      "harness": "pi",
      "task": "gse171524_plasma_bcr_neutralizing_clonotype",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 842,
      "cost": 2.0426634999999997,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dominant_pairing_heavy_gate",
            "role": "gate",
            "field": "$.dominant_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV4-34"
            ]
          },
          {
            "name": "dominant_pairing_light_gate",
            "role": "gate",
            "field": "$.dominant_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGLV3-19"
            ]
          },
          {
            "name": "dominant_neutralizer_match_gate",
            "role": "gate",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_known_neutralizer_match"
            ]
          },
          {
            "name": "neutralizer_pairing_heavy_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV1-18"
            ]
          },
          {
            "name": "neutralizer_pairing_light_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGKV3-20"
            ]
          },
          {
            "name": "neutralizer_recurrence_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "observed_in_2_to_5_donors"
            ]
          },
          {
            "name": "neutralizer_rank_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "top_5_pairing",
              "top_6_to_20_pairing"
            ]
          },
          {
            "name": "dominant_clone_isotype_gate",
            "role": "gate",
            "field": "$.dominant_clone_isotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "IgG"
            ]
          },
          {
            "name": "blineage_class_switch_contrast_gate",
            "role": "gate",
            "field": "$.blineage_class_switch_contrast",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_class_switched_control_higher_unswitched",
              "class_switched_in_both_cohorts"
            ]
          },
          {
            "name": "dominant_match_narrative_trap_or_null",
            "role": "hard_fail",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_rbd_neutralizer",
              "matches_known_autoreactive_clone",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_heavy",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_light",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_recurrence_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_rank_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "cross_axis_found_dominant_but_failed_to_scan_for_embedded_neutralizer",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_pairing_neutralizer_match": {
                "op": "in",
                "values": [
                  "no_known_neutralizer_match"
                ]
              },
              "neutralizer_pairing_heavy": {
                "op": "in",
                "values": [
                  "none_detected"
                ]
              }
            }
          },
          {
            "name": "neutralizer_pairing_not_exactly_the_s309_pair",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "op": "not",
            "expected": {
              "op": "and",
              "args": [
                {
                  "op": "field",
                  "name": "neutralizer_pairing_heavy",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGHV1-18"
                    ]
                  }
                },
                {
                  "op": "field",
                  "name": "neutralizer_pairing_light",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGKV3-20"
                    ]
                  }
                }
              ]
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-4-pi",
      "modelName": "gpt-5.4",
      "harness": "pi",
      "task": "gse171524_plasma_bcr_neutralizing_clonotype",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 676,
      "cost": 0.685895,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dominant_pairing_heavy_gate",
            "role": "gate",
            "field": "$.dominant_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV4-34"
            ]
          },
          {
            "name": "dominant_pairing_light_gate",
            "role": "gate",
            "field": "$.dominant_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGLV3-19"
            ]
          },
          {
            "name": "dominant_neutralizer_match_gate",
            "role": "gate",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_known_neutralizer_match"
            ]
          },
          {
            "name": "neutralizer_pairing_heavy_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV1-18"
            ]
          },
          {
            "name": "neutralizer_pairing_light_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGKV3-20"
            ]
          },
          {
            "name": "neutralizer_recurrence_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "observed_in_2_to_5_donors"
            ]
          },
          {
            "name": "neutralizer_rank_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "top_5_pairing",
              "top_6_to_20_pairing"
            ]
          },
          {
            "name": "dominant_clone_isotype_gate",
            "role": "gate",
            "field": "$.dominant_clone_isotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "IgG"
            ]
          },
          {
            "name": "blineage_class_switch_contrast_gate",
            "role": "gate",
            "field": "$.blineage_class_switch_contrast",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_class_switched_control_higher_unswitched",
              "class_switched_in_both_cohorts"
            ]
          },
          {
            "name": "dominant_match_narrative_trap_or_null",
            "role": "hard_fail",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_rbd_neutralizer",
              "matches_known_autoreactive_clone",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_heavy",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_light",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_recurrence_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_rank_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "cross_axis_found_dominant_but_failed_to_scan_for_embedded_neutralizer",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_pairing_neutralizer_match": {
                "op": "in",
                "values": [
                  "no_known_neutralizer_match"
                ]
              },
              "neutralizer_pairing_heavy": {
                "op": "in",
                "values": [
                  "none_detected"
                ]
              }
            }
          },
          {
            "name": "neutralizer_pairing_not_exactly_the_s309_pair",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "op": "not",
            "expected": {
              "op": "and",
              "args": [
                {
                  "op": "field",
                  "name": "neutralizer_pairing_heavy",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGHV1-18"
                    ]
                  }
                },
                {
                  "op": "field",
                  "name": "neutralizer_pairing_light",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGKV3-20"
                    ]
                  }
                }
              ]
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-5-codex",
      "modelName": "gpt-5.5",
      "harness": "openai-codex",
      "task": "gse171524_plasma_bcr_neutralizing_clonotype",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 256,
      "cost": 0.9692529999999999,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dominant_pairing_heavy_gate",
            "role": "gate",
            "field": "$.dominant_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV4-34"
            ]
          },
          {
            "name": "dominant_pairing_light_gate",
            "role": "gate",
            "field": "$.dominant_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGLV3-19"
            ]
          },
          {
            "name": "dominant_neutralizer_match_gate",
            "role": "gate",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_known_neutralizer_match"
            ]
          },
          {
            "name": "neutralizer_pairing_heavy_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV1-18"
            ]
          },
          {
            "name": "neutralizer_pairing_light_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGKV3-20"
            ]
          },
          {
            "name": "neutralizer_recurrence_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "observed_in_2_to_5_donors"
            ]
          },
          {
            "name": "neutralizer_rank_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "top_5_pairing",
              "top_6_to_20_pairing"
            ]
          },
          {
            "name": "dominant_clone_isotype_gate",
            "role": "gate",
            "field": "$.dominant_clone_isotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "IgG"
            ]
          },
          {
            "name": "blineage_class_switch_contrast_gate",
            "role": "gate",
            "field": "$.blineage_class_switch_contrast",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_class_switched_control_higher_unswitched",
              "class_switched_in_both_cohorts"
            ]
          },
          {
            "name": "dominant_match_narrative_trap_or_null",
            "role": "hard_fail",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_rbd_neutralizer",
              "matches_known_autoreactive_clone",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_heavy",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_light",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_recurrence_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_rank_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "cross_axis_found_dominant_but_failed_to_scan_for_embedded_neutralizer",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_pairing_neutralizer_match": {
                "op": "in",
                "values": [
                  "no_known_neutralizer_match"
                ]
              },
              "neutralizer_pairing_heavy": {
                "op": "in",
                "values": [
                  "none_detected"
                ]
              }
            }
          },
          {
            "name": "neutralizer_pairing_not_exactly_the_s309_pair",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "op": "not",
            "expected": {
              "op": "and",
              "args": [
                {
                  "op": "field",
                  "name": "neutralizer_pairing_heavy",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGHV1-18"
                    ]
                  }
                },
                {
                  "op": "field",
                  "name": "neutralizer_pairing_light",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGKV3-20"
                    ]
                  }
                }
              ]
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-5-codex",
      "modelName": "gpt-5.5",
      "harness": "openai-codex",
      "task": "gse171524_plasma_bcr_neutralizing_clonotype",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 275,
      "cost": 1.0258990000000001,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dominant_pairing_heavy_gate",
            "role": "gate",
            "field": "$.dominant_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV4-34"
            ]
          },
          {
            "name": "dominant_pairing_light_gate",
            "role": "gate",
            "field": "$.dominant_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGLV3-19"
            ]
          },
          {
            "name": "dominant_neutralizer_match_gate",
            "role": "gate",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_known_neutralizer_match"
            ]
          },
          {
            "name": "neutralizer_pairing_heavy_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV1-18"
            ]
          },
          {
            "name": "neutralizer_pairing_light_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGKV3-20"
            ]
          },
          {
            "name": "neutralizer_recurrence_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "observed_in_2_to_5_donors"
            ]
          },
          {
            "name": "neutralizer_rank_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "top_5_pairing",
              "top_6_to_20_pairing"
            ]
          },
          {
            "name": "dominant_clone_isotype_gate",
            "role": "gate",
            "field": "$.dominant_clone_isotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "IgG"
            ]
          },
          {
            "name": "blineage_class_switch_contrast_gate",
            "role": "gate",
            "field": "$.blineage_class_switch_contrast",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_class_switched_control_higher_unswitched",
              "class_switched_in_both_cohorts"
            ]
          },
          {
            "name": "dominant_match_narrative_trap_or_null",
            "role": "hard_fail",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_rbd_neutralizer",
              "matches_known_autoreactive_clone",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_heavy",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_light",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_recurrence_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_rank_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "cross_axis_found_dominant_but_failed_to_scan_for_embedded_neutralizer",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_pairing_neutralizer_match": {
                "op": "in",
                "values": [
                  "no_known_neutralizer_match"
                ]
              },
              "neutralizer_pairing_heavy": {
                "op": "in",
                "values": [
                  "none_detected"
                ]
              }
            }
          },
          {
            "name": "neutralizer_pairing_not_exactly_the_s309_pair",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "op": "not",
            "expected": {
              "op": "and",
              "args": [
                {
                  "op": "field",
                  "name": "neutralizer_pairing_heavy",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGHV1-18"
                    ]
                  }
                },
                {
                  "op": "field",
                  "name": "neutralizer_pairing_light",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGKV3-20"
                    ]
                  }
                }
              ]
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-5-codex",
      "modelName": "gpt-5.5",
      "harness": "openai-codex",
      "task": "gse171524_plasma_bcr_neutralizing_clonotype",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 340,
      "cost": 1.087486,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dominant_pairing_heavy_gate",
            "role": "gate",
            "field": "$.dominant_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV4-34"
            ]
          },
          {
            "name": "dominant_pairing_light_gate",
            "role": "gate",
            "field": "$.dominant_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGLV3-19"
            ]
          },
          {
            "name": "dominant_neutralizer_match_gate",
            "role": "gate",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_known_neutralizer_match"
            ]
          },
          {
            "name": "neutralizer_pairing_heavy_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV1-18"
            ]
          },
          {
            "name": "neutralizer_pairing_light_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGKV3-20"
            ]
          },
          {
            "name": "neutralizer_recurrence_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "observed_in_2_to_5_donors"
            ]
          },
          {
            "name": "neutralizer_rank_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "top_5_pairing",
              "top_6_to_20_pairing"
            ]
          },
          {
            "name": "dominant_clone_isotype_gate",
            "role": "gate",
            "field": "$.dominant_clone_isotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "IgG"
            ]
          },
          {
            "name": "blineage_class_switch_contrast_gate",
            "role": "gate",
            "field": "$.blineage_class_switch_contrast",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_class_switched_control_higher_unswitched",
              "class_switched_in_both_cohorts"
            ]
          },
          {
            "name": "dominant_match_narrative_trap_or_null",
            "role": "hard_fail",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_rbd_neutralizer",
              "matches_known_autoreactive_clone",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_heavy",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_light",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_recurrence_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_rank_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "cross_axis_found_dominant_but_failed_to_scan_for_embedded_neutralizer",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_pairing_neutralizer_match": {
                "op": "in",
                "values": [
                  "no_known_neutralizer_match"
                ]
              },
              "neutralizer_pairing_heavy": {
                "op": "in",
                "values": [
                  "none_detected"
                ]
              }
            }
          },
          {
            "name": "neutralizer_pairing_not_exactly_the_s309_pair",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "op": "not",
            "expected": {
              "op": "and",
              "args": [
                {
                  "op": "field",
                  "name": "neutralizer_pairing_heavy",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGHV1-18"
                    ]
                  }
                },
                {
                  "op": "field",
                  "name": "neutralizer_pairing_light",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGKV3-20"
                    ]
                  }
                }
              ]
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-5-pi",
      "modelName": "gpt-5.5",
      "harness": "pi",
      "task": "gse171524_plasma_bcr_neutralizing_clonotype",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 444,
      "cost": 1.169156,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dominant_pairing_heavy_gate",
            "role": "gate",
            "field": "$.dominant_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV4-34"
            ]
          },
          {
            "name": "dominant_pairing_light_gate",
            "role": "gate",
            "field": "$.dominant_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGLV3-19"
            ]
          },
          {
            "name": "dominant_neutralizer_match_gate",
            "role": "gate",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_known_neutralizer_match"
            ]
          },
          {
            "name": "neutralizer_pairing_heavy_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV1-18"
            ]
          },
          {
            "name": "neutralizer_pairing_light_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGKV3-20"
            ]
          },
          {
            "name": "neutralizer_recurrence_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "observed_in_2_to_5_donors"
            ]
          },
          {
            "name": "neutralizer_rank_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "top_5_pairing",
              "top_6_to_20_pairing"
            ]
          },
          {
            "name": "dominant_clone_isotype_gate",
            "role": "gate",
            "field": "$.dominant_clone_isotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "IgG"
            ]
          },
          {
            "name": "blineage_class_switch_contrast_gate",
            "role": "gate",
            "field": "$.blineage_class_switch_contrast",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_class_switched_control_higher_unswitched",
              "class_switched_in_both_cohorts"
            ]
          },
          {
            "name": "dominant_match_narrative_trap_or_null",
            "role": "hard_fail",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_rbd_neutralizer",
              "matches_known_autoreactive_clone",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_heavy",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_light",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_recurrence_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_rank_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "cross_axis_found_dominant_but_failed_to_scan_for_embedded_neutralizer",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_pairing_neutralizer_match": {
                "op": "in",
                "values": [
                  "no_known_neutralizer_match"
                ]
              },
              "neutralizer_pairing_heavy": {
                "op": "in",
                "values": [
                  "none_detected"
                ]
              }
            }
          },
          {
            "name": "neutralizer_pairing_not_exactly_the_s309_pair",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "op": "not",
            "expected": {
              "op": "and",
              "args": [
                {
                  "op": "field",
                  "name": "neutralizer_pairing_heavy",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGHV1-18"
                    ]
                  }
                },
                {
                  "op": "field",
                  "name": "neutralizer_pairing_light",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGKV3-20"
                    ]
                  }
                }
              ]
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-5-pi",
      "modelName": "gpt-5.5",
      "harness": "pi",
      "task": "gse171524_plasma_bcr_neutralizing_clonotype",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 313,
      "cost": 1.125341,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dominant_pairing_heavy_gate",
            "role": "gate",
            "field": "$.dominant_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV4-34"
            ]
          },
          {
            "name": "dominant_pairing_light_gate",
            "role": "gate",
            "field": "$.dominant_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGLV3-19"
            ]
          },
          {
            "name": "dominant_neutralizer_match_gate",
            "role": "gate",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_known_neutralizer_match"
            ]
          },
          {
            "name": "neutralizer_pairing_heavy_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV1-18"
            ]
          },
          {
            "name": "neutralizer_pairing_light_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGKV3-20"
            ]
          },
          {
            "name": "neutralizer_recurrence_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "observed_in_2_to_5_donors"
            ]
          },
          {
            "name": "neutralizer_rank_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "top_5_pairing",
              "top_6_to_20_pairing"
            ]
          },
          {
            "name": "dominant_clone_isotype_gate",
            "role": "gate",
            "field": "$.dominant_clone_isotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "IgG"
            ]
          },
          {
            "name": "blineage_class_switch_contrast_gate",
            "role": "gate",
            "field": "$.blineage_class_switch_contrast",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_class_switched_control_higher_unswitched",
              "class_switched_in_both_cohorts"
            ]
          },
          {
            "name": "dominant_match_narrative_trap_or_null",
            "role": "hard_fail",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_rbd_neutralizer",
              "matches_known_autoreactive_clone",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_heavy",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_light",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_recurrence_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_rank_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "cross_axis_found_dominant_but_failed_to_scan_for_embedded_neutralizer",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_pairing_neutralizer_match": {
                "op": "in",
                "values": [
                  "no_known_neutralizer_match"
                ]
              },
              "neutralizer_pairing_heavy": {
                "op": "in",
                "values": [
                  "none_detected"
                ]
              }
            }
          },
          {
            "name": "neutralizer_pairing_not_exactly_the_s309_pair",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "op": "not",
            "expected": {
              "op": "and",
              "args": [
                {
                  "op": "field",
                  "name": "neutralizer_pairing_heavy",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGHV1-18"
                    ]
                  }
                },
                {
                  "op": "field",
                  "name": "neutralizer_pairing_light",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGKV3-20"
                    ]
                  }
                }
              ]
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-5-pi",
      "modelName": "gpt-5.5",
      "harness": "pi",
      "task": "gse171524_plasma_bcr_neutralizing_clonotype",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 483,
      "cost": 1.2837560000000001,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dominant_pairing_heavy_gate",
            "role": "gate",
            "field": "$.dominant_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV4-34"
            ]
          },
          {
            "name": "dominant_pairing_light_gate",
            "role": "gate",
            "field": "$.dominant_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGLV3-19"
            ]
          },
          {
            "name": "dominant_neutralizer_match_gate",
            "role": "gate",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_known_neutralizer_match"
            ]
          },
          {
            "name": "neutralizer_pairing_heavy_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV1-18"
            ]
          },
          {
            "name": "neutralizer_pairing_light_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGKV3-20"
            ]
          },
          {
            "name": "neutralizer_recurrence_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "observed_in_2_to_5_donors"
            ]
          },
          {
            "name": "neutralizer_rank_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "top_5_pairing",
              "top_6_to_20_pairing"
            ]
          },
          {
            "name": "dominant_clone_isotype_gate",
            "role": "gate",
            "field": "$.dominant_clone_isotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "IgG"
            ]
          },
          {
            "name": "blineage_class_switch_contrast_gate",
            "role": "gate",
            "field": "$.blineage_class_switch_contrast",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_class_switched_control_higher_unswitched",
              "class_switched_in_both_cohorts"
            ]
          },
          {
            "name": "dominant_match_narrative_trap_or_null",
            "role": "hard_fail",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_rbd_neutralizer",
              "matches_known_autoreactive_clone",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_heavy",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_light",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_recurrence_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_rank_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "cross_axis_found_dominant_but_failed_to_scan_for_embedded_neutralizer",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_pairing_neutralizer_match": {
                "op": "in",
                "values": [
                  "no_known_neutralizer_match"
                ]
              },
              "neutralizer_pairing_heavy": {
                "op": "in",
                "values": [
                  "none_detected"
                ]
              }
            }
          },
          {
            "name": "neutralizer_pairing_not_exactly_the_s309_pair",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "op": "not",
            "expected": {
              "op": "and",
              "args": [
                {
                  "op": "field",
                  "name": "neutralizer_pairing_heavy",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGHV1-18"
                    ]
                  }
                },
                {
                  "op": "field",
                  "name": "neutralizer_pairing_light",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGKV3-20"
                    ]
                  }
                }
              ]
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-20-beta-reasoning-pi",
      "modelName": "grok-4.20-beta-0309-reasoning",
      "harness": "pi",
      "task": "gse171524_plasma_bcr_neutralizing_clonotype",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 147,
      "cost": 0.06406300000000001,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dominant_pairing_heavy_gate",
            "role": "gate",
            "field": "$.dominant_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV4-34"
            ]
          },
          {
            "name": "dominant_pairing_light_gate",
            "role": "gate",
            "field": "$.dominant_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGLV3-19"
            ]
          },
          {
            "name": "dominant_neutralizer_match_gate",
            "role": "gate",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_known_neutralizer_match"
            ]
          },
          {
            "name": "neutralizer_pairing_heavy_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV1-18"
            ]
          },
          {
            "name": "neutralizer_pairing_light_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGKV3-20"
            ]
          },
          {
            "name": "neutralizer_recurrence_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "observed_in_2_to_5_donors"
            ]
          },
          {
            "name": "neutralizer_rank_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "top_5_pairing",
              "top_6_to_20_pairing"
            ]
          },
          {
            "name": "dominant_clone_isotype_gate",
            "role": "gate",
            "field": "$.dominant_clone_isotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "IgG"
            ]
          },
          {
            "name": "blineage_class_switch_contrast_gate",
            "role": "gate",
            "field": "$.blineage_class_switch_contrast",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_class_switched_control_higher_unswitched",
              "class_switched_in_both_cohorts"
            ]
          },
          {
            "name": "dominant_match_narrative_trap_or_null",
            "role": "hard_fail",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_rbd_neutralizer",
              "matches_known_autoreactive_clone",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_heavy",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_light",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_recurrence_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_rank_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "cross_axis_found_dominant_but_failed_to_scan_for_embedded_neutralizer",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_pairing_neutralizer_match": {
                "op": "in",
                "values": [
                  "no_known_neutralizer_match"
                ]
              },
              "neutralizer_pairing_heavy": {
                "op": "in",
                "values": [
                  "none_detected"
                ]
              }
            }
          },
          {
            "name": "neutralizer_pairing_not_exactly_the_s309_pair",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "op": "not",
            "expected": {
              "op": "and",
              "args": [
                {
                  "op": "field",
                  "name": "neutralizer_pairing_heavy",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGHV1-18"
                    ]
                  }
                },
                {
                  "op": "field",
                  "name": "neutralizer_pairing_light",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGKV3-20"
                    ]
                  }
                }
              ]
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-20-beta-reasoning-pi",
      "modelName": "grok-4.20-beta-0309-reasoning",
      "harness": "pi",
      "task": "gse171524_plasma_bcr_neutralizing_clonotype",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 62,
      "cost": 0.04296250000000001,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dominant_pairing_heavy_gate",
            "role": "gate",
            "field": "$.dominant_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV4-34"
            ]
          },
          {
            "name": "dominant_pairing_light_gate",
            "role": "gate",
            "field": "$.dominant_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGLV3-19"
            ]
          },
          {
            "name": "dominant_neutralizer_match_gate",
            "role": "gate",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_known_neutralizer_match"
            ]
          },
          {
            "name": "neutralizer_pairing_heavy_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV1-18"
            ]
          },
          {
            "name": "neutralizer_pairing_light_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGKV3-20"
            ]
          },
          {
            "name": "neutralizer_recurrence_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "observed_in_2_to_5_donors"
            ]
          },
          {
            "name": "neutralizer_rank_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "top_5_pairing",
              "top_6_to_20_pairing"
            ]
          },
          {
            "name": "dominant_clone_isotype_gate",
            "role": "gate",
            "field": "$.dominant_clone_isotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "IgG"
            ]
          },
          {
            "name": "blineage_class_switch_contrast_gate",
            "role": "gate",
            "field": "$.blineage_class_switch_contrast",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_class_switched_control_higher_unswitched",
              "class_switched_in_both_cohorts"
            ]
          },
          {
            "name": "dominant_match_narrative_trap_or_null",
            "role": "hard_fail",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_rbd_neutralizer",
              "matches_known_autoreactive_clone",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_heavy",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_light",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_recurrence_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_rank_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "cross_axis_found_dominant_but_failed_to_scan_for_embedded_neutralizer",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_pairing_neutralizer_match": {
                "op": "in",
                "values": [
                  "no_known_neutralizer_match"
                ]
              },
              "neutralizer_pairing_heavy": {
                "op": "in",
                "values": [
                  "none_detected"
                ]
              }
            }
          },
          {
            "name": "neutralizer_pairing_not_exactly_the_s309_pair",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "op": "not",
            "expected": {
              "op": "and",
              "args": [
                {
                  "op": "field",
                  "name": "neutralizer_pairing_heavy",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGHV1-18"
                    ]
                  }
                },
                {
                  "op": "field",
                  "name": "neutralizer_pairing_light",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGKV3-20"
                    ]
                  }
                }
              ]
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-20-beta-reasoning-pi",
      "modelName": "grok-4.20-beta-0309-reasoning",
      "harness": "pi",
      "task": "gse171524_plasma_bcr_neutralizing_clonotype",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 123,
      "cost": 0.03863015,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dominant_pairing_heavy_gate",
            "role": "gate",
            "field": "$.dominant_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV4-34"
            ]
          },
          {
            "name": "dominant_pairing_light_gate",
            "role": "gate",
            "field": "$.dominant_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGLV3-19"
            ]
          },
          {
            "name": "dominant_neutralizer_match_gate",
            "role": "gate",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_known_neutralizer_match"
            ]
          },
          {
            "name": "neutralizer_pairing_heavy_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV1-18"
            ]
          },
          {
            "name": "neutralizer_pairing_light_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGKV3-20"
            ]
          },
          {
            "name": "neutralizer_recurrence_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "observed_in_2_to_5_donors"
            ]
          },
          {
            "name": "neutralizer_rank_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "top_5_pairing",
              "top_6_to_20_pairing"
            ]
          },
          {
            "name": "dominant_clone_isotype_gate",
            "role": "gate",
            "field": "$.dominant_clone_isotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "IgG"
            ]
          },
          {
            "name": "blineage_class_switch_contrast_gate",
            "role": "gate",
            "field": "$.blineage_class_switch_contrast",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_class_switched_control_higher_unswitched",
              "class_switched_in_both_cohorts"
            ]
          },
          {
            "name": "dominant_match_narrative_trap_or_null",
            "role": "hard_fail",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_rbd_neutralizer",
              "matches_known_autoreactive_clone",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_heavy",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_light",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_recurrence_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_rank_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "cross_axis_found_dominant_but_failed_to_scan_for_embedded_neutralizer",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_pairing_neutralizer_match": {
                "op": "in",
                "values": [
                  "no_known_neutralizer_match"
                ]
              },
              "neutralizer_pairing_heavy": {
                "op": "in",
                "values": [
                  "none_detected"
                ]
              }
            }
          },
          {
            "name": "neutralizer_pairing_not_exactly_the_s309_pair",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "op": "not",
            "expected": {
              "op": "and",
              "args": [
                {
                  "op": "field",
                  "name": "neutralizer_pairing_heavy",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGHV1-18"
                    ]
                  }
                },
                {
                  "op": "field",
                  "name": "neutralizer_pairing_light",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGKV3-20"
                    ]
                  }
                }
              ]
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-3-pi",
      "modelName": "grok-4.3",
      "harness": "pi",
      "task": "gse171524_plasma_bcr_neutralizing_clonotype",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 89,
      "cost": 0.023120950000000005,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dominant_pairing_heavy_gate",
            "role": "gate",
            "field": "$.dominant_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV4-34"
            ]
          },
          {
            "name": "dominant_pairing_light_gate",
            "role": "gate",
            "field": "$.dominant_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGLV3-19"
            ]
          },
          {
            "name": "dominant_neutralizer_match_gate",
            "role": "gate",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_known_neutralizer_match"
            ]
          },
          {
            "name": "neutralizer_pairing_heavy_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV1-18"
            ]
          },
          {
            "name": "neutralizer_pairing_light_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGKV3-20"
            ]
          },
          {
            "name": "neutralizer_recurrence_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "observed_in_2_to_5_donors"
            ]
          },
          {
            "name": "neutralizer_rank_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "top_5_pairing",
              "top_6_to_20_pairing"
            ]
          },
          {
            "name": "dominant_clone_isotype_gate",
            "role": "gate",
            "field": "$.dominant_clone_isotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "IgG"
            ]
          },
          {
            "name": "blineage_class_switch_contrast_gate",
            "role": "gate",
            "field": "$.blineage_class_switch_contrast",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_class_switched_control_higher_unswitched",
              "class_switched_in_both_cohorts"
            ]
          },
          {
            "name": "dominant_match_narrative_trap_or_null",
            "role": "hard_fail",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_rbd_neutralizer",
              "matches_known_autoreactive_clone",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_heavy",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_light",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_recurrence_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_rank_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "cross_axis_found_dominant_but_failed_to_scan_for_embedded_neutralizer",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_pairing_neutralizer_match": {
                "op": "in",
                "values": [
                  "no_known_neutralizer_match"
                ]
              },
              "neutralizer_pairing_heavy": {
                "op": "in",
                "values": [
                  "none_detected"
                ]
              }
            }
          },
          {
            "name": "neutralizer_pairing_not_exactly_the_s309_pair",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "op": "not",
            "expected": {
              "op": "and",
              "args": [
                {
                  "op": "field",
                  "name": "neutralizer_pairing_heavy",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGHV1-18"
                    ]
                  }
                },
                {
                  "op": "field",
                  "name": "neutralizer_pairing_light",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGKV3-20"
                    ]
                  }
                }
              ]
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-3-pi",
      "modelName": "grok-4.3",
      "harness": "pi",
      "task": "gse171524_plasma_bcr_neutralizing_clonotype",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 146,
      "cost": 0.03692795,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dominant_pairing_heavy_gate",
            "role": "gate",
            "field": "$.dominant_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV4-34"
            ]
          },
          {
            "name": "dominant_pairing_light_gate",
            "role": "gate",
            "field": "$.dominant_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGLV3-19"
            ]
          },
          {
            "name": "dominant_neutralizer_match_gate",
            "role": "gate",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_known_neutralizer_match"
            ]
          },
          {
            "name": "neutralizer_pairing_heavy_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV1-18"
            ]
          },
          {
            "name": "neutralizer_pairing_light_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGKV3-20"
            ]
          },
          {
            "name": "neutralizer_recurrence_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "observed_in_2_to_5_donors"
            ]
          },
          {
            "name": "neutralizer_rank_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "top_5_pairing",
              "top_6_to_20_pairing"
            ]
          },
          {
            "name": "dominant_clone_isotype_gate",
            "role": "gate",
            "field": "$.dominant_clone_isotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "IgG"
            ]
          },
          {
            "name": "blineage_class_switch_contrast_gate",
            "role": "gate",
            "field": "$.blineage_class_switch_contrast",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_class_switched_control_higher_unswitched",
              "class_switched_in_both_cohorts"
            ]
          },
          {
            "name": "dominant_match_narrative_trap_or_null",
            "role": "hard_fail",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_rbd_neutralizer",
              "matches_known_autoreactive_clone",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_heavy",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_light",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_recurrence_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_rank_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "cross_axis_found_dominant_but_failed_to_scan_for_embedded_neutralizer",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_pairing_neutralizer_match": {
                "op": "in",
                "values": [
                  "no_known_neutralizer_match"
                ]
              },
              "neutralizer_pairing_heavy": {
                "op": "in",
                "values": [
                  "none_detected"
                ]
              }
            }
          },
          {
            "name": "neutralizer_pairing_not_exactly_the_s309_pair",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "op": "not",
            "expected": {
              "op": "and",
              "args": [
                {
                  "op": "field",
                  "name": "neutralizer_pairing_heavy",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGHV1-18"
                    ]
                  }
                },
                {
                  "op": "field",
                  "name": "neutralizer_pairing_light",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGKV3-20"
                    ]
                  }
                }
              ]
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-3-pi",
      "modelName": "grok-4.3",
      "harness": "pi",
      "task": "gse171524_plasma_bcr_neutralizing_clonotype",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 139,
      "cost": 0.0240496,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dominant_pairing_heavy_gate",
            "role": "gate",
            "field": "$.dominant_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV4-34"
            ]
          },
          {
            "name": "dominant_pairing_light_gate",
            "role": "gate",
            "field": "$.dominant_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGLV3-19"
            ]
          },
          {
            "name": "dominant_neutralizer_match_gate",
            "role": "gate",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_known_neutralizer_match"
            ]
          },
          {
            "name": "neutralizer_pairing_heavy_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV1-18"
            ]
          },
          {
            "name": "neutralizer_pairing_light_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGKV3-20"
            ]
          },
          {
            "name": "neutralizer_recurrence_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "observed_in_2_to_5_donors"
            ]
          },
          {
            "name": "neutralizer_rank_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "top_5_pairing",
              "top_6_to_20_pairing"
            ]
          },
          {
            "name": "dominant_clone_isotype_gate",
            "role": "gate",
            "field": "$.dominant_clone_isotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "IgG"
            ]
          },
          {
            "name": "blineage_class_switch_contrast_gate",
            "role": "gate",
            "field": "$.blineage_class_switch_contrast",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_class_switched_control_higher_unswitched",
              "class_switched_in_both_cohorts"
            ]
          },
          {
            "name": "dominant_match_narrative_trap_or_null",
            "role": "hard_fail",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_rbd_neutralizer",
              "matches_known_autoreactive_clone",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_heavy",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_light",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_recurrence_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_rank_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "cross_axis_found_dominant_but_failed_to_scan_for_embedded_neutralizer",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_pairing_neutralizer_match": {
                "op": "in",
                "values": [
                  "no_known_neutralizer_match"
                ]
              },
              "neutralizer_pairing_heavy": {
                "op": "in",
                "values": [
                  "none_detected"
                ]
              }
            }
          },
          {
            "name": "neutralizer_pairing_not_exactly_the_s309_pair",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "op": "not",
            "expected": {
              "op": "and",
              "args": [
                {
                  "op": "field",
                  "name": "neutralizer_pairing_heavy",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGHV1-18"
                    ]
                  }
                },
                {
                  "op": "field",
                  "name": "neutralizer_pairing_light",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGKV3-20"
                    ]
                  }
                }
              ]
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-kimi-k2p6-pi",
      "modelName": "kimi-k2p6",
      "harness": "pi",
      "task": "gse171524_plasma_bcr_neutralizing_clonotype",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 1191,
      "cost": 0.50433619,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dominant_pairing_heavy_gate",
            "role": "gate",
            "field": "$.dominant_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV4-34"
            ]
          },
          {
            "name": "dominant_pairing_light_gate",
            "role": "gate",
            "field": "$.dominant_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGLV3-19"
            ]
          },
          {
            "name": "dominant_neutralizer_match_gate",
            "role": "gate",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_known_neutralizer_match"
            ]
          },
          {
            "name": "neutralizer_pairing_heavy_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV1-18"
            ]
          },
          {
            "name": "neutralizer_pairing_light_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGKV3-20"
            ]
          },
          {
            "name": "neutralizer_recurrence_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "observed_in_2_to_5_donors"
            ]
          },
          {
            "name": "neutralizer_rank_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "top_5_pairing",
              "top_6_to_20_pairing"
            ]
          },
          {
            "name": "dominant_clone_isotype_gate",
            "role": "gate",
            "field": "$.dominant_clone_isotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "IgG"
            ]
          },
          {
            "name": "blineage_class_switch_contrast_gate",
            "role": "gate",
            "field": "$.blineage_class_switch_contrast",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_class_switched_control_higher_unswitched",
              "class_switched_in_both_cohorts"
            ]
          },
          {
            "name": "dominant_match_narrative_trap_or_null",
            "role": "hard_fail",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_rbd_neutralizer",
              "matches_known_autoreactive_clone",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_heavy",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_light",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_recurrence_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_rank_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "cross_axis_found_dominant_but_failed_to_scan_for_embedded_neutralizer",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_pairing_neutralizer_match": {
                "op": "in",
                "values": [
                  "no_known_neutralizer_match"
                ]
              },
              "neutralizer_pairing_heavy": {
                "op": "in",
                "values": [
                  "none_detected"
                ]
              }
            }
          },
          {
            "name": "neutralizer_pairing_not_exactly_the_s309_pair",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "op": "not",
            "expected": {
              "op": "and",
              "args": [
                {
                  "op": "field",
                  "name": "neutralizer_pairing_heavy",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGHV1-18"
                    ]
                  }
                },
                {
                  "op": "field",
                  "name": "neutralizer_pairing_light",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGKV3-20"
                    ]
                  }
                }
              ]
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-kimi-k2p6-pi",
      "modelName": "kimi-k2p6",
      "harness": "pi",
      "task": "gse171524_plasma_bcr_neutralizing_clonotype",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 1342,
      "cost": 0.5124464599999999,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dominant_pairing_heavy_gate",
            "role": "gate",
            "field": "$.dominant_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV4-34"
            ]
          },
          {
            "name": "dominant_pairing_light_gate",
            "role": "gate",
            "field": "$.dominant_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGLV3-19"
            ]
          },
          {
            "name": "dominant_neutralizer_match_gate",
            "role": "gate",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_known_neutralizer_match"
            ]
          },
          {
            "name": "neutralizer_pairing_heavy_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV1-18"
            ]
          },
          {
            "name": "neutralizer_pairing_light_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGKV3-20"
            ]
          },
          {
            "name": "neutralizer_recurrence_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "observed_in_2_to_5_donors"
            ]
          },
          {
            "name": "neutralizer_rank_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "top_5_pairing",
              "top_6_to_20_pairing"
            ]
          },
          {
            "name": "dominant_clone_isotype_gate",
            "role": "gate",
            "field": "$.dominant_clone_isotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "IgG"
            ]
          },
          {
            "name": "blineage_class_switch_contrast_gate",
            "role": "gate",
            "field": "$.blineage_class_switch_contrast",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_class_switched_control_higher_unswitched",
              "class_switched_in_both_cohorts"
            ]
          },
          {
            "name": "dominant_match_narrative_trap_or_null",
            "role": "hard_fail",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_rbd_neutralizer",
              "matches_known_autoreactive_clone",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_heavy",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_light",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_recurrence_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_rank_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "cross_axis_found_dominant_but_failed_to_scan_for_embedded_neutralizer",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_pairing_neutralizer_match": {
                "op": "in",
                "values": [
                  "no_known_neutralizer_match"
                ]
              },
              "neutralizer_pairing_heavy": {
                "op": "in",
                "values": [
                  "none_detected"
                ]
              }
            }
          },
          {
            "name": "neutralizer_pairing_not_exactly_the_s309_pair",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "op": "not",
            "expected": {
              "op": "and",
              "args": [
                {
                  "op": "field",
                  "name": "neutralizer_pairing_heavy",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGHV1-18"
                    ]
                  }
                },
                {
                  "op": "field",
                  "name": "neutralizer_pairing_light",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGKV3-20"
                    ]
                  }
                }
              ]
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-kimi-k2p6-pi",
      "modelName": "kimi-k2p6",
      "harness": "pi",
      "task": "gse171524_plasma_bcr_neutralizing_clonotype",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1367,
      "cost": 0.54035104,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dominant_pairing_heavy_gate",
            "role": "gate",
            "field": "$.dominant_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV4-34"
            ]
          },
          {
            "name": "dominant_pairing_light_gate",
            "role": "gate",
            "field": "$.dominant_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGLV3-19"
            ]
          },
          {
            "name": "dominant_neutralizer_match_gate",
            "role": "gate",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_known_neutralizer_match"
            ]
          },
          {
            "name": "neutralizer_pairing_heavy_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV1-18"
            ]
          },
          {
            "name": "neutralizer_pairing_light_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGKV3-20"
            ]
          },
          {
            "name": "neutralizer_recurrence_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "observed_in_2_to_5_donors"
            ]
          },
          {
            "name": "neutralizer_rank_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "top_5_pairing",
              "top_6_to_20_pairing"
            ]
          },
          {
            "name": "dominant_clone_isotype_gate",
            "role": "gate",
            "field": "$.dominant_clone_isotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "IgG"
            ]
          },
          {
            "name": "blineage_class_switch_contrast_gate",
            "role": "gate",
            "field": "$.blineage_class_switch_contrast",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_class_switched_control_higher_unswitched",
              "class_switched_in_both_cohorts"
            ]
          },
          {
            "name": "dominant_match_narrative_trap_or_null",
            "role": "hard_fail",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_rbd_neutralizer",
              "matches_known_autoreactive_clone",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_heavy",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_light",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_recurrence_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_rank_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "cross_axis_found_dominant_but_failed_to_scan_for_embedded_neutralizer",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_pairing_neutralizer_match": {
                "op": "in",
                "values": [
                  "no_known_neutralizer_match"
                ]
              },
              "neutralizer_pairing_heavy": {
                "op": "in",
                "values": [
                  "none_detected"
                ]
              }
            }
          },
          {
            "name": "neutralizer_pairing_not_exactly_the_s309_pair",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "op": "not",
            "expected": {
              "op": "and",
              "args": [
                {
                  "op": "field",
                  "name": "neutralizer_pairing_heavy",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGHV1-18"
                    ]
                  }
                },
                {
                  "op": "field",
                  "name": "neutralizer_pairing_light",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGKV3-20"
                    ]
                  }
                }
              ]
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-6-claude-code",
      "modelName": "claude-opus-4-6",
      "harness": "claude-code",
      "task": "gse297023_q2_senescence_not_mechanism",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 1,
      "passed": false,
      "score": 4,
      "durationS": 2435.83,
      "cost": 2.1782805,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "candidate_is_senescence",
            "role": "gate",
            "field": "$.candidate_mechanism",
            "kind": "value",
            "op": "in",
            "expected": [
              "cellular_senescence",
              "oncogene_induced_senescence",
              "replicative_senescence"
            ]
          },
          {
            "name": "arrest_core_not_engaged",
            "role": "gate",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_not_engaged"
            ]
          },
          {
            "name": "secretory_arm_is_inflammaging",
            "role": "gate",
            "field": "$.secretory_arm_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "acute_phase_inflammaging_response"
            ]
          },
          {
            "name": "secretory_arm_broadly_elevated",
            "role": "gate",
            "field": "$.secretory_arm_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "broadly_elevated"
            ]
          },
          {
            "name": "carries_aging_program",
            "role": "gate",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "carries_bona_fide_aging_program_distinct_from_nominated"
            ]
          },
          {
            "name": "conclusion_senescence_unlikely",
            "role": "gate",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "unlikely_major_contributor"
            ]
          },
          {
            "name": "cf_arrest_core_engaged_recall",
            "role": "hard_fail",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_engaged"
            ]
          },
          {
            "name": "cf_aging_program_conflation_or_explain_away",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "aging_program_indistinguishable_from_nominated_program",
              "aging_program_is_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_no_aging_program_sensitivity_failure",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_detectable_aging_program"
            ]
          },
          {
            "name": "cf_senescence_driver_conclusion",
            "role": "hard_fail",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "likely_primary_driver",
              "plausible_supported_contributor"
            ]
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: FAIL\n  + cohort_spans_young_and_aged: passed=True, score=1.0\n  + candidate_is_senescence: passed=True, score=1.0\n  + arrest_core_not_engaged: passed=True, score=1.0\n  x secretory_arm_is_inflammaging: passed=False, score=0.0\n  x secretory_arm_broadly_elevated: passed=False, score=0.0\n  x carries_aging_program: passed=False, score=0.0\n  + conclusion_senescence_unlikely: passed=True, score=1.0\n  + hard_fail 'cf_arrest_core_engaged_recall': not triggered\n  + hard_fail 'cf_aging_program_conflation_or_explain_away': not triggered\n  x hard_fail 'cf_no_aging_program_sensitivity_failure': TRIGGERED\n  + hard_fail 'cf_senescence_driver_conclusion': not triggered",
      "metrics": {
        "pass_rule": "all",
        "scoring_count": 7,
        "scoring_passed": 4,
        "scoring_total_score": 4,
        "hard_fail_triggered": [
          "cf_no_aging_program_sensitivity_failure"
        ]
      },
      "fieldScores": {
        "cohort_spans_young_and_aged": 1,
        "candidate_is_senescence": 1,
        "arrest_core_not_engaged": 1,
        "secretory_arm_is_inflammaging": 0,
        "secretory_arm_broadly_elevated": 0,
        "carries_aging_program": 0,
        "conclusion_senescence_unlikely": 1
      },
      "graderChecks": [
        {
          "name": "cohort_spans_young_and_aged",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "candidate_is_senescence",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "arrest_core_not_engaged",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "secretory_arm_is_inflammaging",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "secretory_arm_broadly_elevated",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "carries_aging_program",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "conclusion_senescence_unlikely",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "cf_arrest_core_engaged_recall",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_aging_program_conflation_or_explain_away",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_no_aging_program_sensitivity_failure",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "cf_senescence_driver_conclusion",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "age_associated_program_identity": "no_detectable_aging_program",
        "arrest_core_engaged": "arrest_core_not_engaged",
        "candidate_mechanism": "oncogene_induced_senescence",
        "cohort_host_age_design": "young_and_aged_hosts_both_profiled",
        "nominated_mechanism_role": "unlikely_major_contributor",
        "secretory_arm_direction": "scattered_reduction",
        "secretory_arm_identity": "senescence_associated_secretory_phenotype"
      },
      "nTurns": 17,
      "checkpointSignals": {
        "clustering": false,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": false,
        "nerveJointAnchor": false,
        "multiomeAssembly": false,
        "depthNormalizedAtac": false,
        "regulonInference": false,
        "cd8SubsetIntegration": false,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": false,
        "orthologProjection": false,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": true,
        "mousePseudobulkDesign": false,
        "cancerEpithelialDenominator": true,
        "geneLevelDe": false,
        "agingSignatureControl": true,
        "senescencePanelScoring": true
      },
      "trajectoryFile": "scbench-long/trajectories/gse297023_q2_senescence_not_mechanism__claude-opus-4-6__claude-code__r1.json",
      "runId": "claude-opus-4-6__claude-code__gse297023_q2_senescence_not_mechanism__r1",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-opus-4-6-claude-code",
      "modelName": "claude-opus-4-6",
      "harness": "claude-code",
      "task": "gse297023_q2_senescence_not_mechanism",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 2,
      "passed": false,
      "score": 5,
      "durationS": 1964.7,
      "cost": 1.9683112499999997,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "candidate_is_senescence",
            "role": "gate",
            "field": "$.candidate_mechanism",
            "kind": "value",
            "op": "in",
            "expected": [
              "cellular_senescence",
              "oncogene_induced_senescence",
              "replicative_senescence"
            ]
          },
          {
            "name": "arrest_core_not_engaged",
            "role": "gate",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_not_engaged"
            ]
          },
          {
            "name": "secretory_arm_is_inflammaging",
            "role": "gate",
            "field": "$.secretory_arm_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "acute_phase_inflammaging_response"
            ]
          },
          {
            "name": "secretory_arm_broadly_elevated",
            "role": "gate",
            "field": "$.secretory_arm_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "broadly_elevated"
            ]
          },
          {
            "name": "carries_aging_program",
            "role": "gate",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "carries_bona_fide_aging_program_distinct_from_nominated"
            ]
          },
          {
            "name": "conclusion_senescence_unlikely",
            "role": "gate",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "unlikely_major_contributor"
            ]
          },
          {
            "name": "cf_arrest_core_engaged_recall",
            "role": "hard_fail",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_engaged"
            ]
          },
          {
            "name": "cf_aging_program_conflation_or_explain_away",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "aging_program_indistinguishable_from_nominated_program",
              "aging_program_is_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_no_aging_program_sensitivity_failure",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_detectable_aging_program"
            ]
          },
          {
            "name": "cf_senescence_driver_conclusion",
            "role": "hard_fail",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "likely_primary_driver",
              "plausible_supported_contributor"
            ]
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: FAIL\n  + cohort_spans_young_and_aged: passed=True, score=1.0\n  + candidate_is_senescence: passed=True, score=1.0\n  + arrest_core_not_engaged: passed=True, score=1.0\n  x secretory_arm_is_inflammaging: passed=False, score=0.0\n  x secretory_arm_broadly_elevated: passed=False, score=0.0\n  + carries_aging_program: passed=True, score=1.0\n  + conclusion_senescence_unlikely: passed=True, score=1.0\n  + hard_fail 'cf_arrest_core_engaged_recall': not triggered\n  + hard_fail 'cf_aging_program_conflation_or_explain_away': not triggered\n  + hard_fail 'cf_no_aging_program_sensitivity_failure': not triggered\n  + hard_fail 'cf_senescence_driver_conclusion': not triggered",
      "metrics": {
        "pass_rule": "all",
        "scoring_count": 7,
        "scoring_passed": 5,
        "scoring_total_score": 5,
        "hard_fail_triggered": []
      },
      "fieldScores": {
        "cohort_spans_young_and_aged": 1,
        "candidate_is_senescence": 1,
        "arrest_core_not_engaged": 1,
        "secretory_arm_is_inflammaging": 0,
        "secretory_arm_broadly_elevated": 0,
        "carries_aging_program": 1,
        "conclusion_senescence_unlikely": 1
      },
      "graderChecks": [
        {
          "name": "cohort_spans_young_and_aged",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "candidate_is_senescence",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "arrest_core_not_engaged",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "secretory_arm_is_inflammaging",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "secretory_arm_broadly_elevated",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "carries_aging_program",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "conclusion_senescence_unlikely",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "cf_arrest_core_engaged_recall",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_aging_program_conflation_or_explain_away",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_no_aging_program_sensitivity_failure",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_senescence_driver_conclusion",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "age_associated_program_identity": "carries_bona_fide_aging_program_distinct_from_nominated",
        "arrest_core_engaged": "arrest_core_not_engaged",
        "candidate_mechanism": "oncogene_induced_senescence",
        "cohort_host_age_design": "young_and_aged_hosts_both_profiled",
        "nominated_mechanism_role": "unlikely_major_contributor",
        "secretory_arm_direction": "broadly_reduced",
        "secretory_arm_identity": "senescence_associated_secretory_phenotype"
      },
      "nTurns": 24,
      "checkpointSignals": {
        "clustering": false,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": false,
        "nerveJointAnchor": false,
        "multiomeAssembly": false,
        "depthNormalizedAtac": false,
        "regulonInference": false,
        "cd8SubsetIntegration": false,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": false,
        "orthologProjection": false,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": false,
        "mousePseudobulkDesign": false,
        "cancerEpithelialDenominator": true,
        "geneLevelDe": false,
        "agingSignatureControl": true,
        "senescencePanelScoring": true
      },
      "trajectoryFile": "scbench-long/trajectories/gse297023_q2_senescence_not_mechanism__claude-opus-4-6__claude-code__r2.json",
      "runId": "claude-opus-4-6__claude-code__gse297023_q2_senescence_not_mechanism__r2",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-opus-4-6-claude-code",
      "modelName": "claude-opus-4-6",
      "harness": "claude-code",
      "task": "gse297023_q2_senescence_not_mechanism",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 3,
      "passed": false,
      "score": 5,
      "durationS": 1546.47,
      "cost": 1.3290537500000001,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "candidate_is_senescence",
            "role": "gate",
            "field": "$.candidate_mechanism",
            "kind": "value",
            "op": "in",
            "expected": [
              "cellular_senescence",
              "oncogene_induced_senescence",
              "replicative_senescence"
            ]
          },
          {
            "name": "arrest_core_not_engaged",
            "role": "gate",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_not_engaged"
            ]
          },
          {
            "name": "secretory_arm_is_inflammaging",
            "role": "gate",
            "field": "$.secretory_arm_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "acute_phase_inflammaging_response"
            ]
          },
          {
            "name": "secretory_arm_broadly_elevated",
            "role": "gate",
            "field": "$.secretory_arm_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "broadly_elevated"
            ]
          },
          {
            "name": "carries_aging_program",
            "role": "gate",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "carries_bona_fide_aging_program_distinct_from_nominated"
            ]
          },
          {
            "name": "conclusion_senescence_unlikely",
            "role": "gate",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "unlikely_major_contributor"
            ]
          },
          {
            "name": "cf_arrest_core_engaged_recall",
            "role": "hard_fail",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_engaged"
            ]
          },
          {
            "name": "cf_aging_program_conflation_or_explain_away",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "aging_program_indistinguishable_from_nominated_program",
              "aging_program_is_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_no_aging_program_sensitivity_failure",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_detectable_aging_program"
            ]
          },
          {
            "name": "cf_senescence_driver_conclusion",
            "role": "hard_fail",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "likely_primary_driver",
              "plausible_supported_contributor"
            ]
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: FAIL\n  + cohort_spans_young_and_aged: passed=True, score=1.0\n  + candidate_is_senescence: passed=True, score=1.0\n  + arrest_core_not_engaged: passed=True, score=1.0\n  x secretory_arm_is_inflammaging: passed=False, score=0.0\n  x secretory_arm_broadly_elevated: passed=False, score=0.0\n  + carries_aging_program: passed=True, score=1.0\n  + conclusion_senescence_unlikely: passed=True, score=1.0\n  + hard_fail 'cf_arrest_core_engaged_recall': not triggered\n  + hard_fail 'cf_aging_program_conflation_or_explain_away': not triggered\n  + hard_fail 'cf_no_aging_program_sensitivity_failure': not triggered\n  + hard_fail 'cf_senescence_driver_conclusion': not triggered",
      "metrics": {
        "pass_rule": "all",
        "scoring_count": 7,
        "scoring_passed": 5,
        "scoring_total_score": 5,
        "hard_fail_triggered": []
      },
      "fieldScores": {
        "cohort_spans_young_and_aged": 1,
        "candidate_is_senescence": 1,
        "arrest_core_not_engaged": 1,
        "secretory_arm_is_inflammaging": 0,
        "secretory_arm_broadly_elevated": 0,
        "carries_aging_program": 1,
        "conclusion_senescence_unlikely": 1
      },
      "graderChecks": [
        {
          "name": "cohort_spans_young_and_aged",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "candidate_is_senescence",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "arrest_core_not_engaged",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "secretory_arm_is_inflammaging",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "secretory_arm_broadly_elevated",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "carries_aging_program",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "conclusion_senescence_unlikely",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "cf_arrest_core_engaged_recall",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_aging_program_conflation_or_explain_away",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_no_aging_program_sensitivity_failure",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_senescence_driver_conclusion",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "age_associated_program_identity": "carries_bona_fide_aging_program_distinct_from_nominated",
        "arrest_core_engaged": "arrest_core_not_engaged",
        "candidate_mechanism": "oncogene_induced_senescence",
        "cohort_host_age_design": "young_and_aged_hosts_both_profiled",
        "nominated_mechanism_role": "unlikely_major_contributor",
        "secretory_arm_direction": "broadly_reduced",
        "secretory_arm_identity": "senescence_associated_secretory_phenotype"
      },
      "nTurns": 20,
      "checkpointSignals": {
        "clustering": false,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": false,
        "nerveJointAnchor": false,
        "multiomeAssembly": false,
        "depthNormalizedAtac": false,
        "regulonInference": false,
        "cd8SubsetIntegration": false,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": false,
        "orthologProjection": false,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": false,
        "mousePseudobulkDesign": true,
        "cancerEpithelialDenominator": true,
        "geneLevelDe": true,
        "agingSignatureControl": true,
        "senescencePanelScoring": true
      },
      "trajectoryFile": "scbench-long/trajectories/gse297023_q2_senescence_not_mechanism__claude-opus-4-6__claude-code__r3.json",
      "runId": "claude-opus-4-6__claude-code__gse297023_q2_senescence_not_mechanism__r3",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-opus-4-6-pi",
      "modelName": "claude-opus-4-6",
      "harness": "pi",
      "task": "gse297023_q2_senescence_not_mechanism",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 1,
      "passed": false,
      "score": 4,
      "durationS": 2000.29,
      "cost": 1.6752057500000002,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "candidate_is_senescence",
            "role": "gate",
            "field": "$.candidate_mechanism",
            "kind": "value",
            "op": "in",
            "expected": [
              "cellular_senescence",
              "oncogene_induced_senescence",
              "replicative_senescence"
            ]
          },
          {
            "name": "arrest_core_not_engaged",
            "role": "gate",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_not_engaged"
            ]
          },
          {
            "name": "secretory_arm_is_inflammaging",
            "role": "gate",
            "field": "$.secretory_arm_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "acute_phase_inflammaging_response"
            ]
          },
          {
            "name": "secretory_arm_broadly_elevated",
            "role": "gate",
            "field": "$.secretory_arm_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "broadly_elevated"
            ]
          },
          {
            "name": "carries_aging_program",
            "role": "gate",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "carries_bona_fide_aging_program_distinct_from_nominated"
            ]
          },
          {
            "name": "conclusion_senescence_unlikely",
            "role": "gate",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "unlikely_major_contributor"
            ]
          },
          {
            "name": "cf_arrest_core_engaged_recall",
            "role": "hard_fail",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_engaged"
            ]
          },
          {
            "name": "cf_aging_program_conflation_or_explain_away",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "aging_program_indistinguishable_from_nominated_program",
              "aging_program_is_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_no_aging_program_sensitivity_failure",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_detectable_aging_program"
            ]
          },
          {
            "name": "cf_senescence_driver_conclusion",
            "role": "hard_fail",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "likely_primary_driver",
              "plausible_supported_contributor"
            ]
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: FAIL\n  + cohort_spans_young_and_aged: passed=True, score=1.0\n  + candidate_is_senescence: passed=True, score=1.0\n  + arrest_core_not_engaged: passed=True, score=1.0\n  x secretory_arm_is_inflammaging: passed=False, score=0.0\n  x secretory_arm_broadly_elevated: passed=False, score=0.0\n  x carries_aging_program: passed=False, score=0.0\n  + conclusion_senescence_unlikely: passed=True, score=1.0\n  + hard_fail 'cf_arrest_core_engaged_recall': not triggered\n  + hard_fail 'cf_aging_program_conflation_or_explain_away': not triggered\n  x hard_fail 'cf_no_aging_program_sensitivity_failure': TRIGGERED\n  + hard_fail 'cf_senescence_driver_conclusion': not triggered",
      "metrics": {
        "pass_rule": "all",
        "scoring_count": 7,
        "scoring_passed": 4,
        "scoring_total_score": 4,
        "hard_fail_triggered": [
          "cf_no_aging_program_sensitivity_failure"
        ]
      },
      "fieldScores": {
        "cohort_spans_young_and_aged": 1,
        "candidate_is_senescence": 1,
        "arrest_core_not_engaged": 1,
        "secretory_arm_is_inflammaging": 0,
        "secretory_arm_broadly_elevated": 0,
        "carries_aging_program": 0,
        "conclusion_senescence_unlikely": 1
      },
      "graderChecks": [
        {
          "name": "cohort_spans_young_and_aged",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "candidate_is_senescence",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "arrest_core_not_engaged",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "secretory_arm_is_inflammaging",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "secretory_arm_broadly_elevated",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "carries_aging_program",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "conclusion_senescence_unlikely",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "cf_arrest_core_engaged_recall",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_aging_program_conflation_or_explain_away",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_no_aging_program_sensitivity_failure",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "cf_senescence_driver_conclusion",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "age_associated_program_identity": "no_detectable_aging_program",
        "arrest_core_engaged": "arrest_core_not_engaged",
        "candidate_mechanism": "oncogene_induced_senescence",
        "cohort_host_age_design": "young_and_aged_hosts_both_profiled",
        "nominated_mechanism_role": "unlikely_major_contributor",
        "secretory_arm_direction": "broadly_reduced",
        "secretory_arm_identity": "senescence_associated_secretory_phenotype"
      },
      "nTurns": 24,
      "checkpointSignals": {
        "clustering": false,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": false,
        "nerveJointAnchor": false,
        "multiomeAssembly": false,
        "depthNormalizedAtac": true,
        "regulonInference": false,
        "cd8SubsetIntegration": false,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": false,
        "orthologProjection": false,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": true,
        "mousePseudobulkDesign": true,
        "cancerEpithelialDenominator": true,
        "geneLevelDe": true,
        "agingSignatureControl": true,
        "senescencePanelScoring": true
      },
      "trajectoryFile": "scbench-long/trajectories/gse297023_q2_senescence_not_mechanism__claude-opus-4-6__pi__r1.json",
      "runId": "claude-opus-4-6__pi__gse297023_q2_senescence_not_mechanism__r1",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-opus-4-6-pi",
      "modelName": "claude-opus-4-6",
      "harness": "pi",
      "task": "gse297023_q2_senescence_not_mechanism",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 2,
      "passed": false,
      "score": 4,
      "durationS": 1700.23,
      "cost": 1.7542092500000004,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "candidate_is_senescence",
            "role": "gate",
            "field": "$.candidate_mechanism",
            "kind": "value",
            "op": "in",
            "expected": [
              "cellular_senescence",
              "oncogene_induced_senescence",
              "replicative_senescence"
            ]
          },
          {
            "name": "arrest_core_not_engaged",
            "role": "gate",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_not_engaged"
            ]
          },
          {
            "name": "secretory_arm_is_inflammaging",
            "role": "gate",
            "field": "$.secretory_arm_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "acute_phase_inflammaging_response"
            ]
          },
          {
            "name": "secretory_arm_broadly_elevated",
            "role": "gate",
            "field": "$.secretory_arm_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "broadly_elevated"
            ]
          },
          {
            "name": "carries_aging_program",
            "role": "gate",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "carries_bona_fide_aging_program_distinct_from_nominated"
            ]
          },
          {
            "name": "conclusion_senescence_unlikely",
            "role": "gate",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "unlikely_major_contributor"
            ]
          },
          {
            "name": "cf_arrest_core_engaged_recall",
            "role": "hard_fail",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_engaged"
            ]
          },
          {
            "name": "cf_aging_program_conflation_or_explain_away",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "aging_program_indistinguishable_from_nominated_program",
              "aging_program_is_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_no_aging_program_sensitivity_failure",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_detectable_aging_program"
            ]
          },
          {
            "name": "cf_senescence_driver_conclusion",
            "role": "hard_fail",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "likely_primary_driver",
              "plausible_supported_contributor"
            ]
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: FAIL\n  + cohort_spans_young_and_aged: passed=True, score=1.0\n  + candidate_is_senescence: passed=True, score=1.0\n  + arrest_core_not_engaged: passed=True, score=1.0\n  x secretory_arm_is_inflammaging: passed=False, score=0.0\n  x secretory_arm_broadly_elevated: passed=False, score=0.0\n  x carries_aging_program: passed=False, score=0.0\n  + conclusion_senescence_unlikely: passed=True, score=1.0\n  + hard_fail 'cf_arrest_core_engaged_recall': not triggered\n  x hard_fail 'cf_aging_program_conflation_or_explain_away': TRIGGERED\n  + hard_fail 'cf_no_aging_program_sensitivity_failure': not triggered\n  + hard_fail 'cf_senescence_driver_conclusion': not triggered",
      "metrics": {
        "pass_rule": "all",
        "scoring_count": 7,
        "scoring_passed": 4,
        "scoring_total_score": 4,
        "hard_fail_triggered": [
          "cf_aging_program_conflation_or_explain_away"
        ]
      },
      "fieldScores": {
        "cohort_spans_young_and_aged": 1,
        "candidate_is_senescence": 1,
        "arrest_core_not_engaged": 1,
        "secretory_arm_is_inflammaging": 0,
        "secretory_arm_broadly_elevated": 0,
        "carries_aging_program": 0,
        "conclusion_senescence_unlikely": 1
      },
      "graderChecks": [
        {
          "name": "cohort_spans_young_and_aged",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "candidate_is_senescence",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "arrest_core_not_engaged",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "secretory_arm_is_inflammaging",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "secretory_arm_broadly_elevated",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "carries_aging_program",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "conclusion_senescence_unlikely",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "cf_arrest_core_engaged_recall",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_aging_program_conflation_or_explain_away",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "cf_no_aging_program_sensitivity_failure",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_senescence_driver_conclusion",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "age_associated_program_identity": "aging_program_indistinguishable_from_nominated_program",
        "arrest_core_engaged": "arrest_core_not_engaged",
        "candidate_mechanism": "oncogene_induced_senescence",
        "cohort_host_age_design": "young_and_aged_hosts_both_profiled",
        "nominated_mechanism_role": "unlikely_major_contributor",
        "secretory_arm_direction": "broadly_reduced",
        "secretory_arm_identity": "senescence_associated_secretory_phenotype"
      },
      "nTurns": 21,
      "checkpointSignals": {
        "clustering": true,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": false,
        "nerveJointAnchor": false,
        "multiomeAssembly": false,
        "depthNormalizedAtac": true,
        "regulonInference": false,
        "cd8SubsetIntegration": false,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": false,
        "orthologProjection": false,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": true,
        "mousePseudobulkDesign": true,
        "cancerEpithelialDenominator": true,
        "geneLevelDe": true,
        "agingSignatureControl": true,
        "senescencePanelScoring": true
      },
      "trajectoryFile": "scbench-long/trajectories/gse297023_q2_senescence_not_mechanism__claude-opus-4-6__pi__r2.json",
      "runId": "claude-opus-4-6__pi__gse297023_q2_senescence_not_mechanism__r2",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-opus-4-6-pi",
      "modelName": "claude-opus-4-6",
      "harness": "pi",
      "task": "gse297023_q2_senescence_not_mechanism",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 3,
      "passed": false,
      "score": 5,
      "durationS": 2119.52,
      "cost": 2.5573122500000003,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "candidate_is_senescence",
            "role": "gate",
            "field": "$.candidate_mechanism",
            "kind": "value",
            "op": "in",
            "expected": [
              "cellular_senescence",
              "oncogene_induced_senescence",
              "replicative_senescence"
            ]
          },
          {
            "name": "arrest_core_not_engaged",
            "role": "gate",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_not_engaged"
            ]
          },
          {
            "name": "secretory_arm_is_inflammaging",
            "role": "gate",
            "field": "$.secretory_arm_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "acute_phase_inflammaging_response"
            ]
          },
          {
            "name": "secretory_arm_broadly_elevated",
            "role": "gate",
            "field": "$.secretory_arm_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "broadly_elevated"
            ]
          },
          {
            "name": "carries_aging_program",
            "role": "gate",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "carries_bona_fide_aging_program_distinct_from_nominated"
            ]
          },
          {
            "name": "conclusion_senescence_unlikely",
            "role": "gate",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "unlikely_major_contributor"
            ]
          },
          {
            "name": "cf_arrest_core_engaged_recall",
            "role": "hard_fail",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_engaged"
            ]
          },
          {
            "name": "cf_aging_program_conflation_or_explain_away",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "aging_program_indistinguishable_from_nominated_program",
              "aging_program_is_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_no_aging_program_sensitivity_failure",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_detectable_aging_program"
            ]
          },
          {
            "name": "cf_senescence_driver_conclusion",
            "role": "hard_fail",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "likely_primary_driver",
              "plausible_supported_contributor"
            ]
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: FAIL\n  + cohort_spans_young_and_aged: passed=True, score=1.0\n  + candidate_is_senescence: passed=True, score=1.0\n  + arrest_core_not_engaged: passed=True, score=1.0\n  x secretory_arm_is_inflammaging: passed=False, score=0.0\n  x secretory_arm_broadly_elevated: passed=False, score=0.0\n  + carries_aging_program: passed=True, score=1.0\n  + conclusion_senescence_unlikely: passed=True, score=1.0\n  + hard_fail 'cf_arrest_core_engaged_recall': not triggered\n  + hard_fail 'cf_aging_program_conflation_or_explain_away': not triggered\n  + hard_fail 'cf_no_aging_program_sensitivity_failure': not triggered\n  + hard_fail 'cf_senescence_driver_conclusion': not triggered",
      "metrics": {
        "pass_rule": "all",
        "scoring_count": 7,
        "scoring_passed": 5,
        "scoring_total_score": 5,
        "hard_fail_triggered": []
      },
      "fieldScores": {
        "cohort_spans_young_and_aged": 1,
        "candidate_is_senescence": 1,
        "arrest_core_not_engaged": 1,
        "secretory_arm_is_inflammaging": 0,
        "secretory_arm_broadly_elevated": 0,
        "carries_aging_program": 1,
        "conclusion_senescence_unlikely": 1
      },
      "graderChecks": [
        {
          "name": "cohort_spans_young_and_aged",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "candidate_is_senescence",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "arrest_core_not_engaged",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "secretory_arm_is_inflammaging",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "secretory_arm_broadly_elevated",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "carries_aging_program",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "conclusion_senescence_unlikely",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "cf_arrest_core_engaged_recall",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_aging_program_conflation_or_explain_away",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_no_aging_program_sensitivity_failure",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_senescence_driver_conclusion",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "age_associated_program_identity": "carries_bona_fide_aging_program_distinct_from_nominated",
        "arrest_core_engaged": "arrest_core_not_engaged",
        "candidate_mechanism": "oncogene_induced_senescence",
        "cohort_host_age_design": "young_and_aged_hosts_both_profiled",
        "nominated_mechanism_role": "unlikely_major_contributor",
        "secretory_arm_direction": "scattered_reduction",
        "secretory_arm_identity": "senescence_associated_secretory_phenotype"
      },
      "nTurns": 26,
      "checkpointSignals": {
        "clustering": false,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": false,
        "nerveJointAnchor": false,
        "multiomeAssembly": false,
        "depthNormalizedAtac": true,
        "regulonInference": false,
        "cd8SubsetIntegration": false,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": false,
        "orthologProjection": false,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": false,
        "mousePseudobulkDesign": true,
        "cancerEpithelialDenominator": true,
        "geneLevelDe": true,
        "agingSignatureControl": true,
        "senescencePanelScoring": true
      },
      "trajectoryFile": "scbench-long/trajectories/gse297023_q2_senescence_not_mechanism__claude-opus-4-6__pi__r3.json",
      "runId": "claude-opus-4-6__pi__gse297023_q2_senescence_not_mechanism__r3",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-opus-4-7-claude-code",
      "modelName": "claude-opus-4-7",
      "harness": "claude-code",
      "task": "gse297023_q2_senescence_not_mechanism",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 1,
      "passed": true,
      "score": 7,
      "durationS": 457.13,
      "cost": 1.5156004999999997,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "candidate_is_senescence",
            "role": "gate",
            "field": "$.candidate_mechanism",
            "kind": "value",
            "op": "in",
            "expected": [
              "cellular_senescence",
              "oncogene_induced_senescence",
              "replicative_senescence"
            ]
          },
          {
            "name": "arrest_core_not_engaged",
            "role": "gate",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_not_engaged"
            ]
          },
          {
            "name": "secretory_arm_is_inflammaging",
            "role": "gate",
            "field": "$.secretory_arm_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "acute_phase_inflammaging_response"
            ]
          },
          {
            "name": "secretory_arm_broadly_elevated",
            "role": "gate",
            "field": "$.secretory_arm_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "broadly_elevated"
            ]
          },
          {
            "name": "carries_aging_program",
            "role": "gate",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "carries_bona_fide_aging_program_distinct_from_nominated"
            ]
          },
          {
            "name": "conclusion_senescence_unlikely",
            "role": "gate",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "unlikely_major_contributor"
            ]
          },
          {
            "name": "cf_arrest_core_engaged_recall",
            "role": "hard_fail",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_engaged"
            ]
          },
          {
            "name": "cf_aging_program_conflation_or_explain_away",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "aging_program_indistinguishable_from_nominated_program",
              "aging_program_is_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_no_aging_program_sensitivity_failure",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_detectable_aging_program"
            ]
          },
          {
            "name": "cf_senescence_driver_conclusion",
            "role": "hard_fail",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "likely_primary_driver",
              "plausible_supported_contributor"
            ]
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: PASS\n  + cohort_spans_young_and_aged: passed=True, score=1.0\n  + candidate_is_senescence: passed=True, score=1.0\n  + arrest_core_not_engaged: passed=True, score=1.0\n  + secretory_arm_is_inflammaging: passed=True, score=1.0\n  + secretory_arm_broadly_elevated: passed=True, score=1.0\n  + carries_aging_program: passed=True, score=1.0\n  + conclusion_senescence_unlikely: passed=True, score=1.0\n  + hard_fail 'cf_arrest_core_engaged_recall': not triggered\n  + hard_fail 'cf_aging_program_conflation_or_explain_away': not triggered\n  + hard_fail 'cf_no_aging_program_sensitivity_failure': not triggered\n  + hard_fail 'cf_senescence_driver_conclusion': not triggered",
      "metrics": {
        "pass_rule": "all",
        "scoring_count": 7,
        "scoring_passed": 7,
        "scoring_total_score": 7,
        "hard_fail_triggered": []
      },
      "fieldScores": {
        "cohort_spans_young_and_aged": 1,
        "candidate_is_senescence": 1,
        "arrest_core_not_engaged": 1,
        "secretory_arm_is_inflammaging": 1,
        "secretory_arm_broadly_elevated": 1,
        "carries_aging_program": 1,
        "conclusion_senescence_unlikely": 1
      },
      "graderChecks": [
        {
          "name": "cohort_spans_young_and_aged",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "candidate_is_senescence",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "arrest_core_not_engaged",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "secretory_arm_is_inflammaging",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "secretory_arm_broadly_elevated",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "carries_aging_program",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "conclusion_senescence_unlikely",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "cf_arrest_core_engaged_recall",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_aging_program_conflation_or_explain_away",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_no_aging_program_sensitivity_failure",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_senescence_driver_conclusion",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "age_associated_program_identity": "carries_bona_fide_aging_program_distinct_from_nominated",
        "arrest_core_engaged": "arrest_core_not_engaged",
        "candidate_mechanism": "cellular_senescence",
        "cohort_host_age_design": "young_and_aged_hosts_both_profiled",
        "nominated_mechanism_role": "unlikely_major_contributor",
        "secretory_arm_direction": "broadly_elevated",
        "secretory_arm_identity": "acute_phase_inflammaging_response"
      },
      "nTurns": 21,
      "checkpointSignals": {
        "clustering": false,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": false,
        "nerveJointAnchor": false,
        "multiomeAssembly": false,
        "depthNormalizedAtac": false,
        "regulonInference": false,
        "cd8SubsetIntegration": false,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": false,
        "orthologProjection": false,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": false,
        "mousePseudobulkDesign": true,
        "cancerEpithelialDenominator": true,
        "geneLevelDe": true,
        "agingSignatureControl": true,
        "senescencePanelScoring": true
      },
      "trajectoryFile": "scbench-long/trajectories/gse297023_q2_senescence_not_mechanism__claude-opus-4-7__claude-code__r1.json",
      "runId": "claude-opus-4-7__claude-code__gse297023_q2_senescence_not_mechanism__r1",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-opus-4-7-claude-code",
      "modelName": "claude-opus-4-7",
      "harness": "claude-code",
      "task": "gse297023_q2_senescence_not_mechanism",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 2,
      "passed": true,
      "score": 7,
      "durationS": 471.1,
      "cost": 2.07478475,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "candidate_is_senescence",
            "role": "gate",
            "field": "$.candidate_mechanism",
            "kind": "value",
            "op": "in",
            "expected": [
              "cellular_senescence",
              "oncogene_induced_senescence",
              "replicative_senescence"
            ]
          },
          {
            "name": "arrest_core_not_engaged",
            "role": "gate",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_not_engaged"
            ]
          },
          {
            "name": "secretory_arm_is_inflammaging",
            "role": "gate",
            "field": "$.secretory_arm_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "acute_phase_inflammaging_response"
            ]
          },
          {
            "name": "secretory_arm_broadly_elevated",
            "role": "gate",
            "field": "$.secretory_arm_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "broadly_elevated"
            ]
          },
          {
            "name": "carries_aging_program",
            "role": "gate",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "carries_bona_fide_aging_program_distinct_from_nominated"
            ]
          },
          {
            "name": "conclusion_senescence_unlikely",
            "role": "gate",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "unlikely_major_contributor"
            ]
          },
          {
            "name": "cf_arrest_core_engaged_recall",
            "role": "hard_fail",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_engaged"
            ]
          },
          {
            "name": "cf_aging_program_conflation_or_explain_away",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "aging_program_indistinguishable_from_nominated_program",
              "aging_program_is_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_no_aging_program_sensitivity_failure",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_detectable_aging_program"
            ]
          },
          {
            "name": "cf_senescence_driver_conclusion",
            "role": "hard_fail",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "likely_primary_driver",
              "plausible_supported_contributor"
            ]
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: PASS\n  + cohort_spans_young_and_aged: passed=True, score=1.0\n  + candidate_is_senescence: passed=True, score=1.0\n  + arrest_core_not_engaged: passed=True, score=1.0\n  + secretory_arm_is_inflammaging: passed=True, score=1.0\n  + secretory_arm_broadly_elevated: passed=True, score=1.0\n  + carries_aging_program: passed=True, score=1.0\n  + conclusion_senescence_unlikely: passed=True, score=1.0\n  + hard_fail 'cf_arrest_core_engaged_recall': not triggered\n  + hard_fail 'cf_aging_program_conflation_or_explain_away': not triggered\n  + hard_fail 'cf_no_aging_program_sensitivity_failure': not triggered\n  + hard_fail 'cf_senescence_driver_conclusion': not triggered",
      "metrics": {
        "pass_rule": "all",
        "scoring_count": 7,
        "scoring_passed": 7,
        "scoring_total_score": 7,
        "hard_fail_triggered": []
      },
      "fieldScores": {
        "cohort_spans_young_and_aged": 1,
        "candidate_is_senescence": 1,
        "arrest_core_not_engaged": 1,
        "secretory_arm_is_inflammaging": 1,
        "secretory_arm_broadly_elevated": 1,
        "carries_aging_program": 1,
        "conclusion_senescence_unlikely": 1
      },
      "graderChecks": [
        {
          "name": "cohort_spans_young_and_aged",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "candidate_is_senescence",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "arrest_core_not_engaged",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "secretory_arm_is_inflammaging",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "secretory_arm_broadly_elevated",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "carries_aging_program",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "conclusion_senescence_unlikely",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "cf_arrest_core_engaged_recall",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_aging_program_conflation_or_explain_away",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_no_aging_program_sensitivity_failure",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_senescence_driver_conclusion",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "age_associated_program_identity": "carries_bona_fide_aging_program_distinct_from_nominated",
        "arrest_core_engaged": "arrest_core_not_engaged",
        "candidate_mechanism": "cellular_senescence",
        "cohort_host_age_design": "young_and_aged_hosts_both_profiled",
        "nominated_mechanism_role": "unlikely_major_contributor",
        "secretory_arm_direction": "broadly_elevated",
        "secretory_arm_identity": "acute_phase_inflammaging_response"
      },
      "nTurns": 33,
      "checkpointSignals": {
        "clustering": false,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": false,
        "nerveJointAnchor": false,
        "multiomeAssembly": false,
        "depthNormalizedAtac": false,
        "regulonInference": false,
        "cd8SubsetIntegration": false,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": false,
        "orthologProjection": false,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": false,
        "mousePseudobulkDesign": true,
        "cancerEpithelialDenominator": true,
        "geneLevelDe": true,
        "agingSignatureControl": true,
        "senescencePanelScoring": true
      },
      "trajectoryFile": "scbench-long/trajectories/gse297023_q2_senescence_not_mechanism__claude-opus-4-7__claude-code__r2.json",
      "runId": "claude-opus-4-7__claude-code__gse297023_q2_senescence_not_mechanism__r2",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-opus-4-7-claude-code",
      "modelName": "claude-opus-4-7",
      "harness": "claude-code",
      "task": "gse297023_q2_senescence_not_mechanism",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 3,
      "passed": true,
      "score": 7,
      "durationS": 467.15,
      "cost": 1.3933119999999999,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "candidate_is_senescence",
            "role": "gate",
            "field": "$.candidate_mechanism",
            "kind": "value",
            "op": "in",
            "expected": [
              "cellular_senescence",
              "oncogene_induced_senescence",
              "replicative_senescence"
            ]
          },
          {
            "name": "arrest_core_not_engaged",
            "role": "gate",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_not_engaged"
            ]
          },
          {
            "name": "secretory_arm_is_inflammaging",
            "role": "gate",
            "field": "$.secretory_arm_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "acute_phase_inflammaging_response"
            ]
          },
          {
            "name": "secretory_arm_broadly_elevated",
            "role": "gate",
            "field": "$.secretory_arm_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "broadly_elevated"
            ]
          },
          {
            "name": "carries_aging_program",
            "role": "gate",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "carries_bona_fide_aging_program_distinct_from_nominated"
            ]
          },
          {
            "name": "conclusion_senescence_unlikely",
            "role": "gate",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "unlikely_major_contributor"
            ]
          },
          {
            "name": "cf_arrest_core_engaged_recall",
            "role": "hard_fail",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_engaged"
            ]
          },
          {
            "name": "cf_aging_program_conflation_or_explain_away",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "aging_program_indistinguishable_from_nominated_program",
              "aging_program_is_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_no_aging_program_sensitivity_failure",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_detectable_aging_program"
            ]
          },
          {
            "name": "cf_senescence_driver_conclusion",
            "role": "hard_fail",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "likely_primary_driver",
              "plausible_supported_contributor"
            ]
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: PASS\n  + cohort_spans_young_and_aged: passed=True, score=1.0\n  + candidate_is_senescence: passed=True, score=1.0\n  + arrest_core_not_engaged: passed=True, score=1.0\n  + secretory_arm_is_inflammaging: passed=True, score=1.0\n  + secretory_arm_broadly_elevated: passed=True, score=1.0\n  + carries_aging_program: passed=True, score=1.0\n  + conclusion_senescence_unlikely: passed=True, score=1.0\n  + hard_fail 'cf_arrest_core_engaged_recall': not triggered\n  + hard_fail 'cf_aging_program_conflation_or_explain_away': not triggered\n  + hard_fail 'cf_no_aging_program_sensitivity_failure': not triggered\n  + hard_fail 'cf_senescence_driver_conclusion': not triggered",
      "metrics": {
        "pass_rule": "all",
        "scoring_count": 7,
        "scoring_passed": 7,
        "scoring_total_score": 7,
        "hard_fail_triggered": []
      },
      "fieldScores": {
        "cohort_spans_young_and_aged": 1,
        "candidate_is_senescence": 1,
        "arrest_core_not_engaged": 1,
        "secretory_arm_is_inflammaging": 1,
        "secretory_arm_broadly_elevated": 1,
        "carries_aging_program": 1,
        "conclusion_senescence_unlikely": 1
      },
      "graderChecks": [
        {
          "name": "cohort_spans_young_and_aged",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "candidate_is_senescence",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "arrest_core_not_engaged",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "secretory_arm_is_inflammaging",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "secretory_arm_broadly_elevated",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "carries_aging_program",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "conclusion_senescence_unlikely",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "cf_arrest_core_engaged_recall",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_aging_program_conflation_or_explain_away",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_no_aging_program_sensitivity_failure",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_senescence_driver_conclusion",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "age_associated_program_identity": "carries_bona_fide_aging_program_distinct_from_nominated",
        "arrest_core_engaged": "arrest_core_not_engaged",
        "candidate_mechanism": "cellular_senescence",
        "cohort_host_age_design": "young_and_aged_hosts_both_profiled",
        "nominated_mechanism_role": "unlikely_major_contributor",
        "secretory_arm_direction": "broadly_elevated",
        "secretory_arm_identity": "acute_phase_inflammaging_response"
      },
      "nTurns": 20,
      "checkpointSignals": {
        "clustering": false,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": false,
        "nerveJointAnchor": false,
        "multiomeAssembly": false,
        "depthNormalizedAtac": false,
        "regulonInference": false,
        "cd8SubsetIntegration": false,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": false,
        "orthologProjection": false,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": false,
        "mousePseudobulkDesign": true,
        "cancerEpithelialDenominator": true,
        "geneLevelDe": true,
        "agingSignatureControl": true,
        "senescencePanelScoring": true
      },
      "trajectoryFile": "scbench-long/trajectories/gse297023_q2_senescence_not_mechanism__claude-opus-4-7__claude-code__r3.json",
      "runId": "claude-opus-4-7__claude-code__gse297023_q2_senescence_not_mechanism__r3",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-opus-4-7-pi",
      "modelName": "claude-opus-4-7",
      "harness": "pi",
      "task": "gse297023_q2_senescence_not_mechanism",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 1,
      "passed": false,
      "score": 5,
      "durationS": 612.79,
      "cost": 0.65598975,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "candidate_is_senescence",
            "role": "gate",
            "field": "$.candidate_mechanism",
            "kind": "value",
            "op": "in",
            "expected": [
              "cellular_senescence",
              "oncogene_induced_senescence",
              "replicative_senescence"
            ]
          },
          {
            "name": "arrest_core_not_engaged",
            "role": "gate",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_not_engaged"
            ]
          },
          {
            "name": "secretory_arm_is_inflammaging",
            "role": "gate",
            "field": "$.secretory_arm_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "acute_phase_inflammaging_response"
            ]
          },
          {
            "name": "secretory_arm_broadly_elevated",
            "role": "gate",
            "field": "$.secretory_arm_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "broadly_elevated"
            ]
          },
          {
            "name": "carries_aging_program",
            "role": "gate",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "carries_bona_fide_aging_program_distinct_from_nominated"
            ]
          },
          {
            "name": "conclusion_senescence_unlikely",
            "role": "gate",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "unlikely_major_contributor"
            ]
          },
          {
            "name": "cf_arrest_core_engaged_recall",
            "role": "hard_fail",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_engaged"
            ]
          },
          {
            "name": "cf_aging_program_conflation_or_explain_away",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "aging_program_indistinguishable_from_nominated_program",
              "aging_program_is_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_no_aging_program_sensitivity_failure",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_detectable_aging_program"
            ]
          },
          {
            "name": "cf_senescence_driver_conclusion",
            "role": "hard_fail",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "likely_primary_driver",
              "plausible_supported_contributor"
            ]
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: FAIL\n  + cohort_spans_young_and_aged: passed=True, score=1.0\n  + candidate_is_senescence: passed=True, score=1.0\n  + arrest_core_not_engaged: passed=True, score=1.0\n  x secretory_arm_is_inflammaging: passed=False, score=0.0\n  x secretory_arm_broadly_elevated: passed=False, score=0.0\n  + carries_aging_program: passed=True, score=1.0\n  + conclusion_senescence_unlikely: passed=True, score=1.0\n  + hard_fail 'cf_arrest_core_engaged_recall': not triggered\n  + hard_fail 'cf_aging_program_conflation_or_explain_away': not triggered\n  + hard_fail 'cf_no_aging_program_sensitivity_failure': not triggered\n  + hard_fail 'cf_senescence_driver_conclusion': not triggered",
      "metrics": {
        "pass_rule": "all",
        "scoring_count": 7,
        "scoring_passed": 5,
        "scoring_total_score": 5,
        "hard_fail_triggered": []
      },
      "fieldScores": {
        "cohort_spans_young_and_aged": 1,
        "candidate_is_senescence": 1,
        "arrest_core_not_engaged": 1,
        "secretory_arm_is_inflammaging": 0,
        "secretory_arm_broadly_elevated": 0,
        "carries_aging_program": 1,
        "conclusion_senescence_unlikely": 1
      },
      "graderChecks": [
        {
          "name": "cohort_spans_young_and_aged",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "candidate_is_senescence",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "arrest_core_not_engaged",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "secretory_arm_is_inflammaging",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "secretory_arm_broadly_elevated",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "carries_aging_program",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "conclusion_senescence_unlikely",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "cf_arrest_core_engaged_recall",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_aging_program_conflation_or_explain_away",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_no_aging_program_sensitivity_failure",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_senescence_driver_conclusion",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "age_associated_program_identity": "carries_bona_fide_aging_program_distinct_from_nominated",
        "arrest_core_engaged": "arrest_core_not_engaged",
        "candidate_mechanism": "cellular_senescence",
        "cohort_host_age_design": "young_and_aged_hosts_both_profiled",
        "nominated_mechanism_role": "unlikely_major_contributor",
        "secretory_arm_direction": "broadly_reduced",
        "secretory_arm_identity": "senescence_associated_secretory_phenotype"
      },
      "nTurns": 14,
      "checkpointSignals": {
        "clustering": false,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": false,
        "nerveJointAnchor": false,
        "multiomeAssembly": false,
        "depthNormalizedAtac": false,
        "regulonInference": false,
        "cd8SubsetIntegration": false,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": false,
        "orthologProjection": false,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": false,
        "mousePseudobulkDesign": true,
        "cancerEpithelialDenominator": true,
        "geneLevelDe": true,
        "agingSignatureControl": true,
        "senescencePanelScoring": true
      },
      "trajectoryFile": "scbench-long/trajectories/gse297023_q2_senescence_not_mechanism__claude-opus-4-7__pi__r1.json",
      "runId": "claude-opus-4-7__pi__gse297023_q2_senescence_not_mechanism__r1",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-opus-4-7-pi",
      "modelName": "claude-opus-4-7",
      "harness": "pi",
      "task": "gse297023_q2_senescence_not_mechanism",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 2,
      "passed": false,
      "score": 5,
      "durationS": 311.14,
      "cost": 0.5625700000000001,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "candidate_is_senescence",
            "role": "gate",
            "field": "$.candidate_mechanism",
            "kind": "value",
            "op": "in",
            "expected": [
              "cellular_senescence",
              "oncogene_induced_senescence",
              "replicative_senescence"
            ]
          },
          {
            "name": "arrest_core_not_engaged",
            "role": "gate",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_not_engaged"
            ]
          },
          {
            "name": "secretory_arm_is_inflammaging",
            "role": "gate",
            "field": "$.secretory_arm_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "acute_phase_inflammaging_response"
            ]
          },
          {
            "name": "secretory_arm_broadly_elevated",
            "role": "gate",
            "field": "$.secretory_arm_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "broadly_elevated"
            ]
          },
          {
            "name": "carries_aging_program",
            "role": "gate",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "carries_bona_fide_aging_program_distinct_from_nominated"
            ]
          },
          {
            "name": "conclusion_senescence_unlikely",
            "role": "gate",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "unlikely_major_contributor"
            ]
          },
          {
            "name": "cf_arrest_core_engaged_recall",
            "role": "hard_fail",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_engaged"
            ]
          },
          {
            "name": "cf_aging_program_conflation_or_explain_away",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "aging_program_indistinguishable_from_nominated_program",
              "aging_program_is_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_no_aging_program_sensitivity_failure",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_detectable_aging_program"
            ]
          },
          {
            "name": "cf_senescence_driver_conclusion",
            "role": "hard_fail",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "likely_primary_driver",
              "plausible_supported_contributor"
            ]
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: FAIL\n  + cohort_spans_young_and_aged: passed=True, score=1.0\n  + candidate_is_senescence: passed=True, score=1.0\n  + arrest_core_not_engaged: passed=True, score=1.0\n  x secretory_arm_is_inflammaging: passed=False, score=0.0\n  x secretory_arm_broadly_elevated: passed=False, score=0.0\n  + carries_aging_program: passed=True, score=1.0\n  + conclusion_senescence_unlikely: passed=True, score=1.0\n  + hard_fail 'cf_arrest_core_engaged_recall': not triggered\n  + hard_fail 'cf_aging_program_conflation_or_explain_away': not triggered\n  + hard_fail 'cf_no_aging_program_sensitivity_failure': not triggered\n  + hard_fail 'cf_senescence_driver_conclusion': not triggered",
      "metrics": {
        "pass_rule": "all",
        "scoring_count": 7,
        "scoring_passed": 5,
        "scoring_total_score": 5,
        "hard_fail_triggered": []
      },
      "fieldScores": {
        "cohort_spans_young_and_aged": 1,
        "candidate_is_senescence": 1,
        "arrest_core_not_engaged": 1,
        "secretory_arm_is_inflammaging": 0,
        "secretory_arm_broadly_elevated": 0,
        "carries_aging_program": 1,
        "conclusion_senescence_unlikely": 1
      },
      "graderChecks": [
        {
          "name": "cohort_spans_young_and_aged",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "candidate_is_senescence",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "arrest_core_not_engaged",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "secretory_arm_is_inflammaging",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "secretory_arm_broadly_elevated",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "carries_aging_program",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "conclusion_senescence_unlikely",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "cf_arrest_core_engaged_recall",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_aging_program_conflation_or_explain_away",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_no_aging_program_sensitivity_failure",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_senescence_driver_conclusion",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "age_associated_program_identity": "carries_bona_fide_aging_program_distinct_from_nominated",
        "arrest_core_engaged": "arrest_core_not_engaged",
        "candidate_mechanism": "cellular_senescence",
        "cohort_host_age_design": "young_and_aged_hosts_both_profiled",
        "nominated_mechanism_role": "unlikely_major_contributor",
        "secretory_arm_direction": "scattered_elevation",
        "secretory_arm_identity": "senescence_associated_secretory_phenotype"
      },
      "nTurns": 12,
      "checkpointSignals": {
        "clustering": false,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": false,
        "nerveJointAnchor": false,
        "multiomeAssembly": false,
        "depthNormalizedAtac": false,
        "regulonInference": false,
        "cd8SubsetIntegration": false,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": false,
        "orthologProjection": false,
        "epiSubsetSelection": true,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": false,
        "mousePseudobulkDesign": true,
        "cancerEpithelialDenominator": true,
        "geneLevelDe": true,
        "agingSignatureControl": true,
        "senescencePanelScoring": true
      },
      "trajectoryFile": "scbench-long/trajectories/gse297023_q2_senescence_not_mechanism__claude-opus-4-7__pi__r2.json",
      "runId": "claude-opus-4-7__pi__gse297023_q2_senescence_not_mechanism__r2",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-opus-4-7-pi",
      "modelName": "claude-opus-4-7",
      "harness": "pi",
      "task": "gse297023_q2_senescence_not_mechanism",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 3,
      "passed": false,
      "score": 5,
      "durationS": 216.1,
      "cost": 0.44549675,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "candidate_is_senescence",
            "role": "gate",
            "field": "$.candidate_mechanism",
            "kind": "value",
            "op": "in",
            "expected": [
              "cellular_senescence",
              "oncogene_induced_senescence",
              "replicative_senescence"
            ]
          },
          {
            "name": "arrest_core_not_engaged",
            "role": "gate",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_not_engaged"
            ]
          },
          {
            "name": "secretory_arm_is_inflammaging",
            "role": "gate",
            "field": "$.secretory_arm_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "acute_phase_inflammaging_response"
            ]
          },
          {
            "name": "secretory_arm_broadly_elevated",
            "role": "gate",
            "field": "$.secretory_arm_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "broadly_elevated"
            ]
          },
          {
            "name": "carries_aging_program",
            "role": "gate",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "carries_bona_fide_aging_program_distinct_from_nominated"
            ]
          },
          {
            "name": "conclusion_senescence_unlikely",
            "role": "gate",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "unlikely_major_contributor"
            ]
          },
          {
            "name": "cf_arrest_core_engaged_recall",
            "role": "hard_fail",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_engaged"
            ]
          },
          {
            "name": "cf_aging_program_conflation_or_explain_away",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "aging_program_indistinguishable_from_nominated_program",
              "aging_program_is_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_no_aging_program_sensitivity_failure",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_detectable_aging_program"
            ]
          },
          {
            "name": "cf_senescence_driver_conclusion",
            "role": "hard_fail",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "likely_primary_driver",
              "plausible_supported_contributor"
            ]
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: FAIL\n  + cohort_spans_young_and_aged: passed=True, score=1.0\n  + candidate_is_senescence: passed=True, score=1.0\n  + arrest_core_not_engaged: passed=True, score=1.0\n  x secretory_arm_is_inflammaging: passed=False, score=0.0\n  x secretory_arm_broadly_elevated: passed=False, score=0.0\n  + carries_aging_program: passed=True, score=1.0\n  + conclusion_senescence_unlikely: passed=True, score=1.0\n  + hard_fail 'cf_arrest_core_engaged_recall': not triggered\n  + hard_fail 'cf_aging_program_conflation_or_explain_away': not triggered\n  + hard_fail 'cf_no_aging_program_sensitivity_failure': not triggered\n  + hard_fail 'cf_senescence_driver_conclusion': not triggered",
      "metrics": {
        "pass_rule": "all",
        "scoring_count": 7,
        "scoring_passed": 5,
        "scoring_total_score": 5,
        "hard_fail_triggered": []
      },
      "fieldScores": {
        "cohort_spans_young_and_aged": 1,
        "candidate_is_senescence": 1,
        "arrest_core_not_engaged": 1,
        "secretory_arm_is_inflammaging": 0,
        "secretory_arm_broadly_elevated": 0,
        "carries_aging_program": 1,
        "conclusion_senescence_unlikely": 1
      },
      "graderChecks": [
        {
          "name": "cohort_spans_young_and_aged",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "candidate_is_senescence",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "arrest_core_not_engaged",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "secretory_arm_is_inflammaging",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "secretory_arm_broadly_elevated",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "carries_aging_program",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "conclusion_senescence_unlikely",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "cf_arrest_core_engaged_recall",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_aging_program_conflation_or_explain_away",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_no_aging_program_sensitivity_failure",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_senescence_driver_conclusion",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "age_associated_program_identity": "carries_bona_fide_aging_program_distinct_from_nominated",
        "arrest_core_engaged": "arrest_core_not_engaged",
        "candidate_mechanism": "cellular_senescence",
        "cohort_host_age_design": "young_and_aged_hosts_both_profiled",
        "nominated_mechanism_role": "unlikely_major_contributor",
        "secretory_arm_direction": "broadly_reduced",
        "secretory_arm_identity": "senescence_associated_secretory_phenotype"
      },
      "nTurns": 14,
      "checkpointSignals": {
        "clustering": false,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": false,
        "nerveJointAnchor": false,
        "multiomeAssembly": false,
        "depthNormalizedAtac": false,
        "regulonInference": false,
        "cd8SubsetIntegration": false,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": false,
        "orthologProjection": false,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": true,
        "mousePseudobulkDesign": false,
        "cancerEpithelialDenominator": true,
        "geneLevelDe": false,
        "agingSignatureControl": true,
        "senescencePanelScoring": true
      },
      "trajectoryFile": "scbench-long/trajectories/gse297023_q2_senescence_not_mechanism__claude-opus-4-7__pi__r3.json",
      "runId": "claude-opus-4-7__pi__gse297023_q2_senescence_not_mechanism__r3",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-opus-4-8-claude-code",
      "modelName": "claude-opus-4-8",
      "harness": "claude-code",
      "task": "gse297023_q2_senescence_not_mechanism",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 1,
      "passed": true,
      "score": 7,
      "durationS": 1461.99,
      "cost": 3.2074570000000002,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "candidate_is_senescence",
            "role": "gate",
            "field": "$.candidate_mechanism",
            "kind": "value",
            "op": "in",
            "expected": [
              "cellular_senescence",
              "oncogene_induced_senescence",
              "replicative_senescence"
            ]
          },
          {
            "name": "arrest_core_not_engaged",
            "role": "gate",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_not_engaged"
            ]
          },
          {
            "name": "secretory_arm_is_inflammaging",
            "role": "gate",
            "field": "$.secretory_arm_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "acute_phase_inflammaging_response"
            ]
          },
          {
            "name": "secretory_arm_broadly_elevated",
            "role": "gate",
            "field": "$.secretory_arm_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "broadly_elevated"
            ]
          },
          {
            "name": "carries_aging_program",
            "role": "gate",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "carries_bona_fide_aging_program_distinct_from_nominated"
            ]
          },
          {
            "name": "conclusion_senescence_unlikely",
            "role": "gate",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "unlikely_major_contributor"
            ]
          },
          {
            "name": "cf_arrest_core_engaged_recall",
            "role": "hard_fail",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_engaged"
            ]
          },
          {
            "name": "cf_aging_program_conflation_or_explain_away",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "aging_program_indistinguishable_from_nominated_program",
              "aging_program_is_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_no_aging_program_sensitivity_failure",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_detectable_aging_program"
            ]
          },
          {
            "name": "cf_senescence_driver_conclusion",
            "role": "hard_fail",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "likely_primary_driver",
              "plausible_supported_contributor"
            ]
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: PASS\n  + cohort_spans_young_and_aged: passed=True, score=1.0\n  + candidate_is_senescence: passed=True, score=1.0\n  + arrest_core_not_engaged: passed=True, score=1.0\n  + secretory_arm_is_inflammaging: passed=True, score=1.0\n  + secretory_arm_broadly_elevated: passed=True, score=1.0\n  + carries_aging_program: passed=True, score=1.0\n  + conclusion_senescence_unlikely: passed=True, score=1.0\n  + hard_fail 'cf_arrest_core_engaged_recall': not triggered\n  + hard_fail 'cf_aging_program_conflation_or_explain_away': not triggered\n  + hard_fail 'cf_no_aging_program_sensitivity_failure': not triggered\n  + hard_fail 'cf_senescence_driver_conclusion': not triggered",
      "metrics": {
        "pass_rule": "all",
        "scoring_count": 7,
        "scoring_passed": 7,
        "scoring_total_score": 7,
        "hard_fail_triggered": []
      },
      "fieldScores": {
        "cohort_spans_young_and_aged": 1,
        "candidate_is_senescence": 1,
        "arrest_core_not_engaged": 1,
        "secretory_arm_is_inflammaging": 1,
        "secretory_arm_broadly_elevated": 1,
        "carries_aging_program": 1,
        "conclusion_senescence_unlikely": 1
      },
      "graderChecks": [
        {
          "name": "cohort_spans_young_and_aged",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "candidate_is_senescence",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "arrest_core_not_engaged",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "secretory_arm_is_inflammaging",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "secretory_arm_broadly_elevated",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "carries_aging_program",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "conclusion_senescence_unlikely",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "cf_arrest_core_engaged_recall",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_aging_program_conflation_or_explain_away",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_no_aging_program_sensitivity_failure",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_senescence_driver_conclusion",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "age_associated_program_identity": "carries_bona_fide_aging_program_distinct_from_nominated",
        "arrest_core_engaged": "arrest_core_not_engaged",
        "candidate_mechanism": "oncogene_induced_senescence",
        "cohort_host_age_design": "young_and_aged_hosts_both_profiled",
        "nominated_mechanism_role": "unlikely_major_contributor",
        "secretory_arm_direction": "broadly_elevated",
        "secretory_arm_identity": "acute_phase_inflammaging_response"
      },
      "nTurns": 24,
      "checkpointSignals": {
        "clustering": true,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": true,
        "nerveJointAnchor": false,
        "multiomeAssembly": false,
        "depthNormalizedAtac": false,
        "regulonInference": false,
        "cd8SubsetIntegration": true,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": false,
        "orthologProjection": false,
        "epiSubsetSelection": true,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": true,
        "mousePseudobulkDesign": true,
        "cancerEpithelialDenominator": true,
        "geneLevelDe": true,
        "agingSignatureControl": true,
        "senescencePanelScoring": true
      },
      "trajectoryFile": "scbench-long/trajectories/gse297023_q2_senescence_not_mechanism__claude-opus-4-8__claude-code__r1.json",
      "runId": "claude-opus-4-8__claude-code__gse297023_q2_senescence_not_mechanism__r1",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-opus-4-8-claude-code",
      "modelName": "claude-opus-4-8",
      "harness": "claude-code",
      "task": "gse297023_q2_senescence_not_mechanism",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 2,
      "passed": true,
      "score": 7,
      "durationS": 1026.68,
      "cost": 2.5807397499999998,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "candidate_is_senescence",
            "role": "gate",
            "field": "$.candidate_mechanism",
            "kind": "value",
            "op": "in",
            "expected": [
              "cellular_senescence",
              "oncogene_induced_senescence",
              "replicative_senescence"
            ]
          },
          {
            "name": "arrest_core_not_engaged",
            "role": "gate",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_not_engaged"
            ]
          },
          {
            "name": "secretory_arm_is_inflammaging",
            "role": "gate",
            "field": "$.secretory_arm_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "acute_phase_inflammaging_response"
            ]
          },
          {
            "name": "secretory_arm_broadly_elevated",
            "role": "gate",
            "field": "$.secretory_arm_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "broadly_elevated"
            ]
          },
          {
            "name": "carries_aging_program",
            "role": "gate",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "carries_bona_fide_aging_program_distinct_from_nominated"
            ]
          },
          {
            "name": "conclusion_senescence_unlikely",
            "role": "gate",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "unlikely_major_contributor"
            ]
          },
          {
            "name": "cf_arrest_core_engaged_recall",
            "role": "hard_fail",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_engaged"
            ]
          },
          {
            "name": "cf_aging_program_conflation_or_explain_away",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "aging_program_indistinguishable_from_nominated_program",
              "aging_program_is_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_no_aging_program_sensitivity_failure",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_detectable_aging_program"
            ]
          },
          {
            "name": "cf_senescence_driver_conclusion",
            "role": "hard_fail",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "likely_primary_driver",
              "plausible_supported_contributor"
            ]
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: PASS\n  + cohort_spans_young_and_aged: passed=True, score=1.0\n  + candidate_is_senescence: passed=True, score=1.0\n  + arrest_core_not_engaged: passed=True, score=1.0\n  + secretory_arm_is_inflammaging: passed=True, score=1.0\n  + secretory_arm_broadly_elevated: passed=True, score=1.0\n  + carries_aging_program: passed=True, score=1.0\n  + conclusion_senescence_unlikely: passed=True, score=1.0\n  + hard_fail 'cf_arrest_core_engaged_recall': not triggered\n  + hard_fail 'cf_aging_program_conflation_or_explain_away': not triggered\n  + hard_fail 'cf_no_aging_program_sensitivity_failure': not triggered\n  + hard_fail 'cf_senescence_driver_conclusion': not triggered",
      "metrics": {
        "pass_rule": "all",
        "scoring_count": 7,
        "scoring_passed": 7,
        "scoring_total_score": 7,
        "hard_fail_triggered": []
      },
      "fieldScores": {
        "cohort_spans_young_and_aged": 1,
        "candidate_is_senescence": 1,
        "arrest_core_not_engaged": 1,
        "secretory_arm_is_inflammaging": 1,
        "secretory_arm_broadly_elevated": 1,
        "carries_aging_program": 1,
        "conclusion_senescence_unlikely": 1
      },
      "graderChecks": [
        {
          "name": "cohort_spans_young_and_aged",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "candidate_is_senescence",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "arrest_core_not_engaged",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "secretory_arm_is_inflammaging",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "secretory_arm_broadly_elevated",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "carries_aging_program",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "conclusion_senescence_unlikely",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "cf_arrest_core_engaged_recall",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_aging_program_conflation_or_explain_away",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_no_aging_program_sensitivity_failure",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_senescence_driver_conclusion",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "age_associated_program_identity": "carries_bona_fide_aging_program_distinct_from_nominated",
        "arrest_core_engaged": "arrest_core_not_engaged",
        "candidate_mechanism": "cellular_senescence",
        "cohort_host_age_design": "young_and_aged_hosts_both_profiled",
        "nominated_mechanism_role": "unlikely_major_contributor",
        "secretory_arm_direction": "broadly_elevated",
        "secretory_arm_identity": "acute_phase_inflammaging_response"
      },
      "nTurns": 25,
      "checkpointSignals": {
        "clustering": false,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": false,
        "nerveJointAnchor": false,
        "multiomeAssembly": false,
        "depthNormalizedAtac": false,
        "regulonInference": false,
        "cd8SubsetIntegration": true,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": false,
        "orthologProjection": false,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": true,
        "mousePseudobulkDesign": true,
        "cancerEpithelialDenominator": true,
        "geneLevelDe": true,
        "agingSignatureControl": true,
        "senescencePanelScoring": true
      },
      "trajectoryFile": "scbench-long/trajectories/gse297023_q2_senescence_not_mechanism__claude-opus-4-8__claude-code__r2.json",
      "runId": "claude-opus-4-8__claude-code__gse297023_q2_senescence_not_mechanism__r2",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-opus-4-8-claude-code",
      "modelName": "claude-opus-4-8",
      "harness": "claude-code",
      "task": "gse297023_q2_senescence_not_mechanism",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 3,
      "passed": false,
      "score": 6,
      "durationS": 1178.47,
      "cost": 2.85852475,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "candidate_is_senescence",
            "role": "gate",
            "field": "$.candidate_mechanism",
            "kind": "value",
            "op": "in",
            "expected": [
              "cellular_senescence",
              "oncogene_induced_senescence",
              "replicative_senescence"
            ]
          },
          {
            "name": "arrest_core_not_engaged",
            "role": "gate",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_not_engaged"
            ]
          },
          {
            "name": "secretory_arm_is_inflammaging",
            "role": "gate",
            "field": "$.secretory_arm_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "acute_phase_inflammaging_response"
            ]
          },
          {
            "name": "secretory_arm_broadly_elevated",
            "role": "gate",
            "field": "$.secretory_arm_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "broadly_elevated"
            ]
          },
          {
            "name": "carries_aging_program",
            "role": "gate",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "carries_bona_fide_aging_program_distinct_from_nominated"
            ]
          },
          {
            "name": "conclusion_senescence_unlikely",
            "role": "gate",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "unlikely_major_contributor"
            ]
          },
          {
            "name": "cf_arrest_core_engaged_recall",
            "role": "hard_fail",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_engaged"
            ]
          },
          {
            "name": "cf_aging_program_conflation_or_explain_away",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "aging_program_indistinguishable_from_nominated_program",
              "aging_program_is_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_no_aging_program_sensitivity_failure",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_detectable_aging_program"
            ]
          },
          {
            "name": "cf_senescence_driver_conclusion",
            "role": "hard_fail",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "likely_primary_driver",
              "plausible_supported_contributor"
            ]
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: FAIL\n  + cohort_spans_young_and_aged: passed=True, score=1.0\n  + candidate_is_senescence: passed=True, score=1.0\n  + arrest_core_not_engaged: passed=True, score=1.0\n  + secretory_arm_is_inflammaging: passed=True, score=1.0\n  + secretory_arm_broadly_elevated: passed=True, score=1.0\n  x carries_aging_program: passed=False, score=0.0\n  + conclusion_senescence_unlikely: passed=True, score=1.0\n  + hard_fail 'cf_arrest_core_engaged_recall': not triggered\n  + hard_fail 'cf_aging_program_conflation_or_explain_away': not triggered\n  + hard_fail 'cf_no_aging_program_sensitivity_failure': not triggered\n  + hard_fail 'cf_senescence_driver_conclusion': not triggered",
      "metrics": {
        "pass_rule": "all",
        "scoring_count": 7,
        "scoring_passed": 6,
        "scoring_total_score": 6,
        "hard_fail_triggered": []
      },
      "fieldScores": {
        "cohort_spans_young_and_aged": 1,
        "candidate_is_senescence": 1,
        "arrest_core_not_engaged": 1,
        "secretory_arm_is_inflammaging": 1,
        "secretory_arm_broadly_elevated": 1,
        "carries_aging_program": 0,
        "conclusion_senescence_unlikely": 1
      },
      "graderChecks": [
        {
          "name": "cohort_spans_young_and_aged",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "candidate_is_senescence",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "arrest_core_not_engaged",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "secretory_arm_is_inflammaging",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "secretory_arm_broadly_elevated",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "carries_aging_program",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "conclusion_senescence_unlikely",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "cf_arrest_core_engaged_recall",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_aging_program_conflation_or_explain_away",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_no_aging_program_sensitivity_failure",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_senescence_driver_conclusion",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "age_associated_program_identity": "aging_program_reflects_microenvironment_contamination",
        "arrest_core_engaged": "arrest_core_not_engaged",
        "candidate_mechanism": "cellular_senescence",
        "cohort_host_age_design": "young_and_aged_hosts_both_profiled",
        "nominated_mechanism_role": "unlikely_major_contributor",
        "secretory_arm_direction": "broadly_elevated",
        "secretory_arm_identity": "acute_phase_inflammaging_response"
      },
      "nTurns": 24,
      "checkpointSignals": {
        "clustering": true,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": false,
        "nerveJointAnchor": false,
        "multiomeAssembly": false,
        "depthNormalizedAtac": false,
        "regulonInference": false,
        "cd8SubsetIntegration": true,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": false,
        "orthologProjection": false,
        "epiSubsetSelection": true,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": true,
        "mousePseudobulkDesign": true,
        "cancerEpithelialDenominator": true,
        "geneLevelDe": true,
        "agingSignatureControl": true,
        "senescencePanelScoring": true
      },
      "trajectoryFile": "scbench-long/trajectories/gse297023_q2_senescence_not_mechanism__claude-opus-4-8__claude-code__r3.json",
      "runId": "claude-opus-4-8__claude-code__gse297023_q2_senescence_not_mechanism__r3",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-opus-4-8-pi",
      "modelName": "claude-opus-4-8",
      "harness": "pi",
      "task": "gse297023_q2_senescence_not_mechanism",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 1,
      "passed": true,
      "score": 7,
      "durationS": 889.4,
      "cost": 1.6469277500000001,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "candidate_is_senescence",
            "role": "gate",
            "field": "$.candidate_mechanism",
            "kind": "value",
            "op": "in",
            "expected": [
              "cellular_senescence",
              "oncogene_induced_senescence",
              "replicative_senescence"
            ]
          },
          {
            "name": "arrest_core_not_engaged",
            "role": "gate",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_not_engaged"
            ]
          },
          {
            "name": "secretory_arm_is_inflammaging",
            "role": "gate",
            "field": "$.secretory_arm_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "acute_phase_inflammaging_response"
            ]
          },
          {
            "name": "secretory_arm_broadly_elevated",
            "role": "gate",
            "field": "$.secretory_arm_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "broadly_elevated"
            ]
          },
          {
            "name": "carries_aging_program",
            "role": "gate",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "carries_bona_fide_aging_program_distinct_from_nominated"
            ]
          },
          {
            "name": "conclusion_senescence_unlikely",
            "role": "gate",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "unlikely_major_contributor"
            ]
          },
          {
            "name": "cf_arrest_core_engaged_recall",
            "role": "hard_fail",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_engaged"
            ]
          },
          {
            "name": "cf_aging_program_conflation_or_explain_away",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "aging_program_indistinguishable_from_nominated_program",
              "aging_program_is_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_no_aging_program_sensitivity_failure",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_detectable_aging_program"
            ]
          },
          {
            "name": "cf_senescence_driver_conclusion",
            "role": "hard_fail",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "likely_primary_driver",
              "plausible_supported_contributor"
            ]
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: PASS\n  + cohort_spans_young_and_aged: passed=True, score=1.0\n  + candidate_is_senescence: passed=True, score=1.0\n  + arrest_core_not_engaged: passed=True, score=1.0\n  + secretory_arm_is_inflammaging: passed=True, score=1.0\n  + secretory_arm_broadly_elevated: passed=True, score=1.0\n  + carries_aging_program: passed=True, score=1.0\n  + conclusion_senescence_unlikely: passed=True, score=1.0\n  + hard_fail 'cf_arrest_core_engaged_recall': not triggered\n  + hard_fail 'cf_aging_program_conflation_or_explain_away': not triggered\n  + hard_fail 'cf_no_aging_program_sensitivity_failure': not triggered\n  + hard_fail 'cf_senescence_driver_conclusion': not triggered",
      "metrics": {
        "pass_rule": "all",
        "scoring_count": 7,
        "scoring_passed": 7,
        "scoring_total_score": 7,
        "hard_fail_triggered": []
      },
      "fieldScores": {
        "cohort_spans_young_and_aged": 1,
        "candidate_is_senescence": 1,
        "arrest_core_not_engaged": 1,
        "secretory_arm_is_inflammaging": 1,
        "secretory_arm_broadly_elevated": 1,
        "carries_aging_program": 1,
        "conclusion_senescence_unlikely": 1
      },
      "graderChecks": [
        {
          "name": "cohort_spans_young_and_aged",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "candidate_is_senescence",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "arrest_core_not_engaged",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "secretory_arm_is_inflammaging",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "secretory_arm_broadly_elevated",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "carries_aging_program",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "conclusion_senescence_unlikely",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "cf_arrest_core_engaged_recall",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_aging_program_conflation_or_explain_away",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_no_aging_program_sensitivity_failure",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_senescence_driver_conclusion",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "age_associated_program_identity": "carries_bona_fide_aging_program_distinct_from_nominated",
        "arrest_core_engaged": "arrest_core_not_engaged",
        "candidate_mechanism": "cellular_senescence",
        "cohort_host_age_design": "young_and_aged_hosts_both_profiled",
        "nominated_mechanism_role": "unlikely_major_contributor",
        "secretory_arm_direction": "broadly_elevated",
        "secretory_arm_identity": "acute_phase_inflammaging_response"
      },
      "nTurns": 24,
      "checkpointSignals": {
        "clustering": true,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": true,
        "nerveJointAnchor": false,
        "multiomeAssembly": false,
        "depthNormalizedAtac": false,
        "regulonInference": false,
        "cd8SubsetIntegration": true,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": false,
        "orthologProjection": false,
        "epiSubsetSelection": true,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": true,
        "mousePseudobulkDesign": true,
        "cancerEpithelialDenominator": true,
        "geneLevelDe": true,
        "agingSignatureControl": true,
        "senescencePanelScoring": true
      },
      "trajectoryFile": "scbench-long/trajectories/gse297023_q2_senescence_not_mechanism__claude-opus-4-8__pi__r1.json",
      "runId": "claude-opus-4-8__pi__gse297023_q2_senescence_not_mechanism__r1",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-opus-4-8-pi",
      "modelName": "claude-opus-4-8",
      "harness": "pi",
      "task": "gse297023_q2_senescence_not_mechanism",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 2,
      "passed": true,
      "score": 7,
      "durationS": 546.18,
      "cost": 1.0904015,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "candidate_is_senescence",
            "role": "gate",
            "field": "$.candidate_mechanism",
            "kind": "value",
            "op": "in",
            "expected": [
              "cellular_senescence",
              "oncogene_induced_senescence",
              "replicative_senescence"
            ]
          },
          {
            "name": "arrest_core_not_engaged",
            "role": "gate",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_not_engaged"
            ]
          },
          {
            "name": "secretory_arm_is_inflammaging",
            "role": "gate",
            "field": "$.secretory_arm_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "acute_phase_inflammaging_response"
            ]
          },
          {
            "name": "secretory_arm_broadly_elevated",
            "role": "gate",
            "field": "$.secretory_arm_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "broadly_elevated"
            ]
          },
          {
            "name": "carries_aging_program",
            "role": "gate",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "carries_bona_fide_aging_program_distinct_from_nominated"
            ]
          },
          {
            "name": "conclusion_senescence_unlikely",
            "role": "gate",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "unlikely_major_contributor"
            ]
          },
          {
            "name": "cf_arrest_core_engaged_recall",
            "role": "hard_fail",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_engaged"
            ]
          },
          {
            "name": "cf_aging_program_conflation_or_explain_away",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "aging_program_indistinguishable_from_nominated_program",
              "aging_program_is_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_no_aging_program_sensitivity_failure",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_detectable_aging_program"
            ]
          },
          {
            "name": "cf_senescence_driver_conclusion",
            "role": "hard_fail",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "likely_primary_driver",
              "plausible_supported_contributor"
            ]
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: PASS\n  + cohort_spans_young_and_aged: passed=True, score=1.0\n  + candidate_is_senescence: passed=True, score=1.0\n  + arrest_core_not_engaged: passed=True, score=1.0\n  + secretory_arm_is_inflammaging: passed=True, score=1.0\n  + secretory_arm_broadly_elevated: passed=True, score=1.0\n  + carries_aging_program: passed=True, score=1.0\n  + conclusion_senescence_unlikely: passed=True, score=1.0\n  + hard_fail 'cf_arrest_core_engaged_recall': not triggered\n  + hard_fail 'cf_aging_program_conflation_or_explain_away': not triggered\n  + hard_fail 'cf_no_aging_program_sensitivity_failure': not triggered\n  + hard_fail 'cf_senescence_driver_conclusion': not triggered",
      "metrics": {
        "pass_rule": "all",
        "scoring_count": 7,
        "scoring_passed": 7,
        "scoring_total_score": 7,
        "hard_fail_triggered": []
      },
      "fieldScores": {
        "cohort_spans_young_and_aged": 1,
        "candidate_is_senescence": 1,
        "arrest_core_not_engaged": 1,
        "secretory_arm_is_inflammaging": 1,
        "secretory_arm_broadly_elevated": 1,
        "carries_aging_program": 1,
        "conclusion_senescence_unlikely": 1
      },
      "graderChecks": [
        {
          "name": "cohort_spans_young_and_aged",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "candidate_is_senescence",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "arrest_core_not_engaged",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "secretory_arm_is_inflammaging",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "secretory_arm_broadly_elevated",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "carries_aging_program",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "conclusion_senescence_unlikely",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "cf_arrest_core_engaged_recall",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_aging_program_conflation_or_explain_away",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_no_aging_program_sensitivity_failure",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_senescence_driver_conclusion",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "age_associated_program_identity": "carries_bona_fide_aging_program_distinct_from_nominated",
        "arrest_core_engaged": "arrest_core_not_engaged",
        "candidate_mechanism": "cellular_senescence",
        "cohort_host_age_design": "young_and_aged_hosts_both_profiled",
        "nominated_mechanism_role": "unlikely_major_contributor",
        "secretory_arm_direction": "broadly_elevated",
        "secretory_arm_identity": "acute_phase_inflammaging_response"
      },
      "nTurns": 19,
      "checkpointSignals": {
        "clustering": false,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": false,
        "nerveJointAnchor": false,
        "multiomeAssembly": false,
        "depthNormalizedAtac": false,
        "regulonInference": false,
        "cd8SubsetIntegration": false,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": false,
        "orthologProjection": false,
        "epiSubsetSelection": true,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": false,
        "mousePseudobulkDesign": true,
        "cancerEpithelialDenominator": true,
        "geneLevelDe": true,
        "agingSignatureControl": true,
        "senescencePanelScoring": true
      },
      "trajectoryFile": "scbench-long/trajectories/gse297023_q2_senescence_not_mechanism__claude-opus-4-8__pi__r2.json",
      "runId": "claude-opus-4-8__pi__gse297023_q2_senescence_not_mechanism__r2",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-opus-4-8-pi",
      "modelName": "claude-opus-4-8",
      "harness": "pi",
      "task": "gse297023_q2_senescence_not_mechanism",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 3,
      "passed": true,
      "score": 7,
      "durationS": 1220.43,
      "cost": 2.399626500000001,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "candidate_is_senescence",
            "role": "gate",
            "field": "$.candidate_mechanism",
            "kind": "value",
            "op": "in",
            "expected": [
              "cellular_senescence",
              "oncogene_induced_senescence",
              "replicative_senescence"
            ]
          },
          {
            "name": "arrest_core_not_engaged",
            "role": "gate",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_not_engaged"
            ]
          },
          {
            "name": "secretory_arm_is_inflammaging",
            "role": "gate",
            "field": "$.secretory_arm_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "acute_phase_inflammaging_response"
            ]
          },
          {
            "name": "secretory_arm_broadly_elevated",
            "role": "gate",
            "field": "$.secretory_arm_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "broadly_elevated"
            ]
          },
          {
            "name": "carries_aging_program",
            "role": "gate",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "carries_bona_fide_aging_program_distinct_from_nominated"
            ]
          },
          {
            "name": "conclusion_senescence_unlikely",
            "role": "gate",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "unlikely_major_contributor"
            ]
          },
          {
            "name": "cf_arrest_core_engaged_recall",
            "role": "hard_fail",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_engaged"
            ]
          },
          {
            "name": "cf_aging_program_conflation_or_explain_away",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "aging_program_indistinguishable_from_nominated_program",
              "aging_program_is_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_no_aging_program_sensitivity_failure",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_detectable_aging_program"
            ]
          },
          {
            "name": "cf_senescence_driver_conclusion",
            "role": "hard_fail",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "likely_primary_driver",
              "plausible_supported_contributor"
            ]
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: PASS\n  + cohort_spans_young_and_aged: passed=True, score=1.0\n  + candidate_is_senescence: passed=True, score=1.0\n  + arrest_core_not_engaged: passed=True, score=1.0\n  + secretory_arm_is_inflammaging: passed=True, score=1.0\n  + secretory_arm_broadly_elevated: passed=True, score=1.0\n  + carries_aging_program: passed=True, score=1.0\n  + conclusion_senescence_unlikely: passed=True, score=1.0\n  + hard_fail 'cf_arrest_core_engaged_recall': not triggered\n  + hard_fail 'cf_aging_program_conflation_or_explain_away': not triggered\n  + hard_fail 'cf_no_aging_program_sensitivity_failure': not triggered\n  + hard_fail 'cf_senescence_driver_conclusion': not triggered",
      "metrics": {
        "pass_rule": "all",
        "scoring_count": 7,
        "scoring_passed": 7,
        "scoring_total_score": 7,
        "hard_fail_triggered": []
      },
      "fieldScores": {
        "cohort_spans_young_and_aged": 1,
        "candidate_is_senescence": 1,
        "arrest_core_not_engaged": 1,
        "secretory_arm_is_inflammaging": 1,
        "secretory_arm_broadly_elevated": 1,
        "carries_aging_program": 1,
        "conclusion_senescence_unlikely": 1
      },
      "graderChecks": [
        {
          "name": "cohort_spans_young_and_aged",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "candidate_is_senescence",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "arrest_core_not_engaged",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "secretory_arm_is_inflammaging",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "secretory_arm_broadly_elevated",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "carries_aging_program",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "conclusion_senescence_unlikely",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "cf_arrest_core_engaged_recall",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_aging_program_conflation_or_explain_away",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_no_aging_program_sensitivity_failure",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_senescence_driver_conclusion",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "age_associated_program_identity": "carries_bona_fide_aging_program_distinct_from_nominated",
        "arrest_core_engaged": "arrest_core_not_engaged",
        "candidate_mechanism": "cellular_senescence",
        "cohort_host_age_design": "young_and_aged_hosts_both_profiled",
        "nominated_mechanism_role": "unlikely_major_contributor",
        "secretory_arm_direction": "broadly_elevated",
        "secretory_arm_identity": "acute_phase_inflammaging_response"
      },
      "nTurns": 23,
      "checkpointSignals": {
        "clustering": true,
        "markerAnnotation": true,
        "scoreBasedAnnotation": true,
        "normalization": false,
        "nerveJointAnchor": false,
        "multiomeAssembly": false,
        "depthNormalizedAtac": false,
        "regulonInference": false,
        "cd8SubsetIntegration": true,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": false,
        "orthologProjection": false,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": false,
        "mousePseudobulkDesign": true,
        "cancerEpithelialDenominator": true,
        "geneLevelDe": true,
        "agingSignatureControl": true,
        "senescencePanelScoring": true
      },
      "trajectoryFile": "scbench-long/trajectories/gse297023_q2_senescence_not_mechanism__claude-opus-4-8__pi__r3.json",
      "runId": "claude-opus-4-8__pi__gse297023_q2_senescence_not_mechanism__r3",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-sonnet-4-6-pi",
      "modelName": "claude-sonnet-4-6",
      "harness": "pi",
      "task": "gse297023_q2_senescence_not_mechanism",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 1,
      "passed": true,
      "score": 7,
      "durationS": 1666.89,
      "cost": 0.83180505,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "candidate_is_senescence",
            "role": "gate",
            "field": "$.candidate_mechanism",
            "kind": "value",
            "op": "in",
            "expected": [
              "cellular_senescence",
              "oncogene_induced_senescence",
              "replicative_senescence"
            ]
          },
          {
            "name": "arrest_core_not_engaged",
            "role": "gate",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_not_engaged"
            ]
          },
          {
            "name": "secretory_arm_is_inflammaging",
            "role": "gate",
            "field": "$.secretory_arm_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "acute_phase_inflammaging_response"
            ]
          },
          {
            "name": "secretory_arm_broadly_elevated",
            "role": "gate",
            "field": "$.secretory_arm_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "broadly_elevated"
            ]
          },
          {
            "name": "carries_aging_program",
            "role": "gate",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "carries_bona_fide_aging_program_distinct_from_nominated"
            ]
          },
          {
            "name": "conclusion_senescence_unlikely",
            "role": "gate",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "unlikely_major_contributor"
            ]
          },
          {
            "name": "cf_arrest_core_engaged_recall",
            "role": "hard_fail",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_engaged"
            ]
          },
          {
            "name": "cf_aging_program_conflation_or_explain_away",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "aging_program_indistinguishable_from_nominated_program",
              "aging_program_is_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_no_aging_program_sensitivity_failure",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_detectable_aging_program"
            ]
          },
          {
            "name": "cf_senescence_driver_conclusion",
            "role": "hard_fail",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "likely_primary_driver",
              "plausible_supported_contributor"
            ]
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: PASS\n  + cohort_spans_young_and_aged: passed=True, score=1.0\n  + candidate_is_senescence: passed=True, score=1.0\n  + arrest_core_not_engaged: passed=True, score=1.0\n  + secretory_arm_is_inflammaging: passed=True, score=1.0\n  + secretory_arm_broadly_elevated: passed=True, score=1.0\n  + carries_aging_program: passed=True, score=1.0\n  + conclusion_senescence_unlikely: passed=True, score=1.0\n  + hard_fail 'cf_arrest_core_engaged_recall': not triggered\n  + hard_fail 'cf_aging_program_conflation_or_explain_away': not triggered\n  + hard_fail 'cf_no_aging_program_sensitivity_failure': not triggered\n  + hard_fail 'cf_senescence_driver_conclusion': not triggered",
      "metrics": {
        "pass_rule": "all",
        "scoring_count": 7,
        "scoring_passed": 7,
        "scoring_total_score": 7,
        "hard_fail_triggered": []
      },
      "fieldScores": {
        "cohort_spans_young_and_aged": 1,
        "candidate_is_senescence": 1,
        "arrest_core_not_engaged": 1,
        "secretory_arm_is_inflammaging": 1,
        "secretory_arm_broadly_elevated": 1,
        "carries_aging_program": 1,
        "conclusion_senescence_unlikely": 1
      },
      "graderChecks": [
        {
          "name": "cohort_spans_young_and_aged",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "candidate_is_senescence",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "arrest_core_not_engaged",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "secretory_arm_is_inflammaging",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "secretory_arm_broadly_elevated",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "carries_aging_program",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "conclusion_senescence_unlikely",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "cf_arrest_core_engaged_recall",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_aging_program_conflation_or_explain_away",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_no_aging_program_sensitivity_failure",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_senescence_driver_conclusion",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "age_associated_program_identity": "carries_bona_fide_aging_program_distinct_from_nominated",
        "arrest_core_engaged": "arrest_core_not_engaged",
        "candidate_mechanism": "cellular_senescence",
        "cohort_host_age_design": "young_and_aged_hosts_both_profiled",
        "nominated_mechanism_role": "unlikely_major_contributor",
        "secretory_arm_direction": "broadly_elevated",
        "secretory_arm_identity": "acute_phase_inflammaging_response"
      },
      "nTurns": 23,
      "checkpointSignals": {
        "clustering": true,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": false,
        "nerveJointAnchor": false,
        "multiomeAssembly": false,
        "depthNormalizedAtac": true,
        "regulonInference": false,
        "cd8SubsetIntegration": false,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": false,
        "orthologProjection": false,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": false,
        "mousePseudobulkDesign": true,
        "cancerEpithelialDenominator": true,
        "geneLevelDe": true,
        "agingSignatureControl": true,
        "senescencePanelScoring": true
      },
      "trajectoryFile": "scbench-long/trajectories/gse297023_q2_senescence_not_mechanism__claude-sonnet-4-6__pi__r1.json",
      "runId": "claude-sonnet-4-6__pi__gse297023_q2_senescence_not_mechanism__r1",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-sonnet-4-6-pi",
      "modelName": "claude-sonnet-4-6",
      "harness": "pi",
      "task": "gse297023_q2_senescence_not_mechanism",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 2,
      "passed": false,
      "score": 5,
      "durationS": 1467.54,
      "cost": 0.78863835,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "candidate_is_senescence",
            "role": "gate",
            "field": "$.candidate_mechanism",
            "kind": "value",
            "op": "in",
            "expected": [
              "cellular_senescence",
              "oncogene_induced_senescence",
              "replicative_senescence"
            ]
          },
          {
            "name": "arrest_core_not_engaged",
            "role": "gate",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_not_engaged"
            ]
          },
          {
            "name": "secretory_arm_is_inflammaging",
            "role": "gate",
            "field": "$.secretory_arm_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "acute_phase_inflammaging_response"
            ]
          },
          {
            "name": "secretory_arm_broadly_elevated",
            "role": "gate",
            "field": "$.secretory_arm_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "broadly_elevated"
            ]
          },
          {
            "name": "carries_aging_program",
            "role": "gate",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "carries_bona_fide_aging_program_distinct_from_nominated"
            ]
          },
          {
            "name": "conclusion_senescence_unlikely",
            "role": "gate",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "unlikely_major_contributor"
            ]
          },
          {
            "name": "cf_arrest_core_engaged_recall",
            "role": "hard_fail",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_engaged"
            ]
          },
          {
            "name": "cf_aging_program_conflation_or_explain_away",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "aging_program_indistinguishable_from_nominated_program",
              "aging_program_is_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_no_aging_program_sensitivity_failure",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_detectable_aging_program"
            ]
          },
          {
            "name": "cf_senescence_driver_conclusion",
            "role": "hard_fail",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "likely_primary_driver",
              "plausible_supported_contributor"
            ]
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: FAIL\n  + cohort_spans_young_and_aged: passed=True, score=1.0\n  + candidate_is_senescence: passed=True, score=1.0\n  + arrest_core_not_engaged: passed=True, score=1.0\n  x secretory_arm_is_inflammaging: passed=False, score=0.0\n  x secretory_arm_broadly_elevated: passed=False, score=0.0\n  + carries_aging_program: passed=True, score=1.0\n  + conclusion_senescence_unlikely: passed=True, score=1.0\n  + hard_fail 'cf_arrest_core_engaged_recall': not triggered\n  + hard_fail 'cf_aging_program_conflation_or_explain_away': not triggered\n  + hard_fail 'cf_no_aging_program_sensitivity_failure': not triggered\n  + hard_fail 'cf_senescence_driver_conclusion': not triggered",
      "metrics": {
        "pass_rule": "all",
        "scoring_count": 7,
        "scoring_passed": 5,
        "scoring_total_score": 5,
        "hard_fail_triggered": []
      },
      "fieldScores": {
        "cohort_spans_young_and_aged": 1,
        "candidate_is_senescence": 1,
        "arrest_core_not_engaged": 1,
        "secretory_arm_is_inflammaging": 0,
        "secretory_arm_broadly_elevated": 0,
        "carries_aging_program": 1,
        "conclusion_senescence_unlikely": 1
      },
      "graderChecks": [
        {
          "name": "cohort_spans_young_and_aged",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "candidate_is_senescence",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "arrest_core_not_engaged",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "secretory_arm_is_inflammaging",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "secretory_arm_broadly_elevated",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "carries_aging_program",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "conclusion_senescence_unlikely",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "cf_arrest_core_engaged_recall",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_aging_program_conflation_or_explain_away",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_no_aging_program_sensitivity_failure",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_senescence_driver_conclusion",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "age_associated_program_identity": "carries_bona_fide_aging_program_distinct_from_nominated",
        "arrest_core_engaged": "arrest_core_not_engaged",
        "candidate_mechanism": "oncogene_induced_senescence",
        "cohort_host_age_design": "young_and_aged_hosts_both_profiled",
        "nominated_mechanism_role": "unlikely_major_contributor",
        "secretory_arm_direction": "broadly_reduced",
        "secretory_arm_identity": "senescence_associated_secretory_phenotype"
      },
      "nTurns": 20,
      "checkpointSignals": {
        "clustering": true,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": false,
        "nerveJointAnchor": false,
        "multiomeAssembly": false,
        "depthNormalizedAtac": false,
        "regulonInference": false,
        "cd8SubsetIntegration": false,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": false,
        "orthologProjection": false,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": false,
        "mousePseudobulkDesign": false,
        "cancerEpithelialDenominator": true,
        "geneLevelDe": false,
        "agingSignatureControl": true,
        "senescencePanelScoring": true
      },
      "trajectoryFile": "scbench-long/trajectories/gse297023_q2_senescence_not_mechanism__claude-sonnet-4-6__pi__r2.json",
      "runId": "claude-sonnet-4-6__pi__gse297023_q2_senescence_not_mechanism__r2",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-sonnet-4-6-pi",
      "modelName": "claude-sonnet-4-6",
      "harness": "pi",
      "task": "gse297023_q2_senescence_not_mechanism",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 3,
      "passed": false,
      "score": 3,
      "durationS": 1480.04,
      "cost": 1.0803085499999998,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "candidate_is_senescence",
            "role": "gate",
            "field": "$.candidate_mechanism",
            "kind": "value",
            "op": "in",
            "expected": [
              "cellular_senescence",
              "oncogene_induced_senescence",
              "replicative_senescence"
            ]
          },
          {
            "name": "arrest_core_not_engaged",
            "role": "gate",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_not_engaged"
            ]
          },
          {
            "name": "secretory_arm_is_inflammaging",
            "role": "gate",
            "field": "$.secretory_arm_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "acute_phase_inflammaging_response"
            ]
          },
          {
            "name": "secretory_arm_broadly_elevated",
            "role": "gate",
            "field": "$.secretory_arm_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "broadly_elevated"
            ]
          },
          {
            "name": "carries_aging_program",
            "role": "gate",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "carries_bona_fide_aging_program_distinct_from_nominated"
            ]
          },
          {
            "name": "conclusion_senescence_unlikely",
            "role": "gate",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "unlikely_major_contributor"
            ]
          },
          {
            "name": "cf_arrest_core_engaged_recall",
            "role": "hard_fail",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_engaged"
            ]
          },
          {
            "name": "cf_aging_program_conflation_or_explain_away",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "aging_program_indistinguishable_from_nominated_program",
              "aging_program_is_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_no_aging_program_sensitivity_failure",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_detectable_aging_program"
            ]
          },
          {
            "name": "cf_senescence_driver_conclusion",
            "role": "hard_fail",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "likely_primary_driver",
              "plausible_supported_contributor"
            ]
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: FAIL\n  + cohort_spans_young_and_aged: passed=True, score=1.0\n  + candidate_is_senescence: passed=True, score=1.0\n  + arrest_core_not_engaged: passed=True, score=1.0\n  x secretory_arm_is_inflammaging: passed=False, score=0.0\n  x secretory_arm_broadly_elevated: passed=False, score=0.0\n  x carries_aging_program: passed=False, score=0.0\n  x conclusion_senescence_unlikely: passed=False, score=0.0\n  + hard_fail 'cf_arrest_core_engaged_recall': not triggered\n  x hard_fail 'cf_aging_program_conflation_or_explain_away': TRIGGERED\n  + hard_fail 'cf_no_aging_program_sensitivity_failure': not triggered\n  x hard_fail 'cf_senescence_driver_conclusion': TRIGGERED",
      "metrics": {
        "pass_rule": "all",
        "scoring_count": 7,
        "scoring_passed": 3,
        "scoring_total_score": 3,
        "hard_fail_triggered": [
          "cf_aging_program_conflation_or_explain_away",
          "cf_senescence_driver_conclusion"
        ]
      },
      "fieldScores": {
        "cohort_spans_young_and_aged": 1,
        "candidate_is_senescence": 1,
        "arrest_core_not_engaged": 1,
        "secretory_arm_is_inflammaging": 0,
        "secretory_arm_broadly_elevated": 0,
        "carries_aging_program": 0,
        "conclusion_senescence_unlikely": 0
      },
      "graderChecks": [
        {
          "name": "cohort_spans_young_and_aged",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "candidate_is_senescence",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "arrest_core_not_engaged",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "secretory_arm_is_inflammaging",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "secretory_arm_broadly_elevated",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "carries_aging_program",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "conclusion_senescence_unlikely",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "cf_arrest_core_engaged_recall",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_aging_program_conflation_or_explain_away",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "cf_no_aging_program_sensitivity_failure",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_senescence_driver_conclusion",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        }
      ],
      "agentAnswer": {
        "age_associated_program_identity": "aging_program_indistinguishable_from_nominated_program",
        "arrest_core_engaged": "arrest_core_not_engaged",
        "candidate_mechanism": "oncogene_induced_senescence",
        "cohort_host_age_design": "young_and_aged_hosts_both_profiled",
        "nominated_mechanism_role": "plausible_supported_contributor",
        "secretory_arm_direction": "broadly_reduced",
        "secretory_arm_identity": "senescence_associated_secretory_phenotype"
      },
      "nTurns": 25,
      "checkpointSignals": {
        "clustering": true,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": false,
        "nerveJointAnchor": false,
        "multiomeAssembly": false,
        "depthNormalizedAtac": false,
        "regulonInference": false,
        "cd8SubsetIntegration": false,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": false,
        "orthologProjection": true,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": true,
        "mousePseudobulkDesign": true,
        "cancerEpithelialDenominator": true,
        "geneLevelDe": true,
        "agingSignatureControl": true,
        "senescencePanelScoring": true
      },
      "trajectoryFile": "scbench-long/trajectories/gse297023_q2_senescence_not_mechanism__claude-sonnet-4-6__pi__r3.json",
      "runId": "claude-sonnet-4-6__pi__gse297023_q2_senescence_not_mechanism__r3",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-gemini-3-1-pro-preview-pi",
      "modelName": "gemini-3.1-pro-preview",
      "harness": "pi",
      "task": "gse297023_q2_senescence_not_mechanism",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 1,
      "passed": true,
      "score": 7,
      "durationS": 730.19,
      "cost": 0.44277779999999994,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "candidate_is_senescence",
            "role": "gate",
            "field": "$.candidate_mechanism",
            "kind": "value",
            "op": "in",
            "expected": [
              "cellular_senescence",
              "oncogene_induced_senescence",
              "replicative_senescence"
            ]
          },
          {
            "name": "arrest_core_not_engaged",
            "role": "gate",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_not_engaged"
            ]
          },
          {
            "name": "secretory_arm_is_inflammaging",
            "role": "gate",
            "field": "$.secretory_arm_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "acute_phase_inflammaging_response"
            ]
          },
          {
            "name": "secretory_arm_broadly_elevated",
            "role": "gate",
            "field": "$.secretory_arm_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "broadly_elevated"
            ]
          },
          {
            "name": "carries_aging_program",
            "role": "gate",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "carries_bona_fide_aging_program_distinct_from_nominated"
            ]
          },
          {
            "name": "conclusion_senescence_unlikely",
            "role": "gate",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "unlikely_major_contributor"
            ]
          },
          {
            "name": "cf_arrest_core_engaged_recall",
            "role": "hard_fail",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_engaged"
            ]
          },
          {
            "name": "cf_aging_program_conflation_or_explain_away",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "aging_program_indistinguishable_from_nominated_program",
              "aging_program_is_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_no_aging_program_sensitivity_failure",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_detectable_aging_program"
            ]
          },
          {
            "name": "cf_senescence_driver_conclusion",
            "role": "hard_fail",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "likely_primary_driver",
              "plausible_supported_contributor"
            ]
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: PASS\n  + cohort_spans_young_and_aged: passed=True, score=1.0\n  + candidate_is_senescence: passed=True, score=1.0\n  + arrest_core_not_engaged: passed=True, score=1.0\n  + secretory_arm_is_inflammaging: passed=True, score=1.0\n  + secretory_arm_broadly_elevated: passed=True, score=1.0\n  + carries_aging_program: passed=True, score=1.0\n  + conclusion_senescence_unlikely: passed=True, score=1.0\n  + hard_fail 'cf_arrest_core_engaged_recall': not triggered\n  + hard_fail 'cf_aging_program_conflation_or_explain_away': not triggered\n  + hard_fail 'cf_no_aging_program_sensitivity_failure': not triggered\n  + hard_fail 'cf_senescence_driver_conclusion': not triggered",
      "metrics": {
        "pass_rule": "all",
        "scoring_count": 7,
        "scoring_passed": 7,
        "scoring_total_score": 7,
        "hard_fail_triggered": []
      },
      "fieldScores": {
        "cohort_spans_young_and_aged": 1,
        "candidate_is_senescence": 1,
        "arrest_core_not_engaged": 1,
        "secretory_arm_is_inflammaging": 1,
        "secretory_arm_broadly_elevated": 1,
        "carries_aging_program": 1,
        "conclusion_senescence_unlikely": 1
      },
      "graderChecks": [
        {
          "name": "cohort_spans_young_and_aged",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "candidate_is_senescence",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "arrest_core_not_engaged",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "secretory_arm_is_inflammaging",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "secretory_arm_broadly_elevated",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "carries_aging_program",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "conclusion_senescence_unlikely",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "cf_arrest_core_engaged_recall",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_aging_program_conflation_or_explain_away",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_no_aging_program_sensitivity_failure",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_senescence_driver_conclusion",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "age_associated_program_identity": "carries_bona_fide_aging_program_distinct_from_nominated",
        "arrest_core_engaged": "arrest_core_not_engaged",
        "candidate_mechanism": "cellular_senescence",
        "cohort_host_age_design": "young_and_aged_hosts_both_profiled",
        "nominated_mechanism_role": "unlikely_major_contributor",
        "secretory_arm_direction": "broadly_elevated",
        "secretory_arm_identity": "acute_phase_inflammaging_response"
      },
      "nTurns": 16,
      "checkpointSignals": {
        "clustering": false,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": false,
        "nerveJointAnchor": false,
        "multiomeAssembly": false,
        "depthNormalizedAtac": false,
        "regulonInference": false,
        "cd8SubsetIntegration": false,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": false,
        "orthologProjection": false,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": false,
        "mousePseudobulkDesign": false,
        "cancerEpithelialDenominator": true,
        "geneLevelDe": false,
        "agingSignatureControl": true,
        "senescencePanelScoring": true
      },
      "trajectoryFile": "scbench-long/trajectories/gse297023_q2_senescence_not_mechanism__gemini-3.1-pro-preview__pi__r1.json",
      "runId": "gemini-3.1-pro-preview__pi__gse297023_q2_senescence_not_mechanism__r1",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-gemini-3-1-pro-preview-pi",
      "modelName": "gemini-3.1-pro-preview",
      "harness": "pi",
      "task": "gse297023_q2_senescence_not_mechanism",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 2,
      "passed": false,
      "score": 5,
      "durationS": 1200.96,
      "cost": 0.456124,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "candidate_is_senescence",
            "role": "gate",
            "field": "$.candidate_mechanism",
            "kind": "value",
            "op": "in",
            "expected": [
              "cellular_senescence",
              "oncogene_induced_senescence",
              "replicative_senescence"
            ]
          },
          {
            "name": "arrest_core_not_engaged",
            "role": "gate",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_not_engaged"
            ]
          },
          {
            "name": "secretory_arm_is_inflammaging",
            "role": "gate",
            "field": "$.secretory_arm_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "acute_phase_inflammaging_response"
            ]
          },
          {
            "name": "secretory_arm_broadly_elevated",
            "role": "gate",
            "field": "$.secretory_arm_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "broadly_elevated"
            ]
          },
          {
            "name": "carries_aging_program",
            "role": "gate",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "carries_bona_fide_aging_program_distinct_from_nominated"
            ]
          },
          {
            "name": "conclusion_senescence_unlikely",
            "role": "gate",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "unlikely_major_contributor"
            ]
          },
          {
            "name": "cf_arrest_core_engaged_recall",
            "role": "hard_fail",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_engaged"
            ]
          },
          {
            "name": "cf_aging_program_conflation_or_explain_away",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "aging_program_indistinguishable_from_nominated_program",
              "aging_program_is_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_no_aging_program_sensitivity_failure",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_detectable_aging_program"
            ]
          },
          {
            "name": "cf_senescence_driver_conclusion",
            "role": "hard_fail",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "likely_primary_driver",
              "plausible_supported_contributor"
            ]
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: FAIL\n  + cohort_spans_young_and_aged: passed=True, score=1.0\n  + candidate_is_senescence: passed=True, score=1.0\n  + arrest_core_not_engaged: passed=True, score=1.0\n  x secretory_arm_is_inflammaging: passed=False, score=0.0\n  x secretory_arm_broadly_elevated: passed=False, score=0.0\n  + carries_aging_program: passed=True, score=1.0\n  + conclusion_senescence_unlikely: passed=True, score=1.0\n  + hard_fail 'cf_arrest_core_engaged_recall': not triggered\n  + hard_fail 'cf_aging_program_conflation_or_explain_away': not triggered\n  + hard_fail 'cf_no_aging_program_sensitivity_failure': not triggered\n  + hard_fail 'cf_senescence_driver_conclusion': not triggered",
      "metrics": {
        "pass_rule": "all",
        "scoring_count": 7,
        "scoring_passed": 5,
        "scoring_total_score": 5,
        "hard_fail_triggered": []
      },
      "fieldScores": {
        "cohort_spans_young_and_aged": 1,
        "candidate_is_senescence": 1,
        "arrest_core_not_engaged": 1,
        "secretory_arm_is_inflammaging": 0,
        "secretory_arm_broadly_elevated": 0,
        "carries_aging_program": 1,
        "conclusion_senescence_unlikely": 1
      },
      "graderChecks": [
        {
          "name": "cohort_spans_young_and_aged",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "candidate_is_senescence",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "arrest_core_not_engaged",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "secretory_arm_is_inflammaging",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "secretory_arm_broadly_elevated",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "carries_aging_program",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "conclusion_senescence_unlikely",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "cf_arrest_core_engaged_recall",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_aging_program_conflation_or_explain_away",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_no_aging_program_sensitivity_failure",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_senescence_driver_conclusion",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "age_associated_program_identity": "carries_bona_fide_aging_program_distinct_from_nominated",
        "arrest_core_engaged": "arrest_core_not_engaged",
        "candidate_mechanism": "cellular_senescence",
        "cohort_host_age_design": "young_and_aged_hosts_both_profiled",
        "nominated_mechanism_role": "unlikely_major_contributor",
        "secretory_arm_direction": "broadly_reduced",
        "secretory_arm_identity": "senescence_associated_secretory_phenotype"
      },
      "nTurns": 27,
      "checkpointSignals": {
        "clustering": false,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": false,
        "nerveJointAnchor": false,
        "multiomeAssembly": false,
        "depthNormalizedAtac": false,
        "regulonInference": false,
        "cd8SubsetIntegration": false,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": false,
        "orthologProjection": false,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": false,
        "mousePseudobulkDesign": true,
        "cancerEpithelialDenominator": true,
        "geneLevelDe": true,
        "agingSignatureControl": true,
        "senescencePanelScoring": true
      },
      "trajectoryFile": "scbench-long/trajectories/gse297023_q2_senescence_not_mechanism__gemini-3.1-pro-preview__pi__r2.json",
      "runId": "gemini-3.1-pro-preview__pi__gse297023_q2_senescence_not_mechanism__r2",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-gemini-3-1-pro-preview-pi",
      "modelName": "gemini-3.1-pro-preview",
      "harness": "pi",
      "task": "gse297023_q2_senescence_not_mechanism",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 3,
      "passed": true,
      "score": 7,
      "durationS": 691.68,
      "cost": 0.34545079999999995,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "candidate_is_senescence",
            "role": "gate",
            "field": "$.candidate_mechanism",
            "kind": "value",
            "op": "in",
            "expected": [
              "cellular_senescence",
              "oncogene_induced_senescence",
              "replicative_senescence"
            ]
          },
          {
            "name": "arrest_core_not_engaged",
            "role": "gate",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_not_engaged"
            ]
          },
          {
            "name": "secretory_arm_is_inflammaging",
            "role": "gate",
            "field": "$.secretory_arm_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "acute_phase_inflammaging_response"
            ]
          },
          {
            "name": "secretory_arm_broadly_elevated",
            "role": "gate",
            "field": "$.secretory_arm_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "broadly_elevated"
            ]
          },
          {
            "name": "carries_aging_program",
            "role": "gate",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "carries_bona_fide_aging_program_distinct_from_nominated"
            ]
          },
          {
            "name": "conclusion_senescence_unlikely",
            "role": "gate",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "unlikely_major_contributor"
            ]
          },
          {
            "name": "cf_arrest_core_engaged_recall",
            "role": "hard_fail",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_engaged"
            ]
          },
          {
            "name": "cf_aging_program_conflation_or_explain_away",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "aging_program_indistinguishable_from_nominated_program",
              "aging_program_is_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_no_aging_program_sensitivity_failure",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_detectable_aging_program"
            ]
          },
          {
            "name": "cf_senescence_driver_conclusion",
            "role": "hard_fail",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "likely_primary_driver",
              "plausible_supported_contributor"
            ]
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: PASS\n  + cohort_spans_young_and_aged: passed=True, score=1.0\n  + candidate_is_senescence: passed=True, score=1.0\n  + arrest_core_not_engaged: passed=True, score=1.0\n  + secretory_arm_is_inflammaging: passed=True, score=1.0\n  + secretory_arm_broadly_elevated: passed=True, score=1.0\n  + carries_aging_program: passed=True, score=1.0\n  + conclusion_senescence_unlikely: passed=True, score=1.0\n  + hard_fail 'cf_arrest_core_engaged_recall': not triggered\n  + hard_fail 'cf_aging_program_conflation_or_explain_away': not triggered\n  + hard_fail 'cf_no_aging_program_sensitivity_failure': not triggered\n  + hard_fail 'cf_senescence_driver_conclusion': not triggered",
      "metrics": {
        "pass_rule": "all",
        "scoring_count": 7,
        "scoring_passed": 7,
        "scoring_total_score": 7,
        "hard_fail_triggered": []
      },
      "fieldScores": {
        "cohort_spans_young_and_aged": 1,
        "candidate_is_senescence": 1,
        "arrest_core_not_engaged": 1,
        "secretory_arm_is_inflammaging": 1,
        "secretory_arm_broadly_elevated": 1,
        "carries_aging_program": 1,
        "conclusion_senescence_unlikely": 1
      },
      "graderChecks": [
        {
          "name": "cohort_spans_young_and_aged",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "candidate_is_senescence",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "arrest_core_not_engaged",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "secretory_arm_is_inflammaging",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "secretory_arm_broadly_elevated",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "carries_aging_program",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "conclusion_senescence_unlikely",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "cf_arrest_core_engaged_recall",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_aging_program_conflation_or_explain_away",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_no_aging_program_sensitivity_failure",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_senescence_driver_conclusion",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "age_associated_program_identity": "carries_bona_fide_aging_program_distinct_from_nominated",
        "arrest_core_engaged": "arrest_core_not_engaged",
        "candidate_mechanism": "cellular_senescence",
        "cohort_host_age_design": "young_and_aged_hosts_both_profiled",
        "nominated_mechanism_role": "unlikely_major_contributor",
        "secretory_arm_direction": "broadly_elevated",
        "secretory_arm_identity": "acute_phase_inflammaging_response"
      },
      "nTurns": 16,
      "checkpointSignals": {
        "clustering": false,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": false,
        "nerveJointAnchor": false,
        "multiomeAssembly": false,
        "depthNormalizedAtac": false,
        "regulonInference": false,
        "cd8SubsetIntegration": false,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": false,
        "orthologProjection": false,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": false,
        "mousePseudobulkDesign": false,
        "cancerEpithelialDenominator": true,
        "geneLevelDe": true,
        "agingSignatureControl": true,
        "senescencePanelScoring": true
      },
      "trajectoryFile": "scbench-long/trajectories/gse297023_q2_senescence_not_mechanism__gemini-3.1-pro-preview__pi__r3.json",
      "runId": "gemini-3.1-pro-preview__pi__gse297023_q2_senescence_not_mechanism__r3",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-gemini-3-5-flash-pi",
      "modelName": "gemini-3.5-flash",
      "harness": "pi",
      "task": "gse297023_q2_senescence_not_mechanism",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 1,
      "passed": false,
      "score": 5,
      "durationS": 291.11,
      "cost": 0.42536369999999996,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "candidate_is_senescence",
            "role": "gate",
            "field": "$.candidate_mechanism",
            "kind": "value",
            "op": "in",
            "expected": [
              "cellular_senescence",
              "oncogene_induced_senescence",
              "replicative_senescence"
            ]
          },
          {
            "name": "arrest_core_not_engaged",
            "role": "gate",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_not_engaged"
            ]
          },
          {
            "name": "secretory_arm_is_inflammaging",
            "role": "gate",
            "field": "$.secretory_arm_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "acute_phase_inflammaging_response"
            ]
          },
          {
            "name": "secretory_arm_broadly_elevated",
            "role": "gate",
            "field": "$.secretory_arm_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "broadly_elevated"
            ]
          },
          {
            "name": "carries_aging_program",
            "role": "gate",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "carries_bona_fide_aging_program_distinct_from_nominated"
            ]
          },
          {
            "name": "conclusion_senescence_unlikely",
            "role": "gate",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "unlikely_major_contributor"
            ]
          },
          {
            "name": "cf_arrest_core_engaged_recall",
            "role": "hard_fail",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_engaged"
            ]
          },
          {
            "name": "cf_aging_program_conflation_or_explain_away",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "aging_program_indistinguishable_from_nominated_program",
              "aging_program_is_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_no_aging_program_sensitivity_failure",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_detectable_aging_program"
            ]
          },
          {
            "name": "cf_senescence_driver_conclusion",
            "role": "hard_fail",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "likely_primary_driver",
              "plausible_supported_contributor"
            ]
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: FAIL\n  + cohort_spans_young_and_aged: passed=True, score=1.0\n  + candidate_is_senescence: passed=True, score=1.0\n  + arrest_core_not_engaged: passed=True, score=1.0\n  x secretory_arm_is_inflammaging: passed=False, score=0.0\n  x secretory_arm_broadly_elevated: passed=False, score=0.0\n  + carries_aging_program: passed=True, score=1.0\n  + conclusion_senescence_unlikely: passed=True, score=1.0\n  + hard_fail 'cf_arrest_core_engaged_recall': not triggered\n  + hard_fail 'cf_aging_program_conflation_or_explain_away': not triggered\n  + hard_fail 'cf_no_aging_program_sensitivity_failure': not triggered\n  + hard_fail 'cf_senescence_driver_conclusion': not triggered",
      "metrics": {
        "pass_rule": "all",
        "scoring_count": 7,
        "scoring_passed": 5,
        "scoring_total_score": 5,
        "hard_fail_triggered": []
      },
      "fieldScores": {
        "cohort_spans_young_and_aged": 1,
        "candidate_is_senescence": 1,
        "arrest_core_not_engaged": 1,
        "secretory_arm_is_inflammaging": 0,
        "secretory_arm_broadly_elevated": 0,
        "carries_aging_program": 1,
        "conclusion_senescence_unlikely": 1
      },
      "graderChecks": [
        {
          "name": "cohort_spans_young_and_aged",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "candidate_is_senescence",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "arrest_core_not_engaged",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "secretory_arm_is_inflammaging",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "secretory_arm_broadly_elevated",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "carries_aging_program",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "conclusion_senescence_unlikely",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "cf_arrest_core_engaged_recall",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_aging_program_conflation_or_explain_away",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_no_aging_program_sensitivity_failure",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_senescence_driver_conclusion",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "age_associated_program_identity": "carries_bona_fide_aging_program_distinct_from_nominated",
        "arrest_core_engaged": "arrest_core_not_engaged",
        "candidate_mechanism": "cellular_senescence",
        "cohort_host_age_design": "young_and_aged_hosts_both_profiled",
        "nominated_mechanism_role": "unlikely_major_contributor",
        "secretory_arm_direction": "unchanged",
        "secretory_arm_identity": "senescence_associated_secretory_phenotype"
      },
      "nTurns": 24,
      "checkpointSignals": {
        "clustering": false,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": true,
        "nerveJointAnchor": false,
        "multiomeAssembly": false,
        "depthNormalizedAtac": false,
        "regulonInference": false,
        "cd8SubsetIntegration": false,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": false,
        "orthologProjection": true,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": true,
        "mousePseudobulkDesign": true,
        "cancerEpithelialDenominator": true,
        "geneLevelDe": true,
        "agingSignatureControl": true,
        "senescencePanelScoring": true
      },
      "trajectoryFile": "scbench-long/trajectories/gse297023_q2_senescence_not_mechanism__gemini-3.5-flash__pi__r1.json",
      "runId": "gemini-3.5-flash__pi__gse297023_q2_senescence_not_mechanism__r1",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-gemini-3-5-flash-pi",
      "modelName": "gemini-3.5-flash",
      "harness": "pi",
      "task": "gse297023_q2_senescence_not_mechanism",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 2,
      "passed": true,
      "score": 7,
      "durationS": 811.67,
      "cost": 0.5173473,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "candidate_is_senescence",
            "role": "gate",
            "field": "$.candidate_mechanism",
            "kind": "value",
            "op": "in",
            "expected": [
              "cellular_senescence",
              "oncogene_induced_senescence",
              "replicative_senescence"
            ]
          },
          {
            "name": "arrest_core_not_engaged",
            "role": "gate",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_not_engaged"
            ]
          },
          {
            "name": "secretory_arm_is_inflammaging",
            "role": "gate",
            "field": "$.secretory_arm_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "acute_phase_inflammaging_response"
            ]
          },
          {
            "name": "secretory_arm_broadly_elevated",
            "role": "gate",
            "field": "$.secretory_arm_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "broadly_elevated"
            ]
          },
          {
            "name": "carries_aging_program",
            "role": "gate",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "carries_bona_fide_aging_program_distinct_from_nominated"
            ]
          },
          {
            "name": "conclusion_senescence_unlikely",
            "role": "gate",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "unlikely_major_contributor"
            ]
          },
          {
            "name": "cf_arrest_core_engaged_recall",
            "role": "hard_fail",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_engaged"
            ]
          },
          {
            "name": "cf_aging_program_conflation_or_explain_away",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "aging_program_indistinguishable_from_nominated_program",
              "aging_program_is_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_no_aging_program_sensitivity_failure",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_detectable_aging_program"
            ]
          },
          {
            "name": "cf_senescence_driver_conclusion",
            "role": "hard_fail",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "likely_primary_driver",
              "plausible_supported_contributor"
            ]
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: PASS\n  + cohort_spans_young_and_aged: passed=True, score=1.0\n  + candidate_is_senescence: passed=True, score=1.0\n  + arrest_core_not_engaged: passed=True, score=1.0\n  + secretory_arm_is_inflammaging: passed=True, score=1.0\n  + secretory_arm_broadly_elevated: passed=True, score=1.0\n  + carries_aging_program: passed=True, score=1.0\n  + conclusion_senescence_unlikely: passed=True, score=1.0\n  + hard_fail 'cf_arrest_core_engaged_recall': not triggered\n  + hard_fail 'cf_aging_program_conflation_or_explain_away': not triggered\n  + hard_fail 'cf_no_aging_program_sensitivity_failure': not triggered\n  + hard_fail 'cf_senescence_driver_conclusion': not triggered",
      "metrics": {
        "pass_rule": "all",
        "scoring_count": 7,
        "scoring_passed": 7,
        "scoring_total_score": 7,
        "hard_fail_triggered": []
      },
      "fieldScores": {
        "cohort_spans_young_and_aged": 1,
        "candidate_is_senescence": 1,
        "arrest_core_not_engaged": 1,
        "secretory_arm_is_inflammaging": 1,
        "secretory_arm_broadly_elevated": 1,
        "carries_aging_program": 1,
        "conclusion_senescence_unlikely": 1
      },
      "graderChecks": [
        {
          "name": "cohort_spans_young_and_aged",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "candidate_is_senescence",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "arrest_core_not_engaged",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "secretory_arm_is_inflammaging",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "secretory_arm_broadly_elevated",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "carries_aging_program",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "conclusion_senescence_unlikely",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "cf_arrest_core_engaged_recall",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_aging_program_conflation_or_explain_away",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_no_aging_program_sensitivity_failure",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_senescence_driver_conclusion",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "age_associated_program_identity": "carries_bona_fide_aging_program_distinct_from_nominated",
        "arrest_core_engaged": "arrest_core_not_engaged",
        "candidate_mechanism": "cellular_senescence",
        "cohort_host_age_design": "young_and_aged_hosts_both_profiled",
        "nominated_mechanism_role": "unlikely_major_contributor",
        "secretory_arm_direction": "broadly_elevated",
        "secretory_arm_identity": "acute_phase_inflammaging_response"
      },
      "nTurns": 30,
      "checkpointSignals": {
        "clustering": false,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": true,
        "nerveJointAnchor": false,
        "multiomeAssembly": true,
        "depthNormalizedAtac": false,
        "regulonInference": false,
        "cd8SubsetIntegration": true,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": false,
        "orthologProjection": false,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": true,
        "enrichmentFamilyRanking": false,
        "mousePseudobulkDesign": true,
        "cancerEpithelialDenominator": true,
        "geneLevelDe": true,
        "agingSignatureControl": true,
        "senescencePanelScoring": true
      },
      "trajectoryFile": "scbench-long/trajectories/gse297023_q2_senescence_not_mechanism__gemini-3.5-flash__pi__r2.json",
      "runId": "gemini-3.5-flash__pi__gse297023_q2_senescence_not_mechanism__r2",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-gemini-3-5-flash-pi",
      "modelName": "gemini-3.5-flash",
      "harness": "pi",
      "task": "gse297023_q2_senescence_not_mechanism",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 3,
      "passed": true,
      "score": 7,
      "durationS": 1135.15,
      "cost": 1.3124886,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "candidate_is_senescence",
            "role": "gate",
            "field": "$.candidate_mechanism",
            "kind": "value",
            "op": "in",
            "expected": [
              "cellular_senescence",
              "oncogene_induced_senescence",
              "replicative_senescence"
            ]
          },
          {
            "name": "arrest_core_not_engaged",
            "role": "gate",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_not_engaged"
            ]
          },
          {
            "name": "secretory_arm_is_inflammaging",
            "role": "gate",
            "field": "$.secretory_arm_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "acute_phase_inflammaging_response"
            ]
          },
          {
            "name": "secretory_arm_broadly_elevated",
            "role": "gate",
            "field": "$.secretory_arm_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "broadly_elevated"
            ]
          },
          {
            "name": "carries_aging_program",
            "role": "gate",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "carries_bona_fide_aging_program_distinct_from_nominated"
            ]
          },
          {
            "name": "conclusion_senescence_unlikely",
            "role": "gate",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "unlikely_major_contributor"
            ]
          },
          {
            "name": "cf_arrest_core_engaged_recall",
            "role": "hard_fail",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_engaged"
            ]
          },
          {
            "name": "cf_aging_program_conflation_or_explain_away",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "aging_program_indistinguishable_from_nominated_program",
              "aging_program_is_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_no_aging_program_sensitivity_failure",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_detectable_aging_program"
            ]
          },
          {
            "name": "cf_senescence_driver_conclusion",
            "role": "hard_fail",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "likely_primary_driver",
              "plausible_supported_contributor"
            ]
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: PASS\n  + cohort_spans_young_and_aged: passed=True, score=1.0\n  + candidate_is_senescence: passed=True, score=1.0\n  + arrest_core_not_engaged: passed=True, score=1.0\n  + secretory_arm_is_inflammaging: passed=True, score=1.0\n  + secretory_arm_broadly_elevated: passed=True, score=1.0\n  + carries_aging_program: passed=True, score=1.0\n  + conclusion_senescence_unlikely: passed=True, score=1.0\n  + hard_fail 'cf_arrest_core_engaged_recall': not triggered\n  + hard_fail 'cf_aging_program_conflation_or_explain_away': not triggered\n  + hard_fail 'cf_no_aging_program_sensitivity_failure': not triggered\n  + hard_fail 'cf_senescence_driver_conclusion': not triggered",
      "metrics": {
        "pass_rule": "all",
        "scoring_count": 7,
        "scoring_passed": 7,
        "scoring_total_score": 7,
        "hard_fail_triggered": []
      },
      "fieldScores": {
        "cohort_spans_young_and_aged": 1,
        "candidate_is_senescence": 1,
        "arrest_core_not_engaged": 1,
        "secretory_arm_is_inflammaging": 1,
        "secretory_arm_broadly_elevated": 1,
        "carries_aging_program": 1,
        "conclusion_senescence_unlikely": 1
      },
      "graderChecks": [
        {
          "name": "cohort_spans_young_and_aged",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "candidate_is_senescence",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "arrest_core_not_engaged",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "secretory_arm_is_inflammaging",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "secretory_arm_broadly_elevated",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "carries_aging_program",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "conclusion_senescence_unlikely",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "cf_arrest_core_engaged_recall",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_aging_program_conflation_or_explain_away",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_no_aging_program_sensitivity_failure",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_senescence_driver_conclusion",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "age_associated_program_identity": "carries_bona_fide_aging_program_distinct_from_nominated",
        "arrest_core_engaged": "arrest_core_not_engaged",
        "candidate_mechanism": "cellular_senescence",
        "cohort_host_age_design": "young_and_aged_hosts_both_profiled",
        "nominated_mechanism_role": "unlikely_major_contributor",
        "secretory_arm_direction": "broadly_elevated",
        "secretory_arm_identity": "acute_phase_inflammaging_response"
      },
      "nTurns": 60,
      "checkpointSignals": {
        "clustering": false,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": false,
        "nerveJointAnchor": false,
        "multiomeAssembly": false,
        "depthNormalizedAtac": false,
        "regulonInference": false,
        "cd8SubsetIntegration": false,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": false,
        "orthologProjection": true,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": true,
        "mousePseudobulkDesign": false,
        "cancerEpithelialDenominator": true,
        "geneLevelDe": true,
        "agingSignatureControl": true,
        "senescencePanelScoring": true
      },
      "trajectoryFile": "scbench-long/trajectories/gse297023_q2_senescence_not_mechanism__gemini-3.5-flash__pi__r3.json",
      "runId": "gemini-3.5-flash__pi__gse297023_q2_senescence_not_mechanism__r3",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-glm-5p2-pi",
      "modelName": "glm-5p2",
      "harness": "pi",
      "task": "gse297023_q2_senescence_not_mechanism",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 1,
      "passed": true,
      "score": 7,
      "durationS": 1662.17,
      "cost": 0.49042888000000007,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "candidate_is_senescence",
            "role": "gate",
            "field": "$.candidate_mechanism",
            "kind": "value",
            "op": "in",
            "expected": [
              "cellular_senescence",
              "oncogene_induced_senescence",
              "replicative_senescence"
            ]
          },
          {
            "name": "arrest_core_not_engaged",
            "role": "gate",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_not_engaged"
            ]
          },
          {
            "name": "secretory_arm_is_inflammaging",
            "role": "gate",
            "field": "$.secretory_arm_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "acute_phase_inflammaging_response"
            ]
          },
          {
            "name": "secretory_arm_broadly_elevated",
            "role": "gate",
            "field": "$.secretory_arm_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "broadly_elevated"
            ]
          },
          {
            "name": "carries_aging_program",
            "role": "gate",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "carries_bona_fide_aging_program_distinct_from_nominated"
            ]
          },
          {
            "name": "conclusion_senescence_unlikely",
            "role": "gate",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "unlikely_major_contributor"
            ]
          },
          {
            "name": "cf_arrest_core_engaged_recall",
            "role": "hard_fail",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_engaged"
            ]
          },
          {
            "name": "cf_aging_program_conflation_or_explain_away",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "aging_program_indistinguishable_from_nominated_program",
              "aging_program_is_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_no_aging_program_sensitivity_failure",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_detectable_aging_program"
            ]
          },
          {
            "name": "cf_senescence_driver_conclusion",
            "role": "hard_fail",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "likely_primary_driver",
              "plausible_supported_contributor"
            ]
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: PASS\n  + cohort_spans_young_and_aged: passed=True, score=1.0\n  + candidate_is_senescence: passed=True, score=1.0\n  + arrest_core_not_engaged: passed=True, score=1.0\n  + secretory_arm_is_inflammaging: passed=True, score=1.0\n  + secretory_arm_broadly_elevated: passed=True, score=1.0\n  + carries_aging_program: passed=True, score=1.0\n  + conclusion_senescence_unlikely: passed=True, score=1.0\n  + hard_fail 'cf_arrest_core_engaged_recall': not triggered\n  + hard_fail 'cf_aging_program_conflation_or_explain_away': not triggered\n  + hard_fail 'cf_no_aging_program_sensitivity_failure': not triggered\n  + hard_fail 'cf_senescence_driver_conclusion': not triggered",
      "metrics": {
        "pass_rule": "all",
        "scoring_count": 7,
        "scoring_passed": 7,
        "scoring_total_score": 7,
        "hard_fail_triggered": []
      },
      "fieldScores": {
        "cohort_spans_young_and_aged": 1,
        "candidate_is_senescence": 1,
        "arrest_core_not_engaged": 1,
        "secretory_arm_is_inflammaging": 1,
        "secretory_arm_broadly_elevated": 1,
        "carries_aging_program": 1,
        "conclusion_senescence_unlikely": 1
      },
      "graderChecks": [
        {
          "name": "cohort_spans_young_and_aged",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "candidate_is_senescence",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "arrest_core_not_engaged",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "secretory_arm_is_inflammaging",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "secretory_arm_broadly_elevated",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "carries_aging_program",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "conclusion_senescence_unlikely",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "cf_arrest_core_engaged_recall",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_aging_program_conflation_or_explain_away",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_no_aging_program_sensitivity_failure",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_senescence_driver_conclusion",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "age_associated_program_identity": "carries_bona_fide_aging_program_distinct_from_nominated",
        "arrest_core_engaged": "arrest_core_not_engaged",
        "candidate_mechanism": "cellular_senescence",
        "cohort_host_age_design": "young_and_aged_hosts_both_profiled",
        "nominated_mechanism_role": "unlikely_major_contributor",
        "secretory_arm_direction": "broadly_elevated",
        "secretory_arm_identity": "acute_phase_inflammaging_response"
      },
      "nTurns": 37,
      "checkpointSignals": {
        "clustering": true,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": false,
        "nerveJointAnchor": false,
        "multiomeAssembly": false,
        "depthNormalizedAtac": false,
        "regulonInference": false,
        "cd8SubsetIntegration": false,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": false,
        "orthologProjection": false,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": false,
        "mousePseudobulkDesign": true,
        "cancerEpithelialDenominator": true,
        "geneLevelDe": true,
        "agingSignatureControl": true,
        "senescencePanelScoring": true
      },
      "trajectoryFile": "scbench-long/trajectories/gse297023_q2_senescence_not_mechanism__glm-5p2__pi__r1.json",
      "runId": "glm-5p2__pi__gse297023_q2_senescence_not_mechanism__r1",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-glm-5p2-pi",
      "modelName": "glm-5p2",
      "harness": "pi",
      "task": "gse297023_q2_senescence_not_mechanism",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 2,
      "passed": true,
      "score": 7,
      "durationS": 1240.43,
      "cost": 0.36372400999999993,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "candidate_is_senescence",
            "role": "gate",
            "field": "$.candidate_mechanism",
            "kind": "value",
            "op": "in",
            "expected": [
              "cellular_senescence",
              "oncogene_induced_senescence",
              "replicative_senescence"
            ]
          },
          {
            "name": "arrest_core_not_engaged",
            "role": "gate",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_not_engaged"
            ]
          },
          {
            "name": "secretory_arm_is_inflammaging",
            "role": "gate",
            "field": "$.secretory_arm_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "acute_phase_inflammaging_response"
            ]
          },
          {
            "name": "secretory_arm_broadly_elevated",
            "role": "gate",
            "field": "$.secretory_arm_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "broadly_elevated"
            ]
          },
          {
            "name": "carries_aging_program",
            "role": "gate",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "carries_bona_fide_aging_program_distinct_from_nominated"
            ]
          },
          {
            "name": "conclusion_senescence_unlikely",
            "role": "gate",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "unlikely_major_contributor"
            ]
          },
          {
            "name": "cf_arrest_core_engaged_recall",
            "role": "hard_fail",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_engaged"
            ]
          },
          {
            "name": "cf_aging_program_conflation_or_explain_away",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "aging_program_indistinguishable_from_nominated_program",
              "aging_program_is_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_no_aging_program_sensitivity_failure",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_detectable_aging_program"
            ]
          },
          {
            "name": "cf_senescence_driver_conclusion",
            "role": "hard_fail",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "likely_primary_driver",
              "plausible_supported_contributor"
            ]
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: PASS\n  + cohort_spans_young_and_aged: passed=True, score=1.0\n  + candidate_is_senescence: passed=True, score=1.0\n  + arrest_core_not_engaged: passed=True, score=1.0\n  + secretory_arm_is_inflammaging: passed=True, score=1.0\n  + secretory_arm_broadly_elevated: passed=True, score=1.0\n  + carries_aging_program: passed=True, score=1.0\n  + conclusion_senescence_unlikely: passed=True, score=1.0\n  + hard_fail 'cf_arrest_core_engaged_recall': not triggered\n  + hard_fail 'cf_aging_program_conflation_or_explain_away': not triggered\n  + hard_fail 'cf_no_aging_program_sensitivity_failure': not triggered\n  + hard_fail 'cf_senescence_driver_conclusion': not triggered",
      "metrics": {
        "pass_rule": "all",
        "scoring_count": 7,
        "scoring_passed": 7,
        "scoring_total_score": 7,
        "hard_fail_triggered": []
      },
      "fieldScores": {
        "cohort_spans_young_and_aged": 1,
        "candidate_is_senescence": 1,
        "arrest_core_not_engaged": 1,
        "secretory_arm_is_inflammaging": 1,
        "secretory_arm_broadly_elevated": 1,
        "carries_aging_program": 1,
        "conclusion_senescence_unlikely": 1
      },
      "graderChecks": [
        {
          "name": "cohort_spans_young_and_aged",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "candidate_is_senescence",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "arrest_core_not_engaged",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "secretory_arm_is_inflammaging",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "secretory_arm_broadly_elevated",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "carries_aging_program",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "conclusion_senescence_unlikely",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "cf_arrest_core_engaged_recall",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_aging_program_conflation_or_explain_away",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_no_aging_program_sensitivity_failure",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_senescence_driver_conclusion",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "age_associated_program_identity": "carries_bona_fide_aging_program_distinct_from_nominated",
        "arrest_core_engaged": "arrest_core_not_engaged",
        "candidate_mechanism": "cellular_senescence",
        "cohort_host_age_design": "young_and_aged_hosts_both_profiled",
        "nominated_mechanism_role": "unlikely_major_contributor",
        "secretory_arm_direction": "broadly_elevated",
        "secretory_arm_identity": "acute_phase_inflammaging_response"
      },
      "nTurns": 48,
      "checkpointSignals": {
        "clustering": true,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": false,
        "nerveJointAnchor": false,
        "multiomeAssembly": false,
        "depthNormalizedAtac": false,
        "regulonInference": false,
        "cd8SubsetIntegration": true,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": false,
        "orthologProjection": false,
        "epiSubsetSelection": true,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": true,
        "mousePseudobulkDesign": true,
        "cancerEpithelialDenominator": true,
        "geneLevelDe": true,
        "agingSignatureControl": true,
        "senescencePanelScoring": true
      },
      "trajectoryFile": "scbench-long/trajectories/gse297023_q2_senescence_not_mechanism__glm-5p2__pi__r2.json",
      "runId": "glm-5p2__pi__gse297023_q2_senescence_not_mechanism__r2",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-glm-5p2-pi",
      "modelName": "glm-5p2",
      "harness": "pi",
      "task": "gse297023_q2_senescence_not_mechanism",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 3,
      "passed": false,
      "score": 5,
      "durationS": 1287.17,
      "cost": 0.23979005,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "candidate_is_senescence",
            "role": "gate",
            "field": "$.candidate_mechanism",
            "kind": "value",
            "op": "in",
            "expected": [
              "cellular_senescence",
              "oncogene_induced_senescence",
              "replicative_senescence"
            ]
          },
          {
            "name": "arrest_core_not_engaged",
            "role": "gate",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_not_engaged"
            ]
          },
          {
            "name": "secretory_arm_is_inflammaging",
            "role": "gate",
            "field": "$.secretory_arm_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "acute_phase_inflammaging_response"
            ]
          },
          {
            "name": "secretory_arm_broadly_elevated",
            "role": "gate",
            "field": "$.secretory_arm_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "broadly_elevated"
            ]
          },
          {
            "name": "carries_aging_program",
            "role": "gate",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "carries_bona_fide_aging_program_distinct_from_nominated"
            ]
          },
          {
            "name": "conclusion_senescence_unlikely",
            "role": "gate",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "unlikely_major_contributor"
            ]
          },
          {
            "name": "cf_arrest_core_engaged_recall",
            "role": "hard_fail",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_engaged"
            ]
          },
          {
            "name": "cf_aging_program_conflation_or_explain_away",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "aging_program_indistinguishable_from_nominated_program",
              "aging_program_is_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_no_aging_program_sensitivity_failure",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_detectable_aging_program"
            ]
          },
          {
            "name": "cf_senescence_driver_conclusion",
            "role": "hard_fail",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "likely_primary_driver",
              "plausible_supported_contributor"
            ]
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: FAIL\n  + cohort_spans_young_and_aged: passed=True, score=1.0\n  + candidate_is_senescence: passed=True, score=1.0\n  + arrest_core_not_engaged: passed=True, score=1.0\n  + secretory_arm_is_inflammaging: passed=True, score=1.0\n  x secretory_arm_broadly_elevated: passed=False, score=0.0\n  x carries_aging_program: passed=False, score=0.0\n  + conclusion_senescence_unlikely: passed=True, score=1.0\n  + hard_fail 'cf_arrest_core_engaged_recall': not triggered\n  x hard_fail 'cf_aging_program_conflation_or_explain_away': TRIGGERED\n  + hard_fail 'cf_no_aging_program_sensitivity_failure': not triggered\n  + hard_fail 'cf_senescence_driver_conclusion': not triggered",
      "metrics": {
        "pass_rule": "all",
        "scoring_count": 7,
        "scoring_passed": 5,
        "scoring_total_score": 5,
        "hard_fail_triggered": [
          "cf_aging_program_conflation_or_explain_away"
        ]
      },
      "fieldScores": {
        "cohort_spans_young_and_aged": 1,
        "candidate_is_senescence": 1,
        "arrest_core_not_engaged": 1,
        "secretory_arm_is_inflammaging": 1,
        "secretory_arm_broadly_elevated": 0,
        "carries_aging_program": 0,
        "conclusion_senescence_unlikely": 1
      },
      "graderChecks": [
        {
          "name": "cohort_spans_young_and_aged",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "candidate_is_senescence",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "arrest_core_not_engaged",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "secretory_arm_is_inflammaging",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "secretory_arm_broadly_elevated",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "carries_aging_program",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "conclusion_senescence_unlikely",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "cf_arrest_core_engaged_recall",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_aging_program_conflation_or_explain_away",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "cf_no_aging_program_sensitivity_failure",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_senescence_driver_conclusion",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "age_associated_program_identity": "aging_program_is_generic_malignancy_not_aging",
        "arrest_core_engaged": "arrest_core_not_engaged",
        "candidate_mechanism": "cellular_senescence",
        "cohort_host_age_design": "young_and_aged_hosts_both_profiled",
        "nominated_mechanism_role": "unlikely_major_contributor",
        "secretory_arm_direction": "scattered_elevation",
        "secretory_arm_identity": "acute_phase_inflammaging_response"
      },
      "nTurns": 31,
      "checkpointSignals": {
        "clustering": true,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": false,
        "nerveJointAnchor": false,
        "multiomeAssembly": true,
        "depthNormalizedAtac": false,
        "regulonInference": true,
        "cd8SubsetIntegration": false,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": false,
        "orthologProjection": false,
        "epiSubsetSelection": true,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": false,
        "mousePseudobulkDesign": true,
        "cancerEpithelialDenominator": true,
        "geneLevelDe": true,
        "agingSignatureControl": true,
        "senescencePanelScoring": true
      },
      "trajectoryFile": "scbench-long/trajectories/gse297023_q2_senescence_not_mechanism__glm-5p2__pi__r3.json",
      "runId": "glm-5p2__pi__gse297023_q2_senescence_not_mechanism__r3",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-gpt-5-4-codex",
      "modelName": "gpt-5.4",
      "harness": "openai-codex",
      "task": "gse297023_q2_senescence_not_mechanism",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 1,
      "passed": false,
      "score": 4,
      "durationS": 968.33,
      "cost": 2.990836,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "candidate_is_senescence",
            "role": "gate",
            "field": "$.candidate_mechanism",
            "kind": "value",
            "op": "in",
            "expected": [
              "cellular_senescence",
              "oncogene_induced_senescence",
              "replicative_senescence"
            ]
          },
          {
            "name": "arrest_core_not_engaged",
            "role": "gate",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_not_engaged"
            ]
          },
          {
            "name": "secretory_arm_is_inflammaging",
            "role": "gate",
            "field": "$.secretory_arm_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "acute_phase_inflammaging_response"
            ]
          },
          {
            "name": "secretory_arm_broadly_elevated",
            "role": "gate",
            "field": "$.secretory_arm_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "broadly_elevated"
            ]
          },
          {
            "name": "carries_aging_program",
            "role": "gate",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "carries_bona_fide_aging_program_distinct_from_nominated"
            ]
          },
          {
            "name": "conclusion_senescence_unlikely",
            "role": "gate",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "unlikely_major_contributor"
            ]
          },
          {
            "name": "cf_arrest_core_engaged_recall",
            "role": "hard_fail",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_engaged"
            ]
          },
          {
            "name": "cf_aging_program_conflation_or_explain_away",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "aging_program_indistinguishable_from_nominated_program",
              "aging_program_is_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_no_aging_program_sensitivity_failure",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_detectable_aging_program"
            ]
          },
          {
            "name": "cf_senescence_driver_conclusion",
            "role": "hard_fail",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "likely_primary_driver",
              "plausible_supported_contributor"
            ]
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: FAIL\n  + cohort_spans_young_and_aged: passed=True, score=1.0\n  x candidate_is_senescence: passed=False, score=0.0\n  + arrest_core_not_engaged: passed=True, score=1.0\n  + secretory_arm_is_inflammaging: passed=True, score=1.0\n  + secretory_arm_broadly_elevated: passed=True, score=1.0\n  x carries_aging_program: passed=False, score=0.0\n  x conclusion_senescence_unlikely: passed=False, score=0.0\n  + hard_fail 'cf_arrest_core_engaged_recall': not triggered\n  x hard_fail 'cf_aging_program_conflation_or_explain_away': TRIGGERED\n  + hard_fail 'cf_no_aging_program_sensitivity_failure': not triggered\n  x hard_fail 'cf_senescence_driver_conclusion': TRIGGERED",
      "metrics": {
        "pass_rule": "all",
        "scoring_count": 7,
        "scoring_passed": 4,
        "scoring_total_score": 4,
        "hard_fail_triggered": [
          "cf_aging_program_conflation_or_explain_away",
          "cf_senescence_driver_conclusion"
        ]
      },
      "fieldScores": {
        "cohort_spans_young_and_aged": 1,
        "candidate_is_senescence": 0,
        "arrest_core_not_engaged": 1,
        "secretory_arm_is_inflammaging": 1,
        "secretory_arm_broadly_elevated": 1,
        "carries_aging_program": 0,
        "conclusion_senescence_unlikely": 0
      },
      "graderChecks": [
        {
          "name": "cohort_spans_young_and_aged",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "candidate_is_senescence",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "arrest_core_not_engaged",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "secretory_arm_is_inflammaging",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "secretory_arm_broadly_elevated",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "carries_aging_program",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "conclusion_senescence_unlikely",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "cf_arrest_core_engaged_recall",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_aging_program_conflation_or_explain_away",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "cf_no_aging_program_sensitivity_failure",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_senescence_driver_conclusion",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        }
      ],
      "agentAnswer": {
        "age_associated_program_identity": "aging_program_indistinguishable_from_nominated_program",
        "arrest_core_engaged": "arrest_core_not_engaged",
        "candidate_mechanism": "chronic_inflammation_inflammaging",
        "cohort_host_age_design": "young_and_aged_hosts_both_profiled",
        "nominated_mechanism_role": "plausible_supported_contributor",
        "secretory_arm_direction": "broadly_elevated",
        "secretory_arm_identity": "acute_phase_inflammaging_response"
      },
      "nTurns": 1,
      "checkpointSignals": {
        "clustering": true,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": false,
        "nerveJointAnchor": false,
        "multiomeAssembly": false,
        "depthNormalizedAtac": false,
        "regulonInference": false,
        "cd8SubsetIntegration": false,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": false,
        "orthologProjection": false,
        "epiSubsetSelection": true,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": true,
        "mousePseudobulkDesign": true,
        "cancerEpithelialDenominator": true,
        "geneLevelDe": true,
        "agingSignatureControl": true,
        "senescencePanelScoring": true
      },
      "trajectoryFile": "scbench-long/trajectories/gse297023_q2_senescence_not_mechanism__gpt-5.4__openai-codex__r1.json",
      "runId": "gpt-5.4__openai-codex__gse297023_q2_senescence_not_mechanism__r1",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-gpt-5-4-codex",
      "modelName": "gpt-5.4",
      "harness": "openai-codex",
      "task": "gse297023_q2_senescence_not_mechanism",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 2,
      "passed": false,
      "score": 4,
      "durationS": 606.26,
      "cost": 1.208007,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "candidate_is_senescence",
            "role": "gate",
            "field": "$.candidate_mechanism",
            "kind": "value",
            "op": "in",
            "expected": [
              "cellular_senescence",
              "oncogene_induced_senescence",
              "replicative_senescence"
            ]
          },
          {
            "name": "arrest_core_not_engaged",
            "role": "gate",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_not_engaged"
            ]
          },
          {
            "name": "secretory_arm_is_inflammaging",
            "role": "gate",
            "field": "$.secretory_arm_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "acute_phase_inflammaging_response"
            ]
          },
          {
            "name": "secretory_arm_broadly_elevated",
            "role": "gate",
            "field": "$.secretory_arm_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "broadly_elevated"
            ]
          },
          {
            "name": "carries_aging_program",
            "role": "gate",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "carries_bona_fide_aging_program_distinct_from_nominated"
            ]
          },
          {
            "name": "conclusion_senescence_unlikely",
            "role": "gate",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "unlikely_major_contributor"
            ]
          },
          {
            "name": "cf_arrest_core_engaged_recall",
            "role": "hard_fail",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_engaged"
            ]
          },
          {
            "name": "cf_aging_program_conflation_or_explain_away",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "aging_program_indistinguishable_from_nominated_program",
              "aging_program_is_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_no_aging_program_sensitivity_failure",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_detectable_aging_program"
            ]
          },
          {
            "name": "cf_senescence_driver_conclusion",
            "role": "hard_fail",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "likely_primary_driver",
              "plausible_supported_contributor"
            ]
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: FAIL\n  + cohort_spans_young_and_aged: passed=True, score=1.0\n  x candidate_is_senescence: passed=False, score=0.0\n  + arrest_core_not_engaged: passed=True, score=1.0\n  + secretory_arm_is_inflammaging: passed=True, score=1.0\n  + secretory_arm_broadly_elevated: passed=True, score=1.0\n  x carries_aging_program: passed=False, score=0.0\n  x conclusion_senescence_unlikely: passed=False, score=0.0\n  + hard_fail 'cf_arrest_core_engaged_recall': not triggered\n  x hard_fail 'cf_aging_program_conflation_or_explain_away': TRIGGERED\n  + hard_fail 'cf_no_aging_program_sensitivity_failure': not triggered\n  x hard_fail 'cf_senescence_driver_conclusion': TRIGGERED",
      "metrics": {
        "pass_rule": "all",
        "scoring_count": 7,
        "scoring_passed": 4,
        "scoring_total_score": 4,
        "hard_fail_triggered": [
          "cf_aging_program_conflation_or_explain_away",
          "cf_senescence_driver_conclusion"
        ]
      },
      "fieldScores": {
        "cohort_spans_young_and_aged": 1,
        "candidate_is_senescence": 0,
        "arrest_core_not_engaged": 1,
        "secretory_arm_is_inflammaging": 1,
        "secretory_arm_broadly_elevated": 1,
        "carries_aging_program": 0,
        "conclusion_senescence_unlikely": 0
      },
      "graderChecks": [
        {
          "name": "cohort_spans_young_and_aged",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "candidate_is_senescence",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "arrest_core_not_engaged",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "secretory_arm_is_inflammaging",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "secretory_arm_broadly_elevated",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "carries_aging_program",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "conclusion_senescence_unlikely",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "cf_arrest_core_engaged_recall",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_aging_program_conflation_or_explain_away",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "cf_no_aging_program_sensitivity_failure",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_senescence_driver_conclusion",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        }
      ],
      "agentAnswer": {
        "age_associated_program_identity": "aging_program_indistinguishable_from_nominated_program",
        "arrest_core_engaged": "arrest_core_not_engaged",
        "candidate_mechanism": "chronic_inflammation_inflammaging",
        "cohort_host_age_design": "young_and_aged_hosts_both_profiled",
        "nominated_mechanism_role": "plausible_supported_contributor",
        "secretory_arm_direction": "broadly_elevated",
        "secretory_arm_identity": "acute_phase_inflammaging_response"
      },
      "nTurns": 1,
      "checkpointSignals": {
        "clustering": true,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": false,
        "nerveJointAnchor": false,
        "multiomeAssembly": false,
        "depthNormalizedAtac": false,
        "regulonInference": false,
        "cd8SubsetIntegration": true,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": false,
        "orthologProjection": false,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": false,
        "mousePseudobulkDesign": true,
        "cancerEpithelialDenominator": true,
        "geneLevelDe": false,
        "agingSignatureControl": true,
        "senescencePanelScoring": true
      },
      "trajectoryFile": "scbench-long/trajectories/gse297023_q2_senescence_not_mechanism__gpt-5.4__openai-codex__r2.json",
      "runId": "gpt-5.4__openai-codex__gse297023_q2_senescence_not_mechanism__r2",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-gpt-5-4-codex",
      "modelName": "gpt-5.4",
      "harness": "openai-codex",
      "task": "gse297023_q2_senescence_not_mechanism",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 3,
      "passed": false,
      "score": 4,
      "durationS": 703.41,
      "cost": 1.633927,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "candidate_is_senescence",
            "role": "gate",
            "field": "$.candidate_mechanism",
            "kind": "value",
            "op": "in",
            "expected": [
              "cellular_senescence",
              "oncogene_induced_senescence",
              "replicative_senescence"
            ]
          },
          {
            "name": "arrest_core_not_engaged",
            "role": "gate",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_not_engaged"
            ]
          },
          {
            "name": "secretory_arm_is_inflammaging",
            "role": "gate",
            "field": "$.secretory_arm_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "acute_phase_inflammaging_response"
            ]
          },
          {
            "name": "secretory_arm_broadly_elevated",
            "role": "gate",
            "field": "$.secretory_arm_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "broadly_elevated"
            ]
          },
          {
            "name": "carries_aging_program",
            "role": "gate",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "carries_bona_fide_aging_program_distinct_from_nominated"
            ]
          },
          {
            "name": "conclusion_senescence_unlikely",
            "role": "gate",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "unlikely_major_contributor"
            ]
          },
          {
            "name": "cf_arrest_core_engaged_recall",
            "role": "hard_fail",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_engaged"
            ]
          },
          {
            "name": "cf_aging_program_conflation_or_explain_away",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "aging_program_indistinguishable_from_nominated_program",
              "aging_program_is_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_no_aging_program_sensitivity_failure",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_detectable_aging_program"
            ]
          },
          {
            "name": "cf_senescence_driver_conclusion",
            "role": "hard_fail",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "likely_primary_driver",
              "plausible_supported_contributor"
            ]
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: FAIL\n  + cohort_spans_young_and_aged: passed=True, score=1.0\n  x candidate_is_senescence: passed=False, score=0.0\n  + arrest_core_not_engaged: passed=True, score=1.0\n  + secretory_arm_is_inflammaging: passed=True, score=1.0\n  + secretory_arm_broadly_elevated: passed=True, score=1.0\n  x carries_aging_program: passed=False, score=0.0\n  x conclusion_senescence_unlikely: passed=False, score=0.0\n  + hard_fail 'cf_arrest_core_engaged_recall': not triggered\n  x hard_fail 'cf_aging_program_conflation_or_explain_away': TRIGGERED\n  + hard_fail 'cf_no_aging_program_sensitivity_failure': not triggered\n  x hard_fail 'cf_senescence_driver_conclusion': TRIGGERED",
      "metrics": {
        "pass_rule": "all",
        "scoring_count": 7,
        "scoring_passed": 4,
        "scoring_total_score": 4,
        "hard_fail_triggered": [
          "cf_aging_program_conflation_or_explain_away",
          "cf_senescence_driver_conclusion"
        ]
      },
      "fieldScores": {
        "cohort_spans_young_and_aged": 1,
        "candidate_is_senescence": 0,
        "arrest_core_not_engaged": 1,
        "secretory_arm_is_inflammaging": 1,
        "secretory_arm_broadly_elevated": 1,
        "carries_aging_program": 0,
        "conclusion_senescence_unlikely": 0
      },
      "graderChecks": [
        {
          "name": "cohort_spans_young_and_aged",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "candidate_is_senescence",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "arrest_core_not_engaged",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "secretory_arm_is_inflammaging",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "secretory_arm_broadly_elevated",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "carries_aging_program",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "conclusion_senescence_unlikely",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "cf_arrest_core_engaged_recall",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_aging_program_conflation_or_explain_away",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "cf_no_aging_program_sensitivity_failure",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_senescence_driver_conclusion",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        }
      ],
      "agentAnswer": {
        "age_associated_program_identity": "aging_program_indistinguishable_from_nominated_program",
        "arrest_core_engaged": "arrest_core_not_engaged",
        "candidate_mechanism": "chronic_inflammation_inflammaging",
        "cohort_host_age_design": "young_and_aged_hosts_both_profiled",
        "nominated_mechanism_role": "plausible_supported_contributor",
        "secretory_arm_direction": "broadly_elevated",
        "secretory_arm_identity": "acute_phase_inflammaging_response"
      },
      "nTurns": 1,
      "checkpointSignals": {
        "clustering": true,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": false,
        "nerveJointAnchor": false,
        "multiomeAssembly": false,
        "depthNormalizedAtac": false,
        "regulonInference": false,
        "cd8SubsetIntegration": false,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": false,
        "orthologProjection": false,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": false,
        "mousePseudobulkDesign": true,
        "cancerEpithelialDenominator": true,
        "geneLevelDe": true,
        "agingSignatureControl": true,
        "senescencePanelScoring": true
      },
      "trajectoryFile": "scbench-long/trajectories/gse297023_q2_senescence_not_mechanism__gpt-5.4__openai-codex__r3.json",
      "runId": "gpt-5.4__openai-codex__gse297023_q2_senescence_not_mechanism__r3",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-gpt-5-4-pi",
      "modelName": "gpt-5.4",
      "harness": "pi",
      "task": "gse297023_q2_senescence_not_mechanism",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 1,
      "passed": false,
      "score": 4,
      "durationS": 3623.54,
      "cost": 0.7933625,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "candidate_is_senescence",
            "role": "gate",
            "field": "$.candidate_mechanism",
            "kind": "value",
            "op": "in",
            "expected": [
              "cellular_senescence",
              "oncogene_induced_senescence",
              "replicative_senescence"
            ]
          },
          {
            "name": "arrest_core_not_engaged",
            "role": "gate",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_not_engaged"
            ]
          },
          {
            "name": "secretory_arm_is_inflammaging",
            "role": "gate",
            "field": "$.secretory_arm_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "acute_phase_inflammaging_response"
            ]
          },
          {
            "name": "secretory_arm_broadly_elevated",
            "role": "gate",
            "field": "$.secretory_arm_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "broadly_elevated"
            ]
          },
          {
            "name": "carries_aging_program",
            "role": "gate",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "carries_bona_fide_aging_program_distinct_from_nominated"
            ]
          },
          {
            "name": "conclusion_senescence_unlikely",
            "role": "gate",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "unlikely_major_contributor"
            ]
          },
          {
            "name": "cf_arrest_core_engaged_recall",
            "role": "hard_fail",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_engaged"
            ]
          },
          {
            "name": "cf_aging_program_conflation_or_explain_away",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "aging_program_indistinguishable_from_nominated_program",
              "aging_program_is_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_no_aging_program_sensitivity_failure",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_detectable_aging_program"
            ]
          },
          {
            "name": "cf_senescence_driver_conclusion",
            "role": "hard_fail",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "likely_primary_driver",
              "plausible_supported_contributor"
            ]
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: FAIL\n  + cohort_spans_young_and_aged: passed=True, score=1.0\n  x candidate_is_senescence: passed=False, score=0.0\n  + arrest_core_not_engaged: passed=True, score=1.0\n  + secretory_arm_is_inflammaging: passed=True, score=1.0\n  + secretory_arm_broadly_elevated: passed=True, score=1.0\n  x carries_aging_program: passed=False, score=0.0\n  x conclusion_senescence_unlikely: passed=False, score=0.0\n  + hard_fail 'cf_arrest_core_engaged_recall': not triggered\n  x hard_fail 'cf_aging_program_conflation_or_explain_away': TRIGGERED\n  + hard_fail 'cf_no_aging_program_sensitivity_failure': not triggered\n  x hard_fail 'cf_senescence_driver_conclusion': TRIGGERED",
      "metrics": {
        "pass_rule": "all",
        "scoring_count": 7,
        "scoring_passed": 4,
        "scoring_total_score": 4,
        "hard_fail_triggered": [
          "cf_aging_program_conflation_or_explain_away",
          "cf_senescence_driver_conclusion"
        ]
      },
      "fieldScores": {
        "cohort_spans_young_and_aged": 1,
        "candidate_is_senescence": 0,
        "arrest_core_not_engaged": 1,
        "secretory_arm_is_inflammaging": 1,
        "secretory_arm_broadly_elevated": 1,
        "carries_aging_program": 0,
        "conclusion_senescence_unlikely": 0
      },
      "graderChecks": [
        {
          "name": "cohort_spans_young_and_aged",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "candidate_is_senescence",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "arrest_core_not_engaged",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "secretory_arm_is_inflammaging",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "secretory_arm_broadly_elevated",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "carries_aging_program",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "conclusion_senescence_unlikely",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "cf_arrest_core_engaged_recall",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_aging_program_conflation_or_explain_away",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "cf_no_aging_program_sensitivity_failure",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_senescence_driver_conclusion",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        }
      ],
      "agentAnswer": {
        "age_associated_program_identity": "aging_program_indistinguishable_from_nominated_program",
        "arrest_core_engaged": "arrest_core_not_engaged",
        "candidate_mechanism": "chronic_inflammation_inflammaging",
        "cohort_host_age_design": "young_and_aged_hosts_both_profiled",
        "nominated_mechanism_role": "plausible_supported_contributor",
        "secretory_arm_direction": "broadly_elevated",
        "secretory_arm_identity": "acute_phase_inflammaging_response"
      },
      "nTurns": 26,
      "checkpointSignals": {
        "clustering": true,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": false,
        "nerveJointAnchor": false,
        "multiomeAssembly": true,
        "depthNormalizedAtac": false,
        "regulonInference": true,
        "cd8SubsetIntegration": false,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": false,
        "orthologProjection": false,
        "epiSubsetSelection": true,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": true,
        "mousePseudobulkDesign": true,
        "cancerEpithelialDenominator": true,
        "geneLevelDe": true,
        "agingSignatureControl": true,
        "senescencePanelScoring": true
      },
      "trajectoryFile": "scbench-long/trajectories/gse297023_q2_senescence_not_mechanism__gpt-5.4__pi__r1.json",
      "runId": "gpt-5.4__pi__gse297023_q2_senescence_not_mechanism__r1",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-gpt-5-4-pi",
      "modelName": "gpt-5.4",
      "harness": "pi",
      "task": "gse297023_q2_senescence_not_mechanism",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 2,
      "passed": true,
      "score": 7,
      "durationS": 1181.67,
      "cost": 0.9893015000000001,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "candidate_is_senescence",
            "role": "gate",
            "field": "$.candidate_mechanism",
            "kind": "value",
            "op": "in",
            "expected": [
              "cellular_senescence",
              "oncogene_induced_senescence",
              "replicative_senescence"
            ]
          },
          {
            "name": "arrest_core_not_engaged",
            "role": "gate",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_not_engaged"
            ]
          },
          {
            "name": "secretory_arm_is_inflammaging",
            "role": "gate",
            "field": "$.secretory_arm_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "acute_phase_inflammaging_response"
            ]
          },
          {
            "name": "secretory_arm_broadly_elevated",
            "role": "gate",
            "field": "$.secretory_arm_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "broadly_elevated"
            ]
          },
          {
            "name": "carries_aging_program",
            "role": "gate",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "carries_bona_fide_aging_program_distinct_from_nominated"
            ]
          },
          {
            "name": "conclusion_senescence_unlikely",
            "role": "gate",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "unlikely_major_contributor"
            ]
          },
          {
            "name": "cf_arrest_core_engaged_recall",
            "role": "hard_fail",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_engaged"
            ]
          },
          {
            "name": "cf_aging_program_conflation_or_explain_away",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "aging_program_indistinguishable_from_nominated_program",
              "aging_program_is_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_no_aging_program_sensitivity_failure",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_detectable_aging_program"
            ]
          },
          {
            "name": "cf_senescence_driver_conclusion",
            "role": "hard_fail",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "likely_primary_driver",
              "plausible_supported_contributor"
            ]
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: PASS\n  + cohort_spans_young_and_aged: passed=True, score=1.0\n  + candidate_is_senescence: passed=True, score=1.0\n  + arrest_core_not_engaged: passed=True, score=1.0\n  + secretory_arm_is_inflammaging: passed=True, score=1.0\n  + secretory_arm_broadly_elevated: passed=True, score=1.0\n  + carries_aging_program: passed=True, score=1.0\n  + conclusion_senescence_unlikely: passed=True, score=1.0\n  + hard_fail 'cf_arrest_core_engaged_recall': not triggered\n  + hard_fail 'cf_aging_program_conflation_or_explain_away': not triggered\n  + hard_fail 'cf_no_aging_program_sensitivity_failure': not triggered\n  + hard_fail 'cf_senescence_driver_conclusion': not triggered",
      "metrics": {
        "pass_rule": "all",
        "scoring_count": 7,
        "scoring_passed": 7,
        "scoring_total_score": 7,
        "hard_fail_triggered": []
      },
      "fieldScores": {
        "cohort_spans_young_and_aged": 1,
        "candidate_is_senescence": 1,
        "arrest_core_not_engaged": 1,
        "secretory_arm_is_inflammaging": 1,
        "secretory_arm_broadly_elevated": 1,
        "carries_aging_program": 1,
        "conclusion_senescence_unlikely": 1
      },
      "graderChecks": [
        {
          "name": "cohort_spans_young_and_aged",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "candidate_is_senescence",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "arrest_core_not_engaged",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "secretory_arm_is_inflammaging",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "secretory_arm_broadly_elevated",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "carries_aging_program",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "conclusion_senescence_unlikely",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "cf_arrest_core_engaged_recall",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_aging_program_conflation_or_explain_away",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_no_aging_program_sensitivity_failure",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_senescence_driver_conclusion",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "age_associated_program_identity": "carries_bona_fide_aging_program_distinct_from_nominated",
        "arrest_core_engaged": "arrest_core_not_engaged",
        "candidate_mechanism": "cellular_senescence",
        "cohort_host_age_design": "young_and_aged_hosts_both_profiled",
        "nominated_mechanism_role": "unlikely_major_contributor",
        "secretory_arm_direction": "broadly_elevated",
        "secretory_arm_identity": "acute_phase_inflammaging_response"
      },
      "nTurns": 28,
      "checkpointSignals": {
        "clustering": true,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": false,
        "nerveJointAnchor": false,
        "multiomeAssembly": false,
        "depthNormalizedAtac": false,
        "regulonInference": false,
        "cd8SubsetIntegration": true,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": false,
        "orthologProjection": false,
        "epiSubsetSelection": true,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": false,
        "mousePseudobulkDesign": true,
        "cancerEpithelialDenominator": true,
        "geneLevelDe": true,
        "agingSignatureControl": true,
        "senescencePanelScoring": true
      },
      "trajectoryFile": "scbench-long/trajectories/gse297023_q2_senescence_not_mechanism__gpt-5.4__pi__r2.json",
      "runId": "gpt-5.4__pi__gse297023_q2_senescence_not_mechanism__r2",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-gpt-5-4-pi",
      "modelName": "gpt-5.4",
      "harness": "pi",
      "task": "gse297023_q2_senescence_not_mechanism",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 3,
      "passed": false,
      "score": 4,
      "durationS": 4282.87,
      "cost": 0.9597095,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "candidate_is_senescence",
            "role": "gate",
            "field": "$.candidate_mechanism",
            "kind": "value",
            "op": "in",
            "expected": [
              "cellular_senescence",
              "oncogene_induced_senescence",
              "replicative_senescence"
            ]
          },
          {
            "name": "arrest_core_not_engaged",
            "role": "gate",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_not_engaged"
            ]
          },
          {
            "name": "secretory_arm_is_inflammaging",
            "role": "gate",
            "field": "$.secretory_arm_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "acute_phase_inflammaging_response"
            ]
          },
          {
            "name": "secretory_arm_broadly_elevated",
            "role": "gate",
            "field": "$.secretory_arm_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "broadly_elevated"
            ]
          },
          {
            "name": "carries_aging_program",
            "role": "gate",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "carries_bona_fide_aging_program_distinct_from_nominated"
            ]
          },
          {
            "name": "conclusion_senescence_unlikely",
            "role": "gate",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "unlikely_major_contributor"
            ]
          },
          {
            "name": "cf_arrest_core_engaged_recall",
            "role": "hard_fail",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_engaged"
            ]
          },
          {
            "name": "cf_aging_program_conflation_or_explain_away",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "aging_program_indistinguishable_from_nominated_program",
              "aging_program_is_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_no_aging_program_sensitivity_failure",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_detectable_aging_program"
            ]
          },
          {
            "name": "cf_senescence_driver_conclusion",
            "role": "hard_fail",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "likely_primary_driver",
              "plausible_supported_contributor"
            ]
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: FAIL\n  + cohort_spans_young_and_aged: passed=True, score=1.0\n  x candidate_is_senescence: passed=False, score=0.0\n  + arrest_core_not_engaged: passed=True, score=1.0\n  + secretory_arm_is_inflammaging: passed=True, score=1.0\n  + secretory_arm_broadly_elevated: passed=True, score=1.0\n  x carries_aging_program: passed=False, score=0.0\n  x conclusion_senescence_unlikely: passed=False, score=0.0\n  + hard_fail 'cf_arrest_core_engaged_recall': not triggered\n  x hard_fail 'cf_aging_program_conflation_or_explain_away': TRIGGERED\n  + hard_fail 'cf_no_aging_program_sensitivity_failure': not triggered\n  x hard_fail 'cf_senescence_driver_conclusion': TRIGGERED",
      "metrics": {
        "pass_rule": "all",
        "scoring_count": 7,
        "scoring_passed": 4,
        "scoring_total_score": 4,
        "hard_fail_triggered": [
          "cf_aging_program_conflation_or_explain_away",
          "cf_senescence_driver_conclusion"
        ]
      },
      "fieldScores": {
        "cohort_spans_young_and_aged": 1,
        "candidate_is_senescence": 0,
        "arrest_core_not_engaged": 1,
        "secretory_arm_is_inflammaging": 1,
        "secretory_arm_broadly_elevated": 1,
        "carries_aging_program": 0,
        "conclusion_senescence_unlikely": 0
      },
      "graderChecks": [
        {
          "name": "cohort_spans_young_and_aged",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "candidate_is_senescence",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "arrest_core_not_engaged",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "secretory_arm_is_inflammaging",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "secretory_arm_broadly_elevated",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "carries_aging_program",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "conclusion_senescence_unlikely",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "cf_arrest_core_engaged_recall",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_aging_program_conflation_or_explain_away",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "cf_no_aging_program_sensitivity_failure",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_senescence_driver_conclusion",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        }
      ],
      "agentAnswer": {
        "age_associated_program_identity": "aging_program_indistinguishable_from_nominated_program",
        "arrest_core_engaged": "arrest_core_not_engaged",
        "candidate_mechanism": "chronic_inflammation_inflammaging",
        "cohort_host_age_design": "young_and_aged_hosts_both_profiled",
        "nominated_mechanism_role": "plausible_supported_contributor",
        "secretory_arm_direction": "broadly_elevated",
        "secretory_arm_identity": "acute_phase_inflammaging_response"
      },
      "nTurns": 32,
      "checkpointSignals": {
        "clustering": true,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": false,
        "nerveJointAnchor": false,
        "multiomeAssembly": false,
        "depthNormalizedAtac": true,
        "regulonInference": false,
        "cd8SubsetIntegration": false,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": false,
        "orthologProjection": false,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": true,
        "mousePseudobulkDesign": true,
        "cancerEpithelialDenominator": true,
        "geneLevelDe": true,
        "agingSignatureControl": true,
        "senescencePanelScoring": true
      },
      "trajectoryFile": "scbench-long/trajectories/gse297023_q2_senescence_not_mechanism__gpt-5.4__pi__r3.json",
      "runId": "gpt-5.4__pi__gse297023_q2_senescence_not_mechanism__r3",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-gpt-5-5-codex",
      "modelName": "gpt-5.5",
      "harness": "openai-codex",
      "task": "gse297023_q2_senescence_not_mechanism",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 1,
      "passed": false,
      "score": 4,
      "durationS": 376.04,
      "cost": 1.0757430000000001,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "candidate_is_senescence",
            "role": "gate",
            "field": "$.candidate_mechanism",
            "kind": "value",
            "op": "in",
            "expected": [
              "cellular_senescence",
              "oncogene_induced_senescence",
              "replicative_senescence"
            ]
          },
          {
            "name": "arrest_core_not_engaged",
            "role": "gate",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_not_engaged"
            ]
          },
          {
            "name": "secretory_arm_is_inflammaging",
            "role": "gate",
            "field": "$.secretory_arm_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "acute_phase_inflammaging_response"
            ]
          },
          {
            "name": "secretory_arm_broadly_elevated",
            "role": "gate",
            "field": "$.secretory_arm_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "broadly_elevated"
            ]
          },
          {
            "name": "carries_aging_program",
            "role": "gate",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "carries_bona_fide_aging_program_distinct_from_nominated"
            ]
          },
          {
            "name": "conclusion_senescence_unlikely",
            "role": "gate",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "unlikely_major_contributor"
            ]
          },
          {
            "name": "cf_arrest_core_engaged_recall",
            "role": "hard_fail",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_engaged"
            ]
          },
          {
            "name": "cf_aging_program_conflation_or_explain_away",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "aging_program_indistinguishable_from_nominated_program",
              "aging_program_is_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_no_aging_program_sensitivity_failure",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_detectable_aging_program"
            ]
          },
          {
            "name": "cf_senescence_driver_conclusion",
            "role": "hard_fail",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "likely_primary_driver",
              "plausible_supported_contributor"
            ]
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: FAIL\n  + cohort_spans_young_and_aged: passed=True, score=1.0\n  x candidate_is_senescence: passed=False, score=0.0\n  + arrest_core_not_engaged: passed=True, score=1.0\n  + secretory_arm_is_inflammaging: passed=True, score=1.0\n  + secretory_arm_broadly_elevated: passed=True, score=1.0\n  x carries_aging_program: passed=False, score=0.0\n  x conclusion_senescence_unlikely: passed=False, score=0.0\n  + hard_fail 'cf_arrest_core_engaged_recall': not triggered\n  x hard_fail 'cf_aging_program_conflation_or_explain_away': TRIGGERED\n  + hard_fail 'cf_no_aging_program_sensitivity_failure': not triggered\n  x hard_fail 'cf_senescence_driver_conclusion': TRIGGERED",
      "metrics": {
        "pass_rule": "all",
        "scoring_count": 7,
        "scoring_passed": 4,
        "scoring_total_score": 4,
        "hard_fail_triggered": [
          "cf_aging_program_conflation_or_explain_away",
          "cf_senescence_driver_conclusion"
        ]
      },
      "fieldScores": {
        "cohort_spans_young_and_aged": 1,
        "candidate_is_senescence": 0,
        "arrest_core_not_engaged": 1,
        "secretory_arm_is_inflammaging": 1,
        "secretory_arm_broadly_elevated": 1,
        "carries_aging_program": 0,
        "conclusion_senescence_unlikely": 0
      },
      "graderChecks": [
        {
          "name": "cohort_spans_young_and_aged",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "candidate_is_senescence",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "arrest_core_not_engaged",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "secretory_arm_is_inflammaging",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "secretory_arm_broadly_elevated",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "carries_aging_program",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "conclusion_senescence_unlikely",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "cf_arrest_core_engaged_recall",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_aging_program_conflation_or_explain_away",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "cf_no_aging_program_sensitivity_failure",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_senescence_driver_conclusion",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        }
      ],
      "agentAnswer": {
        "age_associated_program_identity": "aging_program_indistinguishable_from_nominated_program",
        "arrest_core_engaged": "arrest_core_not_engaged",
        "candidate_mechanism": "chronic_inflammation_inflammaging",
        "cohort_host_age_design": "young_and_aged_hosts_both_profiled",
        "nominated_mechanism_role": "plausible_supported_contributor",
        "secretory_arm_direction": "broadly_elevated",
        "secretory_arm_identity": "acute_phase_inflammaging_response"
      },
      "nTurns": 1,
      "checkpointSignals": {
        "clustering": false,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": false,
        "nerveJointAnchor": false,
        "multiomeAssembly": false,
        "depthNormalizedAtac": true,
        "regulonInference": false,
        "cd8SubsetIntegration": true,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": false,
        "orthologProjection": false,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": false,
        "mousePseudobulkDesign": true,
        "cancerEpithelialDenominator": true,
        "geneLevelDe": true,
        "agingSignatureControl": true,
        "senescencePanelScoring": true
      },
      "trajectoryFile": "scbench-long/trajectories/gse297023_q2_senescence_not_mechanism__gpt-5.5__openai-codex__r1.json",
      "runId": "gpt-5.5__openai-codex__gse297023_q2_senescence_not_mechanism__r1",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-gpt-5-5-codex",
      "modelName": "gpt-5.5",
      "harness": "openai-codex",
      "task": "gse297023_q2_senescence_not_mechanism",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 2,
      "passed": false,
      "score": 4,
      "durationS": 321.06,
      "cost": 1.123397,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "candidate_is_senescence",
            "role": "gate",
            "field": "$.candidate_mechanism",
            "kind": "value",
            "op": "in",
            "expected": [
              "cellular_senescence",
              "oncogene_induced_senescence",
              "replicative_senescence"
            ]
          },
          {
            "name": "arrest_core_not_engaged",
            "role": "gate",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_not_engaged"
            ]
          },
          {
            "name": "secretory_arm_is_inflammaging",
            "role": "gate",
            "field": "$.secretory_arm_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "acute_phase_inflammaging_response"
            ]
          },
          {
            "name": "secretory_arm_broadly_elevated",
            "role": "gate",
            "field": "$.secretory_arm_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "broadly_elevated"
            ]
          },
          {
            "name": "carries_aging_program",
            "role": "gate",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "carries_bona_fide_aging_program_distinct_from_nominated"
            ]
          },
          {
            "name": "conclusion_senescence_unlikely",
            "role": "gate",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "unlikely_major_contributor"
            ]
          },
          {
            "name": "cf_arrest_core_engaged_recall",
            "role": "hard_fail",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_engaged"
            ]
          },
          {
            "name": "cf_aging_program_conflation_or_explain_away",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "aging_program_indistinguishable_from_nominated_program",
              "aging_program_is_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_no_aging_program_sensitivity_failure",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_detectable_aging_program"
            ]
          },
          {
            "name": "cf_senescence_driver_conclusion",
            "role": "hard_fail",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "likely_primary_driver",
              "plausible_supported_contributor"
            ]
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: FAIL\n  + cohort_spans_young_and_aged: passed=True, score=1.0\n  x candidate_is_senescence: passed=False, score=0.0\n  + arrest_core_not_engaged: passed=True, score=1.0\n  + secretory_arm_is_inflammaging: passed=True, score=1.0\n  + secretory_arm_broadly_elevated: passed=True, score=1.0\n  x carries_aging_program: passed=False, score=0.0\n  x conclusion_senescence_unlikely: passed=False, score=0.0\n  + hard_fail 'cf_arrest_core_engaged_recall': not triggered\n  x hard_fail 'cf_aging_program_conflation_or_explain_away': TRIGGERED\n  + hard_fail 'cf_no_aging_program_sensitivity_failure': not triggered\n  x hard_fail 'cf_senescence_driver_conclusion': TRIGGERED",
      "metrics": {
        "pass_rule": "all",
        "scoring_count": 7,
        "scoring_passed": 4,
        "scoring_total_score": 4,
        "hard_fail_triggered": [
          "cf_aging_program_conflation_or_explain_away",
          "cf_senescence_driver_conclusion"
        ]
      },
      "fieldScores": {
        "cohort_spans_young_and_aged": 1,
        "candidate_is_senescence": 0,
        "arrest_core_not_engaged": 1,
        "secretory_arm_is_inflammaging": 1,
        "secretory_arm_broadly_elevated": 1,
        "carries_aging_program": 0,
        "conclusion_senescence_unlikely": 0
      },
      "graderChecks": [
        {
          "name": "cohort_spans_young_and_aged",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "candidate_is_senescence",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "arrest_core_not_engaged",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "secretory_arm_is_inflammaging",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "secretory_arm_broadly_elevated",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "carries_aging_program",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "conclusion_senescence_unlikely",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "cf_arrest_core_engaged_recall",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_aging_program_conflation_or_explain_away",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "cf_no_aging_program_sensitivity_failure",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_senescence_driver_conclusion",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        }
      ],
      "agentAnswer": {
        "age_associated_program_identity": "aging_program_indistinguishable_from_nominated_program",
        "arrest_core_engaged": "arrest_core_not_engaged",
        "candidate_mechanism": "chronic_inflammation_inflammaging",
        "cohort_host_age_design": "young_and_aged_hosts_both_profiled",
        "nominated_mechanism_role": "plausible_supported_contributor",
        "secretory_arm_direction": "broadly_elevated",
        "secretory_arm_identity": "acute_phase_inflammaging_response"
      },
      "nTurns": 1,
      "checkpointSignals": {
        "clustering": false,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": false,
        "nerveJointAnchor": false,
        "multiomeAssembly": false,
        "depthNormalizedAtac": false,
        "regulonInference": false,
        "cd8SubsetIntegration": true,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": false,
        "orthologProjection": false,
        "epiSubsetSelection": true,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": false,
        "mousePseudobulkDesign": true,
        "cancerEpithelialDenominator": true,
        "geneLevelDe": true,
        "agingSignatureControl": true,
        "senescencePanelScoring": true
      },
      "trajectoryFile": "scbench-long/trajectories/gse297023_q2_senescence_not_mechanism__gpt-5.5__openai-codex__r2.json",
      "runId": "gpt-5.5__openai-codex__gse297023_q2_senescence_not_mechanism__r2",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-gpt-5-5-codex",
      "modelName": "gpt-5.5",
      "harness": "openai-codex",
      "task": "gse297023_q2_senescence_not_mechanism",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 3,
      "passed": false,
      "score": 4,
      "durationS": 809.22,
      "cost": 1.251042,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "candidate_is_senescence",
            "role": "gate",
            "field": "$.candidate_mechanism",
            "kind": "value",
            "op": "in",
            "expected": [
              "cellular_senescence",
              "oncogene_induced_senescence",
              "replicative_senescence"
            ]
          },
          {
            "name": "arrest_core_not_engaged",
            "role": "gate",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_not_engaged"
            ]
          },
          {
            "name": "secretory_arm_is_inflammaging",
            "role": "gate",
            "field": "$.secretory_arm_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "acute_phase_inflammaging_response"
            ]
          },
          {
            "name": "secretory_arm_broadly_elevated",
            "role": "gate",
            "field": "$.secretory_arm_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "broadly_elevated"
            ]
          },
          {
            "name": "carries_aging_program",
            "role": "gate",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "carries_bona_fide_aging_program_distinct_from_nominated"
            ]
          },
          {
            "name": "conclusion_senescence_unlikely",
            "role": "gate",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "unlikely_major_contributor"
            ]
          },
          {
            "name": "cf_arrest_core_engaged_recall",
            "role": "hard_fail",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_engaged"
            ]
          },
          {
            "name": "cf_aging_program_conflation_or_explain_away",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "aging_program_indistinguishable_from_nominated_program",
              "aging_program_is_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_no_aging_program_sensitivity_failure",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_detectable_aging_program"
            ]
          },
          {
            "name": "cf_senescence_driver_conclusion",
            "role": "hard_fail",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "likely_primary_driver",
              "plausible_supported_contributor"
            ]
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: FAIL\n  + cohort_spans_young_and_aged: passed=True, score=1.0\n  x candidate_is_senescence: passed=False, score=0.0\n  + arrest_core_not_engaged: passed=True, score=1.0\n  + secretory_arm_is_inflammaging: passed=True, score=1.0\n  + secretory_arm_broadly_elevated: passed=True, score=1.0\n  x carries_aging_program: passed=False, score=0.0\n  x conclusion_senescence_unlikely: passed=False, score=0.0\n  + hard_fail 'cf_arrest_core_engaged_recall': not triggered\n  x hard_fail 'cf_aging_program_conflation_or_explain_away': TRIGGERED\n  + hard_fail 'cf_no_aging_program_sensitivity_failure': not triggered\n  x hard_fail 'cf_senescence_driver_conclusion': TRIGGERED",
      "metrics": {
        "pass_rule": "all",
        "scoring_count": 7,
        "scoring_passed": 4,
        "scoring_total_score": 4,
        "hard_fail_triggered": [
          "cf_aging_program_conflation_or_explain_away",
          "cf_senescence_driver_conclusion"
        ]
      },
      "fieldScores": {
        "cohort_spans_young_and_aged": 1,
        "candidate_is_senescence": 0,
        "arrest_core_not_engaged": 1,
        "secretory_arm_is_inflammaging": 1,
        "secretory_arm_broadly_elevated": 1,
        "carries_aging_program": 0,
        "conclusion_senescence_unlikely": 0
      },
      "graderChecks": [
        {
          "name": "cohort_spans_young_and_aged",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "candidate_is_senescence",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "arrest_core_not_engaged",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "secretory_arm_is_inflammaging",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "secretory_arm_broadly_elevated",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "carries_aging_program",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "conclusion_senescence_unlikely",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "cf_arrest_core_engaged_recall",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_aging_program_conflation_or_explain_away",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "cf_no_aging_program_sensitivity_failure",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_senescence_driver_conclusion",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        }
      ],
      "agentAnswer": {
        "age_associated_program_identity": "aging_program_indistinguishable_from_nominated_program",
        "arrest_core_engaged": "arrest_core_not_engaged",
        "candidate_mechanism": "chronic_inflammation_inflammaging",
        "cohort_host_age_design": "young_and_aged_hosts_both_profiled",
        "nominated_mechanism_role": "plausible_supported_contributor",
        "secretory_arm_direction": "broadly_elevated",
        "secretory_arm_identity": "acute_phase_inflammaging_response"
      },
      "nTurns": 1,
      "checkpointSignals": {
        "clustering": false,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": true,
        "nerveJointAnchor": false,
        "multiomeAssembly": false,
        "depthNormalizedAtac": false,
        "regulonInference": false,
        "cd8SubsetIntegration": false,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": false,
        "orthologProjection": false,
        "epiSubsetSelection": true,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": false,
        "mousePseudobulkDesign": true,
        "cancerEpithelialDenominator": true,
        "geneLevelDe": true,
        "agingSignatureControl": true,
        "senescencePanelScoring": true
      },
      "trajectoryFile": "scbench-long/trajectories/gse297023_q2_senescence_not_mechanism__gpt-5.5__openai-codex__r3.json",
      "runId": "gpt-5.5__openai-codex__gse297023_q2_senescence_not_mechanism__r3",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-gpt-5-5-pi",
      "modelName": "gpt-5.5",
      "harness": "pi",
      "task": "gse297023_q2_senescence_not_mechanism",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 1,
      "passed": false,
      "score": 4,
      "durationS": 526.15,
      "cost": 1.6509220000000004,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "candidate_is_senescence",
            "role": "gate",
            "field": "$.candidate_mechanism",
            "kind": "value",
            "op": "in",
            "expected": [
              "cellular_senescence",
              "oncogene_induced_senescence",
              "replicative_senescence"
            ]
          },
          {
            "name": "arrest_core_not_engaged",
            "role": "gate",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_not_engaged"
            ]
          },
          {
            "name": "secretory_arm_is_inflammaging",
            "role": "gate",
            "field": "$.secretory_arm_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "acute_phase_inflammaging_response"
            ]
          },
          {
            "name": "secretory_arm_broadly_elevated",
            "role": "gate",
            "field": "$.secretory_arm_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "broadly_elevated"
            ]
          },
          {
            "name": "carries_aging_program",
            "role": "gate",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "carries_bona_fide_aging_program_distinct_from_nominated"
            ]
          },
          {
            "name": "conclusion_senescence_unlikely",
            "role": "gate",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "unlikely_major_contributor"
            ]
          },
          {
            "name": "cf_arrest_core_engaged_recall",
            "role": "hard_fail",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_engaged"
            ]
          },
          {
            "name": "cf_aging_program_conflation_or_explain_away",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "aging_program_indistinguishable_from_nominated_program",
              "aging_program_is_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_no_aging_program_sensitivity_failure",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_detectable_aging_program"
            ]
          },
          {
            "name": "cf_senescence_driver_conclusion",
            "role": "hard_fail",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "likely_primary_driver",
              "plausible_supported_contributor"
            ]
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: FAIL\n  + cohort_spans_young_and_aged: passed=True, score=1.0\n  x candidate_is_senescence: passed=False, score=0.0\n  + arrest_core_not_engaged: passed=True, score=1.0\n  + secretory_arm_is_inflammaging: passed=True, score=1.0\n  + secretory_arm_broadly_elevated: passed=True, score=1.0\n  x carries_aging_program: passed=False, score=0.0\n  x conclusion_senescence_unlikely: passed=False, score=0.0\n  + hard_fail 'cf_arrest_core_engaged_recall': not triggered\n  x hard_fail 'cf_aging_program_conflation_or_explain_away': TRIGGERED\n  + hard_fail 'cf_no_aging_program_sensitivity_failure': not triggered\n  x hard_fail 'cf_senescence_driver_conclusion': TRIGGERED",
      "metrics": {
        "pass_rule": "all",
        "scoring_count": 7,
        "scoring_passed": 4,
        "scoring_total_score": 4,
        "hard_fail_triggered": [
          "cf_aging_program_conflation_or_explain_away",
          "cf_senescence_driver_conclusion"
        ]
      },
      "fieldScores": {
        "cohort_spans_young_and_aged": 1,
        "candidate_is_senescence": 0,
        "arrest_core_not_engaged": 1,
        "secretory_arm_is_inflammaging": 1,
        "secretory_arm_broadly_elevated": 1,
        "carries_aging_program": 0,
        "conclusion_senescence_unlikely": 0
      },
      "graderChecks": [
        {
          "name": "cohort_spans_young_and_aged",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "candidate_is_senescence",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "arrest_core_not_engaged",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "secretory_arm_is_inflammaging",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "secretory_arm_broadly_elevated",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "carries_aging_program",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "conclusion_senescence_unlikely",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "cf_arrest_core_engaged_recall",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_aging_program_conflation_or_explain_away",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "cf_no_aging_program_sensitivity_failure",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_senescence_driver_conclusion",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        }
      ],
      "agentAnswer": {
        "age_associated_program_identity": "aging_program_indistinguishable_from_nominated_program",
        "arrest_core_engaged": "arrest_core_not_engaged",
        "candidate_mechanism": "chronic_inflammation_inflammaging",
        "cohort_host_age_design": "young_and_aged_hosts_both_profiled",
        "nominated_mechanism_role": "plausible_supported_contributor",
        "secretory_arm_direction": "broadly_elevated",
        "secretory_arm_identity": "acute_phase_inflammaging_response"
      },
      "nTurns": 28,
      "checkpointSignals": {
        "clustering": true,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": false,
        "nerveJointAnchor": false,
        "multiomeAssembly": false,
        "depthNormalizedAtac": false,
        "regulonInference": false,
        "cd8SubsetIntegration": false,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": false,
        "orthologProjection": false,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": true,
        "mousePseudobulkDesign": true,
        "cancerEpithelialDenominator": true,
        "geneLevelDe": true,
        "agingSignatureControl": true,
        "senescencePanelScoring": true
      },
      "trajectoryFile": "scbench-long/trajectories/gse297023_q2_senescence_not_mechanism__gpt-5.5__pi__r1.json",
      "runId": "gpt-5.5__pi__gse297023_q2_senescence_not_mechanism__r1",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-gpt-5-5-pi",
      "modelName": "gpt-5.5",
      "harness": "pi",
      "task": "gse297023_q2_senescence_not_mechanism",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 2,
      "passed": false,
      "score": 4,
      "durationS": 491.13,
      "cost": 1.2417390000000001,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "candidate_is_senescence",
            "role": "gate",
            "field": "$.candidate_mechanism",
            "kind": "value",
            "op": "in",
            "expected": [
              "cellular_senescence",
              "oncogene_induced_senescence",
              "replicative_senescence"
            ]
          },
          {
            "name": "arrest_core_not_engaged",
            "role": "gate",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_not_engaged"
            ]
          },
          {
            "name": "secretory_arm_is_inflammaging",
            "role": "gate",
            "field": "$.secretory_arm_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "acute_phase_inflammaging_response"
            ]
          },
          {
            "name": "secretory_arm_broadly_elevated",
            "role": "gate",
            "field": "$.secretory_arm_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "broadly_elevated"
            ]
          },
          {
            "name": "carries_aging_program",
            "role": "gate",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "carries_bona_fide_aging_program_distinct_from_nominated"
            ]
          },
          {
            "name": "conclusion_senescence_unlikely",
            "role": "gate",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "unlikely_major_contributor"
            ]
          },
          {
            "name": "cf_arrest_core_engaged_recall",
            "role": "hard_fail",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_engaged"
            ]
          },
          {
            "name": "cf_aging_program_conflation_or_explain_away",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "aging_program_indistinguishable_from_nominated_program",
              "aging_program_is_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_no_aging_program_sensitivity_failure",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_detectable_aging_program"
            ]
          },
          {
            "name": "cf_senescence_driver_conclusion",
            "role": "hard_fail",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "likely_primary_driver",
              "plausible_supported_contributor"
            ]
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: FAIL\n  + cohort_spans_young_and_aged: passed=True, score=1.0\n  x candidate_is_senescence: passed=False, score=0.0\n  + arrest_core_not_engaged: passed=True, score=1.0\n  + secretory_arm_is_inflammaging: passed=True, score=1.0\n  + secretory_arm_broadly_elevated: passed=True, score=1.0\n  x carries_aging_program: passed=False, score=0.0\n  x conclusion_senescence_unlikely: passed=False, score=0.0\n  + hard_fail 'cf_arrest_core_engaged_recall': not triggered\n  x hard_fail 'cf_aging_program_conflation_or_explain_away': TRIGGERED\n  + hard_fail 'cf_no_aging_program_sensitivity_failure': not triggered\n  x hard_fail 'cf_senescence_driver_conclusion': TRIGGERED",
      "metrics": {
        "pass_rule": "all",
        "scoring_count": 7,
        "scoring_passed": 4,
        "scoring_total_score": 4,
        "hard_fail_triggered": [
          "cf_aging_program_conflation_or_explain_away",
          "cf_senescence_driver_conclusion"
        ]
      },
      "fieldScores": {
        "cohort_spans_young_and_aged": 1,
        "candidate_is_senescence": 0,
        "arrest_core_not_engaged": 1,
        "secretory_arm_is_inflammaging": 1,
        "secretory_arm_broadly_elevated": 1,
        "carries_aging_program": 0,
        "conclusion_senescence_unlikely": 0
      },
      "graderChecks": [
        {
          "name": "cohort_spans_young_and_aged",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "candidate_is_senescence",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "arrest_core_not_engaged",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "secretory_arm_is_inflammaging",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "secretory_arm_broadly_elevated",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "carries_aging_program",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "conclusion_senescence_unlikely",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "cf_arrest_core_engaged_recall",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_aging_program_conflation_or_explain_away",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "cf_no_aging_program_sensitivity_failure",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_senescence_driver_conclusion",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        }
      ],
      "agentAnswer": {
        "age_associated_program_identity": "aging_program_indistinguishable_from_nominated_program",
        "arrest_core_engaged": "arrest_core_not_engaged",
        "candidate_mechanism": "chronic_inflammation_inflammaging",
        "cohort_host_age_design": "young_and_aged_hosts_both_profiled",
        "nominated_mechanism_role": "plausible_supported_contributor",
        "secretory_arm_direction": "broadly_elevated",
        "secretory_arm_identity": "acute_phase_inflammaging_response"
      },
      "nTurns": 26,
      "checkpointSignals": {
        "clustering": false,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": false,
        "nerveJointAnchor": false,
        "multiomeAssembly": true,
        "depthNormalizedAtac": true,
        "regulonInference": true,
        "cd8SubsetIntegration": false,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": false,
        "orthologProjection": false,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": true,
        "mousePseudobulkDesign": true,
        "cancerEpithelialDenominator": true,
        "geneLevelDe": true,
        "agingSignatureControl": true,
        "senescencePanelScoring": true
      },
      "trajectoryFile": "scbench-long/trajectories/gse297023_q2_senescence_not_mechanism__gpt-5.5__pi__r2.json",
      "runId": "gpt-5.5__pi__gse297023_q2_senescence_not_mechanism__r2",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-gpt-5-5-pi",
      "modelName": "gpt-5.5",
      "harness": "pi",
      "task": "gse297023_q2_senescence_not_mechanism",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 3,
      "passed": false,
      "score": 4,
      "durationS": 991.35,
      "cost": 1.4569649999999998,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "candidate_is_senescence",
            "role": "gate",
            "field": "$.candidate_mechanism",
            "kind": "value",
            "op": "in",
            "expected": [
              "cellular_senescence",
              "oncogene_induced_senescence",
              "replicative_senescence"
            ]
          },
          {
            "name": "arrest_core_not_engaged",
            "role": "gate",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_not_engaged"
            ]
          },
          {
            "name": "secretory_arm_is_inflammaging",
            "role": "gate",
            "field": "$.secretory_arm_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "acute_phase_inflammaging_response"
            ]
          },
          {
            "name": "secretory_arm_broadly_elevated",
            "role": "gate",
            "field": "$.secretory_arm_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "broadly_elevated"
            ]
          },
          {
            "name": "carries_aging_program",
            "role": "gate",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "carries_bona_fide_aging_program_distinct_from_nominated"
            ]
          },
          {
            "name": "conclusion_senescence_unlikely",
            "role": "gate",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "unlikely_major_contributor"
            ]
          },
          {
            "name": "cf_arrest_core_engaged_recall",
            "role": "hard_fail",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_engaged"
            ]
          },
          {
            "name": "cf_aging_program_conflation_or_explain_away",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "aging_program_indistinguishable_from_nominated_program",
              "aging_program_is_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_no_aging_program_sensitivity_failure",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_detectable_aging_program"
            ]
          },
          {
            "name": "cf_senescence_driver_conclusion",
            "role": "hard_fail",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "likely_primary_driver",
              "plausible_supported_contributor"
            ]
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: FAIL\n  + cohort_spans_young_and_aged: passed=True, score=1.0\n  x candidate_is_senescence: passed=False, score=0.0\n  + arrest_core_not_engaged: passed=True, score=1.0\n  + secretory_arm_is_inflammaging: passed=True, score=1.0\n  + secretory_arm_broadly_elevated: passed=True, score=1.0\n  x carries_aging_program: passed=False, score=0.0\n  x conclusion_senescence_unlikely: passed=False, score=0.0\n  + hard_fail 'cf_arrest_core_engaged_recall': not triggered\n  x hard_fail 'cf_aging_program_conflation_or_explain_away': TRIGGERED\n  + hard_fail 'cf_no_aging_program_sensitivity_failure': not triggered\n  x hard_fail 'cf_senescence_driver_conclusion': TRIGGERED",
      "metrics": {
        "pass_rule": "all",
        "scoring_count": 7,
        "scoring_passed": 4,
        "scoring_total_score": 4,
        "hard_fail_triggered": [
          "cf_aging_program_conflation_or_explain_away",
          "cf_senescence_driver_conclusion"
        ]
      },
      "fieldScores": {
        "cohort_spans_young_and_aged": 1,
        "candidate_is_senescence": 0,
        "arrest_core_not_engaged": 1,
        "secretory_arm_is_inflammaging": 1,
        "secretory_arm_broadly_elevated": 1,
        "carries_aging_program": 0,
        "conclusion_senescence_unlikely": 0
      },
      "graderChecks": [
        {
          "name": "cohort_spans_young_and_aged",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "candidate_is_senescence",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "arrest_core_not_engaged",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "secretory_arm_is_inflammaging",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "secretory_arm_broadly_elevated",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "carries_aging_program",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "conclusion_senescence_unlikely",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "cf_arrest_core_engaged_recall",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_aging_program_conflation_or_explain_away",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "cf_no_aging_program_sensitivity_failure",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_senescence_driver_conclusion",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        }
      ],
      "agentAnswer": {
        "age_associated_program_identity": "aging_program_indistinguishable_from_nominated_program",
        "arrest_core_engaged": "arrest_core_not_engaged",
        "candidate_mechanism": "chronic_inflammation_inflammaging",
        "cohort_host_age_design": "young_and_aged_hosts_both_profiled",
        "nominated_mechanism_role": "plausible_supported_contributor",
        "secretory_arm_direction": "broadly_elevated",
        "secretory_arm_identity": "acute_phase_inflammaging_response"
      },
      "nTurns": 20,
      "checkpointSignals": {
        "clustering": true,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": true,
        "nerveJointAnchor": false,
        "multiomeAssembly": true,
        "depthNormalizedAtac": true,
        "regulonInference": true,
        "cd8SubsetIntegration": true,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": false,
        "orthologProjection": false,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": false,
        "mousePseudobulkDesign": true,
        "cancerEpithelialDenominator": true,
        "geneLevelDe": true,
        "agingSignatureControl": true,
        "senescencePanelScoring": true
      },
      "trajectoryFile": "scbench-long/trajectories/gse297023_q2_senescence_not_mechanism__gpt-5.5__pi__r3.json",
      "runId": "gpt-5.5__pi__gse297023_q2_senescence_not_mechanism__r3",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-grok-4-20-beta-reasoning-pi",
      "modelName": "grok-4.20-beta-0309-reasoning",
      "harness": "pi",
      "task": "gse297023_q2_senescence_not_mechanism",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 1,
      "passed": false,
      "score": 5,
      "durationS": 1463.58,
      "cost": 0.11138435000000001,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "candidate_is_senescence",
            "role": "gate",
            "field": "$.candidate_mechanism",
            "kind": "value",
            "op": "in",
            "expected": [
              "cellular_senescence",
              "oncogene_induced_senescence",
              "replicative_senescence"
            ]
          },
          {
            "name": "arrest_core_not_engaged",
            "role": "gate",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_not_engaged"
            ]
          },
          {
            "name": "secretory_arm_is_inflammaging",
            "role": "gate",
            "field": "$.secretory_arm_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "acute_phase_inflammaging_response"
            ]
          },
          {
            "name": "secretory_arm_broadly_elevated",
            "role": "gate",
            "field": "$.secretory_arm_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "broadly_elevated"
            ]
          },
          {
            "name": "carries_aging_program",
            "role": "gate",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "carries_bona_fide_aging_program_distinct_from_nominated"
            ]
          },
          {
            "name": "conclusion_senescence_unlikely",
            "role": "gate",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "unlikely_major_contributor"
            ]
          },
          {
            "name": "cf_arrest_core_engaged_recall",
            "role": "hard_fail",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_engaged"
            ]
          },
          {
            "name": "cf_aging_program_conflation_or_explain_away",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "aging_program_indistinguishable_from_nominated_program",
              "aging_program_is_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_no_aging_program_sensitivity_failure",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_detectable_aging_program"
            ]
          },
          {
            "name": "cf_senescence_driver_conclusion",
            "role": "hard_fail",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "likely_primary_driver",
              "plausible_supported_contributor"
            ]
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: FAIL\n  + cohort_spans_young_and_aged: passed=True, score=1.0\n  + candidate_is_senescence: passed=True, score=1.0\n  + arrest_core_not_engaged: passed=True, score=1.0\n  x secretory_arm_is_inflammaging: passed=False, score=0.0\n  + secretory_arm_broadly_elevated: passed=True, score=1.0\n  x carries_aging_program: passed=False, score=0.0\n  + conclusion_senescence_unlikely: passed=True, score=1.0\n  + hard_fail 'cf_arrest_core_engaged_recall': not triggered\n  x hard_fail 'cf_aging_program_conflation_or_explain_away': TRIGGERED\n  + hard_fail 'cf_no_aging_program_sensitivity_failure': not triggered\n  + hard_fail 'cf_senescence_driver_conclusion': not triggered",
      "metrics": {
        "pass_rule": "all",
        "scoring_count": 7,
        "scoring_passed": 5,
        "scoring_total_score": 5,
        "hard_fail_triggered": [
          "cf_aging_program_conflation_or_explain_away"
        ]
      },
      "fieldScores": {
        "cohort_spans_young_and_aged": 1,
        "candidate_is_senescence": 1,
        "arrest_core_not_engaged": 1,
        "secretory_arm_is_inflammaging": 0,
        "secretory_arm_broadly_elevated": 1,
        "carries_aging_program": 0,
        "conclusion_senescence_unlikely": 1
      },
      "graderChecks": [
        {
          "name": "cohort_spans_young_and_aged",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "candidate_is_senescence",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "arrest_core_not_engaged",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "secretory_arm_is_inflammaging",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "secretory_arm_broadly_elevated",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "carries_aging_program",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "conclusion_senescence_unlikely",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "cf_arrest_core_engaged_recall",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_aging_program_conflation_or_explain_away",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "cf_no_aging_program_sensitivity_failure",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_senescence_driver_conclusion",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "age_associated_program_identity": "aging_program_indistinguishable_from_nominated_program",
        "arrest_core_engaged": "arrest_core_not_engaged",
        "candidate_mechanism": "cellular_senescence",
        "cohort_host_age_design": "young_and_aged_hosts_both_profiled",
        "nominated_mechanism_role": "unlikely_major_contributor",
        "secretory_arm_direction": "broadly_elevated",
        "secretory_arm_identity": "senescence_associated_secretory_phenotype"
      },
      "nTurns": 30,
      "checkpointSignals": {
        "clustering": true,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": false,
        "nerveJointAnchor": false,
        "multiomeAssembly": false,
        "depthNormalizedAtac": false,
        "regulonInference": false,
        "cd8SubsetIntegration": false,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": false,
        "orthologProjection": false,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": false,
        "mousePseudobulkDesign": false,
        "cancerEpithelialDenominator": true,
        "geneLevelDe": false,
        "agingSignatureControl": true,
        "senescencePanelScoring": true
      },
      "trajectoryFile": "scbench-long/trajectories/gse297023_q2_senescence_not_mechanism__grok-4.20-beta-0309-reasoning__pi__r1.json",
      "runId": "grok-4.20-beta-0309-reasoning__pi__gse297023_q2_senescence_not_mechanism__r1",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-grok-4-20-beta-reasoning-pi",
      "modelName": "grok-4.20-beta-0309-reasoning",
      "harness": "pi",
      "task": "gse297023_q2_senescence_not_mechanism",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 2,
      "passed": false,
      "score": 4,
      "durationS": 108.04,
      "cost": 0.02342625,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "candidate_is_senescence",
            "role": "gate",
            "field": "$.candidate_mechanism",
            "kind": "value",
            "op": "in",
            "expected": [
              "cellular_senescence",
              "oncogene_induced_senescence",
              "replicative_senescence"
            ]
          },
          {
            "name": "arrest_core_not_engaged",
            "role": "gate",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_not_engaged"
            ]
          },
          {
            "name": "secretory_arm_is_inflammaging",
            "role": "gate",
            "field": "$.secretory_arm_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "acute_phase_inflammaging_response"
            ]
          },
          {
            "name": "secretory_arm_broadly_elevated",
            "role": "gate",
            "field": "$.secretory_arm_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "broadly_elevated"
            ]
          },
          {
            "name": "carries_aging_program",
            "role": "gate",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "carries_bona_fide_aging_program_distinct_from_nominated"
            ]
          },
          {
            "name": "conclusion_senescence_unlikely",
            "role": "gate",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "unlikely_major_contributor"
            ]
          },
          {
            "name": "cf_arrest_core_engaged_recall",
            "role": "hard_fail",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_engaged"
            ]
          },
          {
            "name": "cf_aging_program_conflation_or_explain_away",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "aging_program_indistinguishable_from_nominated_program",
              "aging_program_is_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_no_aging_program_sensitivity_failure",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_detectable_aging_program"
            ]
          },
          {
            "name": "cf_senescence_driver_conclusion",
            "role": "hard_fail",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "likely_primary_driver",
              "plausible_supported_contributor"
            ]
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: FAIL\n  + cohort_spans_young_and_aged: passed=True, score=1.0\n  + candidate_is_senescence: passed=True, score=1.0\n  + arrest_core_not_engaged: passed=True, score=1.0\n  x secretory_arm_is_inflammaging: passed=False, score=0.0\n  + secretory_arm_broadly_elevated: passed=True, score=1.0\n  x carries_aging_program: passed=False, score=0.0\n  x conclusion_senescence_unlikely: passed=False, score=0.0\n  + hard_fail 'cf_arrest_core_engaged_recall': not triggered\n  x hard_fail 'cf_aging_program_conflation_or_explain_away': TRIGGERED\n  + hard_fail 'cf_no_aging_program_sensitivity_failure': not triggered\n  x hard_fail 'cf_senescence_driver_conclusion': TRIGGERED",
      "metrics": {
        "pass_rule": "all",
        "scoring_count": 7,
        "scoring_passed": 4,
        "scoring_total_score": 4,
        "hard_fail_triggered": [
          "cf_aging_program_conflation_or_explain_away",
          "cf_senescence_driver_conclusion"
        ]
      },
      "fieldScores": {
        "cohort_spans_young_and_aged": 1,
        "candidate_is_senescence": 1,
        "arrest_core_not_engaged": 1,
        "secretory_arm_is_inflammaging": 0,
        "secretory_arm_broadly_elevated": 1,
        "carries_aging_program": 0,
        "conclusion_senescence_unlikely": 0
      },
      "graderChecks": [
        {
          "name": "cohort_spans_young_and_aged",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "candidate_is_senescence",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "arrest_core_not_engaged",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "secretory_arm_is_inflammaging",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "secretory_arm_broadly_elevated",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "carries_aging_program",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "conclusion_senescence_unlikely",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "cf_arrest_core_engaged_recall",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_aging_program_conflation_or_explain_away",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "cf_no_aging_program_sensitivity_failure",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_senescence_driver_conclusion",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        }
      ],
      "agentAnswer": {
        "age_associated_program_identity": "aging_program_indistinguishable_from_nominated_program",
        "arrest_core_engaged": "arrest_core_not_engaged",
        "candidate_mechanism": "oncogene_induced_senescence",
        "cohort_host_age_design": "young_and_aged_hosts_both_profiled",
        "nominated_mechanism_role": "plausible_supported_contributor",
        "secretory_arm_direction": "broadly_elevated",
        "secretory_arm_identity": "senescence_associated_secretory_phenotype"
      },
      "nTurns": 15,
      "checkpointSignals": {
        "clustering": true,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": false,
        "nerveJointAnchor": false,
        "multiomeAssembly": false,
        "depthNormalizedAtac": false,
        "regulonInference": false,
        "cd8SubsetIntegration": false,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": false,
        "orthologProjection": false,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": false,
        "mousePseudobulkDesign": false,
        "cancerEpithelialDenominator": true,
        "geneLevelDe": false,
        "agingSignatureControl": true,
        "senescencePanelScoring": true
      },
      "trajectoryFile": "scbench-long/trajectories/gse297023_q2_senescence_not_mechanism__grok-4.20-beta-0309-reasoning__pi__r2.json",
      "runId": "grok-4.20-beta-0309-reasoning__pi__gse297023_q2_senescence_not_mechanism__r2",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-grok-4-20-beta-reasoning-pi",
      "modelName": "grok-4.20-beta-0309-reasoning",
      "harness": "pi",
      "task": "gse297023_q2_senescence_not_mechanism",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 3,
      "passed": false,
      "score": 4,
      "durationS": 796.96,
      "cost": 0.0329769,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "candidate_is_senescence",
            "role": "gate",
            "field": "$.candidate_mechanism",
            "kind": "value",
            "op": "in",
            "expected": [
              "cellular_senescence",
              "oncogene_induced_senescence",
              "replicative_senescence"
            ]
          },
          {
            "name": "arrest_core_not_engaged",
            "role": "gate",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_not_engaged"
            ]
          },
          {
            "name": "secretory_arm_is_inflammaging",
            "role": "gate",
            "field": "$.secretory_arm_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "acute_phase_inflammaging_response"
            ]
          },
          {
            "name": "secretory_arm_broadly_elevated",
            "role": "gate",
            "field": "$.secretory_arm_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "broadly_elevated"
            ]
          },
          {
            "name": "carries_aging_program",
            "role": "gate",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "carries_bona_fide_aging_program_distinct_from_nominated"
            ]
          },
          {
            "name": "conclusion_senescence_unlikely",
            "role": "gate",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "unlikely_major_contributor"
            ]
          },
          {
            "name": "cf_arrest_core_engaged_recall",
            "role": "hard_fail",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_engaged"
            ]
          },
          {
            "name": "cf_aging_program_conflation_or_explain_away",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "aging_program_indistinguishable_from_nominated_program",
              "aging_program_is_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_no_aging_program_sensitivity_failure",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_detectable_aging_program"
            ]
          },
          {
            "name": "cf_senescence_driver_conclusion",
            "role": "hard_fail",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "likely_primary_driver",
              "plausible_supported_contributor"
            ]
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: FAIL\n  + cohort_spans_young_and_aged: passed=True, score=1.0\n  + candidate_is_senescence: passed=True, score=1.0\n  + arrest_core_not_engaged: passed=True, score=1.0\n  x secretory_arm_is_inflammaging: passed=False, score=0.0\n  x secretory_arm_broadly_elevated: passed=False, score=0.0\n  x carries_aging_program: passed=False, score=0.0\n  + conclusion_senescence_unlikely: passed=True, score=1.0\n  + hard_fail 'cf_arrest_core_engaged_recall': not triggered\n  x hard_fail 'cf_aging_program_conflation_or_explain_away': TRIGGERED\n  + hard_fail 'cf_no_aging_program_sensitivity_failure': not triggered\n  + hard_fail 'cf_senescence_driver_conclusion': not triggered",
      "metrics": {
        "pass_rule": "all",
        "scoring_count": 7,
        "scoring_passed": 4,
        "scoring_total_score": 4,
        "hard_fail_triggered": [
          "cf_aging_program_conflation_or_explain_away"
        ]
      },
      "fieldScores": {
        "cohort_spans_young_and_aged": 1,
        "candidate_is_senescence": 1,
        "arrest_core_not_engaged": 1,
        "secretory_arm_is_inflammaging": 0,
        "secretory_arm_broadly_elevated": 0,
        "carries_aging_program": 0,
        "conclusion_senescence_unlikely": 1
      },
      "graderChecks": [
        {
          "name": "cohort_spans_young_and_aged",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "candidate_is_senescence",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "arrest_core_not_engaged",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "secretory_arm_is_inflammaging",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "secretory_arm_broadly_elevated",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "carries_aging_program",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "conclusion_senescence_unlikely",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "cf_arrest_core_engaged_recall",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_aging_program_conflation_or_explain_away",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "cf_no_aging_program_sensitivity_failure",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_senescence_driver_conclusion",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "age_associated_program_identity": "aging_program_indistinguishable_from_nominated_program",
        "arrest_core_engaged": "arrest_core_not_engaged",
        "candidate_mechanism": "cellular_senescence",
        "cohort_host_age_design": "young_and_aged_hosts_both_profiled",
        "nominated_mechanism_role": "unlikely_major_contributor",
        "secretory_arm_direction": "mixed_bidirectional",
        "secretory_arm_identity": "senescence_associated_secretory_phenotype"
      },
      "nTurns": 14,
      "checkpointSignals": {
        "clustering": true,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": false,
        "nerveJointAnchor": false,
        "multiomeAssembly": false,
        "depthNormalizedAtac": false,
        "regulonInference": false,
        "cd8SubsetIntegration": false,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": false,
        "orthologProjection": false,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": false,
        "mousePseudobulkDesign": false,
        "cancerEpithelialDenominator": true,
        "geneLevelDe": true,
        "agingSignatureControl": true,
        "senescencePanelScoring": true
      },
      "trajectoryFile": "scbench-long/trajectories/gse297023_q2_senescence_not_mechanism__grok-4.20-beta-0309-reasoning__pi__r3.json",
      "runId": "grok-4.20-beta-0309-reasoning__pi__gse297023_q2_senescence_not_mechanism__r3",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-grok-4-3-pi",
      "modelName": "grok-4.3",
      "harness": "pi",
      "task": "gse297023_q2_senescence_not_mechanism",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 1,
      "passed": false,
      "score": 4,
      "durationS": 142.02,
      "cost": 0.017675200000000002,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "candidate_is_senescence",
            "role": "gate",
            "field": "$.candidate_mechanism",
            "kind": "value",
            "op": "in",
            "expected": [
              "cellular_senescence",
              "oncogene_induced_senescence",
              "replicative_senescence"
            ]
          },
          {
            "name": "arrest_core_not_engaged",
            "role": "gate",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_not_engaged"
            ]
          },
          {
            "name": "secretory_arm_is_inflammaging",
            "role": "gate",
            "field": "$.secretory_arm_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "acute_phase_inflammaging_response"
            ]
          },
          {
            "name": "secretory_arm_broadly_elevated",
            "role": "gate",
            "field": "$.secretory_arm_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "broadly_elevated"
            ]
          },
          {
            "name": "carries_aging_program",
            "role": "gate",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "carries_bona_fide_aging_program_distinct_from_nominated"
            ]
          },
          {
            "name": "conclusion_senescence_unlikely",
            "role": "gate",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "unlikely_major_contributor"
            ]
          },
          {
            "name": "cf_arrest_core_engaged_recall",
            "role": "hard_fail",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_engaged"
            ]
          },
          {
            "name": "cf_aging_program_conflation_or_explain_away",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "aging_program_indistinguishable_from_nominated_program",
              "aging_program_is_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_no_aging_program_sensitivity_failure",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_detectable_aging_program"
            ]
          },
          {
            "name": "cf_senescence_driver_conclusion",
            "role": "hard_fail",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "likely_primary_driver",
              "plausible_supported_contributor"
            ]
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: FAIL\n  + cohort_spans_young_and_aged: passed=True, score=1.0\n  + candidate_is_senescence: passed=True, score=1.0\n  + arrest_core_not_engaged: passed=True, score=1.0\n  x secretory_arm_is_inflammaging: passed=False, score=0.0\n  x secretory_arm_broadly_elevated: passed=False, score=0.0\n  x carries_aging_program: passed=False, score=0.0\n  + conclusion_senescence_unlikely: passed=True, score=1.0\n  + hard_fail 'cf_arrest_core_engaged_recall': not triggered\n  + hard_fail 'cf_aging_program_conflation_or_explain_away': not triggered\n  x hard_fail 'cf_no_aging_program_sensitivity_failure': TRIGGERED\n  + hard_fail 'cf_senescence_driver_conclusion': not triggered",
      "metrics": {
        "pass_rule": "all",
        "scoring_count": 7,
        "scoring_passed": 4,
        "scoring_total_score": 4,
        "hard_fail_triggered": [
          "cf_no_aging_program_sensitivity_failure"
        ]
      },
      "fieldScores": {
        "cohort_spans_young_and_aged": 1,
        "candidate_is_senescence": 1,
        "arrest_core_not_engaged": 1,
        "secretory_arm_is_inflammaging": 0,
        "secretory_arm_broadly_elevated": 0,
        "carries_aging_program": 0,
        "conclusion_senescence_unlikely": 1
      },
      "graderChecks": [
        {
          "name": "cohort_spans_young_and_aged",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "candidate_is_senescence",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "arrest_core_not_engaged",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "secretory_arm_is_inflammaging",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "secretory_arm_broadly_elevated",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "carries_aging_program",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "conclusion_senescence_unlikely",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "cf_arrest_core_engaged_recall",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_aging_program_conflation_or_explain_away",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_no_aging_program_sensitivity_failure",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "cf_senescence_driver_conclusion",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "age_associated_program_identity": "no_detectable_aging_program",
        "arrest_core_engaged": "arrest_core_not_engaged",
        "candidate_mechanism": "cellular_senescence",
        "cohort_host_age_design": "young_and_aged_hosts_both_profiled",
        "nominated_mechanism_role": "unlikely_major_contributor",
        "secretory_arm_direction": "broadly_reduced",
        "secretory_arm_identity": "senescence_associated_secretory_phenotype"
      },
      "nTurns": 8,
      "checkpointSignals": {
        "clustering": false,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": false,
        "nerveJointAnchor": false,
        "multiomeAssembly": false,
        "depthNormalizedAtac": false,
        "regulonInference": false,
        "cd8SubsetIntegration": false,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": false,
        "orthologProjection": false,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": false,
        "mousePseudobulkDesign": false,
        "cancerEpithelialDenominator": true,
        "geneLevelDe": false,
        "agingSignatureControl": true,
        "senescencePanelScoring": true
      },
      "trajectoryFile": "scbench-long/trajectories/gse297023_q2_senescence_not_mechanism__grok-4.3__pi__r1.json",
      "runId": "grok-4.3__pi__gse297023_q2_senescence_not_mechanism__r1",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-grok-4-3-pi",
      "modelName": "grok-4.3",
      "harness": "pi",
      "task": "gse297023_q2_senescence_not_mechanism",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 2,
      "passed": false,
      "score": 3,
      "durationS": 115.11,
      "cost": 0.014891500000000002,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "candidate_is_senescence",
            "role": "gate",
            "field": "$.candidate_mechanism",
            "kind": "value",
            "op": "in",
            "expected": [
              "cellular_senescence",
              "oncogene_induced_senescence",
              "replicative_senescence"
            ]
          },
          {
            "name": "arrest_core_not_engaged",
            "role": "gate",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_not_engaged"
            ]
          },
          {
            "name": "secretory_arm_is_inflammaging",
            "role": "gate",
            "field": "$.secretory_arm_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "acute_phase_inflammaging_response"
            ]
          },
          {
            "name": "secretory_arm_broadly_elevated",
            "role": "gate",
            "field": "$.secretory_arm_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "broadly_elevated"
            ]
          },
          {
            "name": "carries_aging_program",
            "role": "gate",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "carries_bona_fide_aging_program_distinct_from_nominated"
            ]
          },
          {
            "name": "conclusion_senescence_unlikely",
            "role": "gate",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "unlikely_major_contributor"
            ]
          },
          {
            "name": "cf_arrest_core_engaged_recall",
            "role": "hard_fail",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_engaged"
            ]
          },
          {
            "name": "cf_aging_program_conflation_or_explain_away",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "aging_program_indistinguishable_from_nominated_program",
              "aging_program_is_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_no_aging_program_sensitivity_failure",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_detectable_aging_program"
            ]
          },
          {
            "name": "cf_senescence_driver_conclusion",
            "role": "hard_fail",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "likely_primary_driver",
              "plausible_supported_contributor"
            ]
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: FAIL\n  + cohort_spans_young_and_aged: passed=True, score=1.0\n  + candidate_is_senescence: passed=True, score=1.0\n  x arrest_core_not_engaged: passed=False, score=0.0\n  x secretory_arm_is_inflammaging: passed=False, score=0.0\n  + secretory_arm_broadly_elevated: passed=True, score=1.0\n  x carries_aging_program: passed=False, score=0.0\n  x conclusion_senescence_unlikely: passed=False, score=0.0\n  x hard_fail 'cf_arrest_core_engaged_recall': TRIGGERED\n  x hard_fail 'cf_aging_program_conflation_or_explain_away': TRIGGERED\n  + hard_fail 'cf_no_aging_program_sensitivity_failure': not triggered\n  x hard_fail 'cf_senescence_driver_conclusion': TRIGGERED",
      "metrics": {
        "pass_rule": "all",
        "scoring_count": 7,
        "scoring_passed": 3,
        "scoring_total_score": 3,
        "hard_fail_triggered": [
          "cf_arrest_core_engaged_recall",
          "cf_aging_program_conflation_or_explain_away",
          "cf_senescence_driver_conclusion"
        ]
      },
      "fieldScores": {
        "cohort_spans_young_and_aged": 1,
        "candidate_is_senescence": 1,
        "arrest_core_not_engaged": 0,
        "secretory_arm_is_inflammaging": 0,
        "secretory_arm_broadly_elevated": 1,
        "carries_aging_program": 0,
        "conclusion_senescence_unlikely": 0
      },
      "graderChecks": [
        {
          "name": "cohort_spans_young_and_aged",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "candidate_is_senescence",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "arrest_core_not_engaged",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "secretory_arm_is_inflammaging",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "secretory_arm_broadly_elevated",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "carries_aging_program",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "conclusion_senescence_unlikely",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "cf_arrest_core_engaged_recall",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "cf_aging_program_conflation_or_explain_away",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "cf_no_aging_program_sensitivity_failure",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_senescence_driver_conclusion",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        }
      ],
      "agentAnswer": {
        "age_associated_program_identity": "aging_program_indistinguishable_from_nominated_program",
        "arrest_core_engaged": "arrest_core_engaged",
        "candidate_mechanism": "oncogene_induced_senescence",
        "cohort_host_age_design": "young_and_aged_hosts_both_profiled",
        "nominated_mechanism_role": "plausible_supported_contributor",
        "secretory_arm_direction": "broadly_elevated",
        "secretory_arm_identity": "senescence_associated_secretory_phenotype"
      },
      "nTurns": 8,
      "checkpointSignals": {
        "clustering": false,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": false,
        "nerveJointAnchor": false,
        "multiomeAssembly": false,
        "depthNormalizedAtac": false,
        "regulonInference": false,
        "cd8SubsetIntegration": false,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": false,
        "orthologProjection": false,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": false,
        "mousePseudobulkDesign": false,
        "cancerEpithelialDenominator": true,
        "geneLevelDe": false,
        "agingSignatureControl": true,
        "senescencePanelScoring": true
      },
      "trajectoryFile": "scbench-long/trajectories/gse297023_q2_senescence_not_mechanism__grok-4.3__pi__r2.json",
      "runId": "grok-4.3__pi__gse297023_q2_senescence_not_mechanism__r2",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-grok-4-3-pi",
      "modelName": "grok-4.3",
      "harness": "pi",
      "task": "gse297023_q2_senescence_not_mechanism",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 3,
      "passed": false,
      "score": 4,
      "durationS": 77.01,
      "cost": 0.008642450000000001,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "candidate_is_senescence",
            "role": "gate",
            "field": "$.candidate_mechanism",
            "kind": "value",
            "op": "in",
            "expected": [
              "cellular_senescence",
              "oncogene_induced_senescence",
              "replicative_senescence"
            ]
          },
          {
            "name": "arrest_core_not_engaged",
            "role": "gate",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_not_engaged"
            ]
          },
          {
            "name": "secretory_arm_is_inflammaging",
            "role": "gate",
            "field": "$.secretory_arm_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "acute_phase_inflammaging_response"
            ]
          },
          {
            "name": "secretory_arm_broadly_elevated",
            "role": "gate",
            "field": "$.secretory_arm_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "broadly_elevated"
            ]
          },
          {
            "name": "carries_aging_program",
            "role": "gate",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "carries_bona_fide_aging_program_distinct_from_nominated"
            ]
          },
          {
            "name": "conclusion_senescence_unlikely",
            "role": "gate",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "unlikely_major_contributor"
            ]
          },
          {
            "name": "cf_arrest_core_engaged_recall",
            "role": "hard_fail",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_engaged"
            ]
          },
          {
            "name": "cf_aging_program_conflation_or_explain_away",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "aging_program_indistinguishable_from_nominated_program",
              "aging_program_is_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_no_aging_program_sensitivity_failure",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_detectable_aging_program"
            ]
          },
          {
            "name": "cf_senescence_driver_conclusion",
            "role": "hard_fail",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "likely_primary_driver",
              "plausible_supported_contributor"
            ]
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: FAIL\n  + cohort_spans_young_and_aged: passed=True, score=1.0\n  + candidate_is_senescence: passed=True, score=1.0\n  + arrest_core_not_engaged: passed=True, score=1.0\n  x secretory_arm_is_inflammaging: passed=False, score=0.0\n  x secretory_arm_broadly_elevated: passed=False, score=0.0\n  x carries_aging_program: passed=False, score=0.0\n  + conclusion_senescence_unlikely: passed=True, score=1.0\n  + hard_fail 'cf_arrest_core_engaged_recall': not triggered\n  x hard_fail 'cf_aging_program_conflation_or_explain_away': TRIGGERED\n  + hard_fail 'cf_no_aging_program_sensitivity_failure': not triggered\n  + hard_fail 'cf_senescence_driver_conclusion': not triggered",
      "metrics": {
        "pass_rule": "all",
        "scoring_count": 7,
        "scoring_passed": 4,
        "scoring_total_score": 4,
        "hard_fail_triggered": [
          "cf_aging_program_conflation_or_explain_away"
        ]
      },
      "fieldScores": {
        "cohort_spans_young_and_aged": 1,
        "candidate_is_senescence": 1,
        "arrest_core_not_engaged": 1,
        "secretory_arm_is_inflammaging": 0,
        "secretory_arm_broadly_elevated": 0,
        "carries_aging_program": 0,
        "conclusion_senescence_unlikely": 1
      },
      "graderChecks": [
        {
          "name": "cohort_spans_young_and_aged",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "candidate_is_senescence",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "arrest_core_not_engaged",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "secretory_arm_is_inflammaging",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "secretory_arm_broadly_elevated",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "carries_aging_program",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "conclusion_senescence_unlikely",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "cf_arrest_core_engaged_recall",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_aging_program_conflation_or_explain_away",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "cf_no_aging_program_sensitivity_failure",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_senescence_driver_conclusion",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "age_associated_program_identity": "aging_program_indistinguishable_from_nominated_program",
        "arrest_core_engaged": "arrest_core_not_engaged",
        "candidate_mechanism": "cellular_senescence",
        "cohort_host_age_design": "young_and_aged_hosts_both_profiled",
        "nominated_mechanism_role": "unlikely_major_contributor",
        "secretory_arm_direction": "broadly_reduced",
        "secretory_arm_identity": "senescence_associated_secretory_phenotype"
      },
      "nTurns": 6,
      "checkpointSignals": {
        "clustering": false,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": false,
        "nerveJointAnchor": false,
        "multiomeAssembly": false,
        "depthNormalizedAtac": false,
        "regulonInference": false,
        "cd8SubsetIntegration": false,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": false,
        "orthologProjection": false,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": false,
        "mousePseudobulkDesign": false,
        "cancerEpithelialDenominator": true,
        "geneLevelDe": false,
        "agingSignatureControl": true,
        "senescencePanelScoring": true
      },
      "trajectoryFile": "scbench-long/trajectories/gse297023_q2_senescence_not_mechanism__grok-4.3__pi__r3.json",
      "runId": "grok-4.3__pi__gse297023_q2_senescence_not_mechanism__r3",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-kimi-k2p6-pi",
      "modelName": "kimi-k2p6",
      "harness": "pi",
      "task": "gse297023_q2_senescence_not_mechanism",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 1,
      "passed": false,
      "score": 4,
      "durationS": 1533.46,
      "cost": 0.46373613,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "candidate_is_senescence",
            "role": "gate",
            "field": "$.candidate_mechanism",
            "kind": "value",
            "op": "in",
            "expected": [
              "cellular_senescence",
              "oncogene_induced_senescence",
              "replicative_senescence"
            ]
          },
          {
            "name": "arrest_core_not_engaged",
            "role": "gate",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_not_engaged"
            ]
          },
          {
            "name": "secretory_arm_is_inflammaging",
            "role": "gate",
            "field": "$.secretory_arm_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "acute_phase_inflammaging_response"
            ]
          },
          {
            "name": "secretory_arm_broadly_elevated",
            "role": "gate",
            "field": "$.secretory_arm_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "broadly_elevated"
            ]
          },
          {
            "name": "carries_aging_program",
            "role": "gate",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "carries_bona_fide_aging_program_distinct_from_nominated"
            ]
          },
          {
            "name": "conclusion_senescence_unlikely",
            "role": "gate",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "unlikely_major_contributor"
            ]
          },
          {
            "name": "cf_arrest_core_engaged_recall",
            "role": "hard_fail",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_engaged"
            ]
          },
          {
            "name": "cf_aging_program_conflation_or_explain_away",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "aging_program_indistinguishable_from_nominated_program",
              "aging_program_is_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_no_aging_program_sensitivity_failure",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_detectable_aging_program"
            ]
          },
          {
            "name": "cf_senescence_driver_conclusion",
            "role": "hard_fail",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "likely_primary_driver",
              "plausible_supported_contributor"
            ]
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: FAIL\n  + cohort_spans_young_and_aged: passed=True, score=1.0\n  x candidate_is_senescence: passed=False, score=0.0\n  + arrest_core_not_engaged: passed=True, score=1.0\n  + secretory_arm_is_inflammaging: passed=True, score=1.0\n  + secretory_arm_broadly_elevated: passed=True, score=1.0\n  x carries_aging_program: passed=False, score=0.0\n  x conclusion_senescence_unlikely: passed=False, score=0.0\n  + hard_fail 'cf_arrest_core_engaged_recall': not triggered\n  x hard_fail 'cf_aging_program_conflation_or_explain_away': TRIGGERED\n  + hard_fail 'cf_no_aging_program_sensitivity_failure': not triggered\n  x hard_fail 'cf_senescence_driver_conclusion': TRIGGERED",
      "metrics": {
        "pass_rule": "all",
        "scoring_count": 7,
        "scoring_passed": 4,
        "scoring_total_score": 4,
        "hard_fail_triggered": [
          "cf_aging_program_conflation_or_explain_away",
          "cf_senescence_driver_conclusion"
        ]
      },
      "fieldScores": {
        "cohort_spans_young_and_aged": 1,
        "candidate_is_senescence": 0,
        "arrest_core_not_engaged": 1,
        "secretory_arm_is_inflammaging": 1,
        "secretory_arm_broadly_elevated": 1,
        "carries_aging_program": 0,
        "conclusion_senescence_unlikely": 0
      },
      "graderChecks": [
        {
          "name": "cohort_spans_young_and_aged",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "candidate_is_senescence",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "arrest_core_not_engaged",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "secretory_arm_is_inflammaging",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "secretory_arm_broadly_elevated",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "carries_aging_program",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "conclusion_senescence_unlikely",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "cf_arrest_core_engaged_recall",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_aging_program_conflation_or_explain_away",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "cf_no_aging_program_sensitivity_failure",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_senescence_driver_conclusion",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        }
      ],
      "agentAnswer": {
        "age_associated_program_identity": "aging_program_indistinguishable_from_nominated_program",
        "arrest_core_engaged": "arrest_core_not_engaged",
        "candidate_mechanism": "chronic_inflammation_inflammaging",
        "cohort_host_age_design": "young_and_aged_hosts_both_profiled",
        "nominated_mechanism_role": "likely_primary_driver",
        "secretory_arm_direction": "broadly_elevated",
        "secretory_arm_identity": "acute_phase_inflammaging_response"
      },
      "nTurns": 25,
      "checkpointSignals": {
        "clustering": false,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": false,
        "nerveJointAnchor": false,
        "multiomeAssembly": false,
        "depthNormalizedAtac": false,
        "regulonInference": false,
        "cd8SubsetIntegration": false,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": false,
        "orthologProjection": false,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": false,
        "mousePseudobulkDesign": true,
        "cancerEpithelialDenominator": true,
        "geneLevelDe": true,
        "agingSignatureControl": true,
        "senescencePanelScoring": true
      },
      "trajectoryFile": "scbench-long/trajectories/gse297023_q2_senescence_not_mechanism__kimi-k2p6__pi__r1.json",
      "runId": "kimi-k2p6__pi__gse297023_q2_senescence_not_mechanism__r1",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-kimi-k2p6-pi",
      "modelName": "kimi-k2p6",
      "harness": "pi",
      "task": "gse297023_q2_senescence_not_mechanism",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 2,
      "passed": false,
      "score": 4,
      "durationS": 1035.22,
      "cost": 0.26242231,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "candidate_is_senescence",
            "role": "gate",
            "field": "$.candidate_mechanism",
            "kind": "value",
            "op": "in",
            "expected": [
              "cellular_senescence",
              "oncogene_induced_senescence",
              "replicative_senescence"
            ]
          },
          {
            "name": "arrest_core_not_engaged",
            "role": "gate",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_not_engaged"
            ]
          },
          {
            "name": "secretory_arm_is_inflammaging",
            "role": "gate",
            "field": "$.secretory_arm_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "acute_phase_inflammaging_response"
            ]
          },
          {
            "name": "secretory_arm_broadly_elevated",
            "role": "gate",
            "field": "$.secretory_arm_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "broadly_elevated"
            ]
          },
          {
            "name": "carries_aging_program",
            "role": "gate",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "carries_bona_fide_aging_program_distinct_from_nominated"
            ]
          },
          {
            "name": "conclusion_senescence_unlikely",
            "role": "gate",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "unlikely_major_contributor"
            ]
          },
          {
            "name": "cf_arrest_core_engaged_recall",
            "role": "hard_fail",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_engaged"
            ]
          },
          {
            "name": "cf_aging_program_conflation_or_explain_away",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "aging_program_indistinguishable_from_nominated_program",
              "aging_program_is_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_no_aging_program_sensitivity_failure",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_detectable_aging_program"
            ]
          },
          {
            "name": "cf_senescence_driver_conclusion",
            "role": "hard_fail",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "likely_primary_driver",
              "plausible_supported_contributor"
            ]
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: FAIL\n  + cohort_spans_young_and_aged: passed=True, score=1.0\n  x candidate_is_senescence: passed=False, score=0.0\n  + arrest_core_not_engaged: passed=True, score=1.0\n  + secretory_arm_is_inflammaging: passed=True, score=1.0\n  + secretory_arm_broadly_elevated: passed=True, score=1.0\n  x carries_aging_program: passed=False, score=0.0\n  x conclusion_senescence_unlikely: passed=False, score=0.0\n  + hard_fail 'cf_arrest_core_engaged_recall': not triggered\n  x hard_fail 'cf_aging_program_conflation_or_explain_away': TRIGGERED\n  + hard_fail 'cf_no_aging_program_sensitivity_failure': not triggered\n  x hard_fail 'cf_senescence_driver_conclusion': TRIGGERED",
      "metrics": {
        "pass_rule": "all",
        "scoring_count": 7,
        "scoring_passed": 4,
        "scoring_total_score": 4,
        "hard_fail_triggered": [
          "cf_aging_program_conflation_or_explain_away",
          "cf_senescence_driver_conclusion"
        ]
      },
      "fieldScores": {
        "cohort_spans_young_and_aged": 1,
        "candidate_is_senescence": 0,
        "arrest_core_not_engaged": 1,
        "secretory_arm_is_inflammaging": 1,
        "secretory_arm_broadly_elevated": 1,
        "carries_aging_program": 0,
        "conclusion_senescence_unlikely": 0
      },
      "graderChecks": [
        {
          "name": "cohort_spans_young_and_aged",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "candidate_is_senescence",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "arrest_core_not_engaged",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "secretory_arm_is_inflammaging",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "secretory_arm_broadly_elevated",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "carries_aging_program",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "conclusion_senescence_unlikely",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "cf_arrest_core_engaged_recall",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_aging_program_conflation_or_explain_away",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        },
        {
          "name": "cf_no_aging_program_sensitivity_failure",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_senescence_driver_conclusion",
          "kind": "hard_fail",
          "passed": false,
          "triggered": true
        }
      ],
      "agentAnswer": {
        "age_associated_program_identity": "aging_program_indistinguishable_from_nominated_program",
        "arrest_core_engaged": "arrest_core_not_engaged",
        "candidate_mechanism": "chronic_inflammation_inflammaging",
        "cohort_host_age_design": "young_and_aged_hosts_both_profiled",
        "nominated_mechanism_role": "likely_primary_driver",
        "secretory_arm_direction": "broadly_elevated",
        "secretory_arm_identity": "acute_phase_inflammaging_response"
      },
      "nTurns": 19,
      "checkpointSignals": {
        "clustering": true,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": false,
        "nerveJointAnchor": false,
        "multiomeAssembly": true,
        "depthNormalizedAtac": false,
        "regulonInference": true,
        "cd8SubsetIntegration": false,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": false,
        "orthologProjection": false,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": false,
        "mousePseudobulkDesign": true,
        "cancerEpithelialDenominator": true,
        "geneLevelDe": true,
        "agingSignatureControl": true,
        "senescencePanelScoring": true
      },
      "trajectoryFile": "scbench-long/trajectories/gse297023_q2_senescence_not_mechanism__kimi-k2p6__pi__r2.json",
      "runId": "kimi-k2p6__pi__gse297023_q2_senescence_not_mechanism__r2",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-kimi-k2p6-pi",
      "modelName": "kimi-k2p6",
      "harness": "pi",
      "task": "gse297023_q2_senescence_not_mechanism",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 3,
      "passed": false,
      "score": 5,
      "durationS": 1540.52,
      "cost": 0.49473280000000003,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "candidate_is_senescence",
            "role": "gate",
            "field": "$.candidate_mechanism",
            "kind": "value",
            "op": "in",
            "expected": [
              "cellular_senescence",
              "oncogene_induced_senescence",
              "replicative_senescence"
            ]
          },
          {
            "name": "arrest_core_not_engaged",
            "role": "gate",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_not_engaged"
            ]
          },
          {
            "name": "secretory_arm_is_inflammaging",
            "role": "gate",
            "field": "$.secretory_arm_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "acute_phase_inflammaging_response"
            ]
          },
          {
            "name": "secretory_arm_broadly_elevated",
            "role": "gate",
            "field": "$.secretory_arm_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "broadly_elevated"
            ]
          },
          {
            "name": "carries_aging_program",
            "role": "gate",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "carries_bona_fide_aging_program_distinct_from_nominated"
            ]
          },
          {
            "name": "conclusion_senescence_unlikely",
            "role": "gate",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "unlikely_major_contributor"
            ]
          },
          {
            "name": "cf_arrest_core_engaged_recall",
            "role": "hard_fail",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_engaged"
            ]
          },
          {
            "name": "cf_aging_program_conflation_or_explain_away",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "aging_program_indistinguishable_from_nominated_program",
              "aging_program_is_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_no_aging_program_sensitivity_failure",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_detectable_aging_program"
            ]
          },
          {
            "name": "cf_senescence_driver_conclusion",
            "role": "hard_fail",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "likely_primary_driver",
              "plausible_supported_contributor"
            ]
          }
        ]
      },
      "reasoning": "all_of [pass_rule=all]: FAIL\n  + cohort_spans_young_and_aged: passed=True, score=1.0\n  + candidate_is_senescence: passed=True, score=1.0\n  + arrest_core_not_engaged: passed=True, score=1.0\n  x secretory_arm_is_inflammaging: passed=False, score=0.0\n  x secretory_arm_broadly_elevated: passed=False, score=0.0\n  + carries_aging_program: passed=True, score=1.0\n  + conclusion_senescence_unlikely: passed=True, score=1.0\n  + hard_fail 'cf_arrest_core_engaged_recall': not triggered\n  + hard_fail 'cf_aging_program_conflation_or_explain_away': not triggered\n  + hard_fail 'cf_no_aging_program_sensitivity_failure': not triggered\n  + hard_fail 'cf_senescence_driver_conclusion': not triggered",
      "metrics": {
        "pass_rule": "all",
        "scoring_count": 7,
        "scoring_passed": 5,
        "scoring_total_score": 5,
        "hard_fail_triggered": []
      },
      "fieldScores": {
        "cohort_spans_young_and_aged": 1,
        "candidate_is_senescence": 1,
        "arrest_core_not_engaged": 1,
        "secretory_arm_is_inflammaging": 0,
        "secretory_arm_broadly_elevated": 0,
        "carries_aging_program": 1,
        "conclusion_senescence_unlikely": 1
      },
      "graderChecks": [
        {
          "name": "cohort_spans_young_and_aged",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "candidate_is_senescence",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "arrest_core_not_engaged",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "secretory_arm_is_inflammaging",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "secretory_arm_broadly_elevated",
          "kind": "gate",
          "passed": false,
          "score": 0
        },
        {
          "name": "carries_aging_program",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "conclusion_senescence_unlikely",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "cf_arrest_core_engaged_recall",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_aging_program_conflation_or_explain_away",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_no_aging_program_sensitivity_failure",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_senescence_driver_conclusion",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "age_associated_program_identity": "carries_bona_fide_aging_program_distinct_from_nominated",
        "arrest_core_engaged": "arrest_core_not_engaged",
        "candidate_mechanism": "cellular_senescence",
        "cohort_host_age_design": "young_and_aged_hosts_both_profiled",
        "nominated_mechanism_role": "unlikely_major_contributor",
        "secretory_arm_direction": "mixed_bidirectional",
        "secretory_arm_identity": "senescence_associated_secretory_phenotype"
      },
      "nTurns": 22,
      "checkpointSignals": {
        "clustering": false,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": false,
        "nerveJointAnchor": false,
        "multiomeAssembly": true,
        "depthNormalizedAtac": true,
        "regulonInference": true,
        "cd8SubsetIntegration": false,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": false,
        "orthologProjection": false,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": false,
        "mousePseudobulkDesign": true,
        "cancerEpithelialDenominator": true,
        "geneLevelDe": true,
        "agingSignatureControl": true,
        "senescencePanelScoring": true
      },
      "trajectoryFile": "scbench-long/trajectories/gse297023_q2_senescence_not_mechanism__kimi-k2p6__pi__r3.json",
      "runId": "kimi-k2p6__pi__gse297023_q2_senescence_not_mechanism__r3",
      "artifactAvailable": true
    },
    {
      "modelId": "sclong-opus-4-6-claude-code",
      "modelName": "claude-opus-4-6",
      "harness": "claude-code",
      "task": "gse297023_q3_age_retained_mapk_reduced",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 3153,
      "cost": 2.4714575,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "age_retained",
            "role": "gate",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_retained"
            ]
          },
          {
            "name": "identity_is_aging",
            "role": "gate",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_aging_signatures"
            ]
          },
          {
            "name": "nature_state_shift",
            "role": "gate",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_state_expression_shift"
            ]
          },
          {
            "name": "pathway_is_mapk",
            "role": "gate",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "mapk"
            ]
          },
          {
            "name": "pathway_reduced_in_aged",
            "role": "gate",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "reduced_in_aged"
            ]
          },
          {
            "name": "cf_mapk_increased_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_in_aged"
            ]
          },
          {
            "name": "cf_age_erased",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_erased"
            ]
          },
          {
            "name": "cf_pervasive_overcall",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_pervasive_artifactual"
            ]
          },
          {
            "name": "cf_explain_away_aging",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_compositional_or_identity",
            "role": "hard_fail",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_proportion_shift",
              "cell_identity_shift",
              "composition_dominant_state_secondary",
              "both_state_and_composition_equally"
            ]
          },
          {
            "name": "cf_biggest_inflammatory_signal_not_mapk",
            "role": "hard_fail",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "jak_stat",
              "nfkb",
              "tnfa"
            ]
          },
          {
            "name": "cf_senescence_misattribution",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_senescence_signature_specifically"
            ]
          },
          {
            "name": "cf_dismiss_as_batch_artifact",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_batch_or_technical_artifact"
            ]
          },
          {
            "name": "cf_mapk_comparable_constitutive_kras_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "comparable_across_ages"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-6-claude-code",
      "modelName": "claude-opus-4-6",
      "harness": "claude-code",
      "task": "gse297023_q3_age_retained_mapk_reduced",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 3092,
      "cost": 2.68725925,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "age_retained",
            "role": "gate",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_retained"
            ]
          },
          {
            "name": "identity_is_aging",
            "role": "gate",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_aging_signatures"
            ]
          },
          {
            "name": "nature_state_shift",
            "role": "gate",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_state_expression_shift"
            ]
          },
          {
            "name": "pathway_is_mapk",
            "role": "gate",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "mapk"
            ]
          },
          {
            "name": "pathway_reduced_in_aged",
            "role": "gate",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "reduced_in_aged"
            ]
          },
          {
            "name": "cf_mapk_increased_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_in_aged"
            ]
          },
          {
            "name": "cf_age_erased",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_erased"
            ]
          },
          {
            "name": "cf_pervasive_overcall",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_pervasive_artifactual"
            ]
          },
          {
            "name": "cf_explain_away_aging",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_compositional_or_identity",
            "role": "hard_fail",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_proportion_shift",
              "cell_identity_shift",
              "composition_dominant_state_secondary",
              "both_state_and_composition_equally"
            ]
          },
          {
            "name": "cf_biggest_inflammatory_signal_not_mapk",
            "role": "hard_fail",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "jak_stat",
              "nfkb",
              "tnfa"
            ]
          },
          {
            "name": "cf_senescence_misattribution",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_senescence_signature_specifically"
            ]
          },
          {
            "name": "cf_dismiss_as_batch_artifact",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_batch_or_technical_artifact"
            ]
          },
          {
            "name": "cf_mapk_comparable_constitutive_kras_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "comparable_across_ages"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-6-claude-code",
      "modelName": "claude-opus-4-6",
      "harness": "claude-code",
      "task": "gse297023_q3_age_retained_mapk_reduced",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 2335,
      "cost": 1.256366,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "age_retained",
            "role": "gate",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_retained"
            ]
          },
          {
            "name": "identity_is_aging",
            "role": "gate",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_aging_signatures"
            ]
          },
          {
            "name": "nature_state_shift",
            "role": "gate",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_state_expression_shift"
            ]
          },
          {
            "name": "pathway_is_mapk",
            "role": "gate",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "mapk"
            ]
          },
          {
            "name": "pathway_reduced_in_aged",
            "role": "gate",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "reduced_in_aged"
            ]
          },
          {
            "name": "cf_mapk_increased_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_in_aged"
            ]
          },
          {
            "name": "cf_age_erased",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_erased"
            ]
          },
          {
            "name": "cf_pervasive_overcall",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_pervasive_artifactual"
            ]
          },
          {
            "name": "cf_explain_away_aging",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_compositional_or_identity",
            "role": "hard_fail",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_proportion_shift",
              "cell_identity_shift",
              "composition_dominant_state_secondary",
              "both_state_and_composition_equally"
            ]
          },
          {
            "name": "cf_biggest_inflammatory_signal_not_mapk",
            "role": "hard_fail",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "jak_stat",
              "nfkb",
              "tnfa"
            ]
          },
          {
            "name": "cf_senescence_misattribution",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_senescence_signature_specifically"
            ]
          },
          {
            "name": "cf_dismiss_as_batch_artifact",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_batch_or_technical_artifact"
            ]
          },
          {
            "name": "cf_mapk_comparable_constitutive_kras_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "comparable_across_ages"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-6-pi",
      "modelName": "claude-opus-4-6",
      "harness": "pi",
      "task": "gse297023_q3_age_retained_mapk_reduced",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 3552,
      "cost": 2.9806055,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "age_retained",
            "role": "gate",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_retained"
            ]
          },
          {
            "name": "identity_is_aging",
            "role": "gate",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_aging_signatures"
            ]
          },
          {
            "name": "nature_state_shift",
            "role": "gate",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_state_expression_shift"
            ]
          },
          {
            "name": "pathway_is_mapk",
            "role": "gate",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "mapk"
            ]
          },
          {
            "name": "pathway_reduced_in_aged",
            "role": "gate",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "reduced_in_aged"
            ]
          },
          {
            "name": "cf_mapk_increased_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_in_aged"
            ]
          },
          {
            "name": "cf_age_erased",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_erased"
            ]
          },
          {
            "name": "cf_pervasive_overcall",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_pervasive_artifactual"
            ]
          },
          {
            "name": "cf_explain_away_aging",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_compositional_or_identity",
            "role": "hard_fail",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_proportion_shift",
              "cell_identity_shift",
              "composition_dominant_state_secondary",
              "both_state_and_composition_equally"
            ]
          },
          {
            "name": "cf_biggest_inflammatory_signal_not_mapk",
            "role": "hard_fail",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "jak_stat",
              "nfkb",
              "tnfa"
            ]
          },
          {
            "name": "cf_senescence_misattribution",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_senescence_signature_specifically"
            ]
          },
          {
            "name": "cf_dismiss_as_batch_artifact",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_batch_or_technical_artifact"
            ]
          },
          {
            "name": "cf_mapk_comparable_constitutive_kras_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "comparable_across_ages"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-6-pi",
      "modelName": "claude-opus-4-6",
      "harness": "pi",
      "task": "gse297023_q3_age_retained_mapk_reduced",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 2396,
      "cost": 2.6825965000000003,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "age_retained",
            "role": "gate",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_retained"
            ]
          },
          {
            "name": "identity_is_aging",
            "role": "gate",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_aging_signatures"
            ]
          },
          {
            "name": "nature_state_shift",
            "role": "gate",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_state_expression_shift"
            ]
          },
          {
            "name": "pathway_is_mapk",
            "role": "gate",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "mapk"
            ]
          },
          {
            "name": "pathway_reduced_in_aged",
            "role": "gate",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "reduced_in_aged"
            ]
          },
          {
            "name": "cf_mapk_increased_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_in_aged"
            ]
          },
          {
            "name": "cf_age_erased",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_erased"
            ]
          },
          {
            "name": "cf_pervasive_overcall",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_pervasive_artifactual"
            ]
          },
          {
            "name": "cf_explain_away_aging",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_compositional_or_identity",
            "role": "hard_fail",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_proportion_shift",
              "cell_identity_shift",
              "composition_dominant_state_secondary",
              "both_state_and_composition_equally"
            ]
          },
          {
            "name": "cf_biggest_inflammatory_signal_not_mapk",
            "role": "hard_fail",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "jak_stat",
              "nfkb",
              "tnfa"
            ]
          },
          {
            "name": "cf_senescence_misattribution",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_senescence_signature_specifically"
            ]
          },
          {
            "name": "cf_dismiss_as_batch_artifact",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_batch_or_technical_artifact"
            ]
          },
          {
            "name": "cf_mapk_comparable_constitutive_kras_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "comparable_across_ages"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-6-pi",
      "modelName": "claude-opus-4-6",
      "harness": "pi",
      "task": "gse297023_q3_age_retained_mapk_reduced",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 3172,
      "cost": 2.12769825,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "age_retained",
            "role": "gate",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_retained"
            ]
          },
          {
            "name": "identity_is_aging",
            "role": "gate",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_aging_signatures"
            ]
          },
          {
            "name": "nature_state_shift",
            "role": "gate",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_state_expression_shift"
            ]
          },
          {
            "name": "pathway_is_mapk",
            "role": "gate",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "mapk"
            ]
          },
          {
            "name": "pathway_reduced_in_aged",
            "role": "gate",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "reduced_in_aged"
            ]
          },
          {
            "name": "cf_mapk_increased_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_in_aged"
            ]
          },
          {
            "name": "cf_age_erased",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_erased"
            ]
          },
          {
            "name": "cf_pervasive_overcall",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_pervasive_artifactual"
            ]
          },
          {
            "name": "cf_explain_away_aging",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_compositional_or_identity",
            "role": "hard_fail",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_proportion_shift",
              "cell_identity_shift",
              "composition_dominant_state_secondary",
              "both_state_and_composition_equally"
            ]
          },
          {
            "name": "cf_biggest_inflammatory_signal_not_mapk",
            "role": "hard_fail",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "jak_stat",
              "nfkb",
              "tnfa"
            ]
          },
          {
            "name": "cf_senescence_misattribution",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_senescence_signature_specifically"
            ]
          },
          {
            "name": "cf_dismiss_as_batch_artifact",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_batch_or_technical_artifact"
            ]
          },
          {
            "name": "cf_mapk_comparable_constitutive_kras_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "comparable_across_ages"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-7-claude-code",
      "modelName": "claude-opus-4-7",
      "harness": "claude-code",
      "task": "gse297023_q3_age_retained_mapk_reduced",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 1299,
      "cost": 3.7783419999999994,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "age_retained",
            "role": "gate",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_retained"
            ]
          },
          {
            "name": "identity_is_aging",
            "role": "gate",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_aging_signatures"
            ]
          },
          {
            "name": "nature_state_shift",
            "role": "gate",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_state_expression_shift"
            ]
          },
          {
            "name": "pathway_is_mapk",
            "role": "gate",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "mapk"
            ]
          },
          {
            "name": "pathway_reduced_in_aged",
            "role": "gate",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "reduced_in_aged"
            ]
          },
          {
            "name": "cf_mapk_increased_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_in_aged"
            ]
          },
          {
            "name": "cf_age_erased",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_erased"
            ]
          },
          {
            "name": "cf_pervasive_overcall",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_pervasive_artifactual"
            ]
          },
          {
            "name": "cf_explain_away_aging",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_compositional_or_identity",
            "role": "hard_fail",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_proportion_shift",
              "cell_identity_shift",
              "composition_dominant_state_secondary",
              "both_state_and_composition_equally"
            ]
          },
          {
            "name": "cf_biggest_inflammatory_signal_not_mapk",
            "role": "hard_fail",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "jak_stat",
              "nfkb",
              "tnfa"
            ]
          },
          {
            "name": "cf_senescence_misattribution",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_senescence_signature_specifically"
            ]
          },
          {
            "name": "cf_dismiss_as_batch_artifact",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_batch_or_technical_artifact"
            ]
          },
          {
            "name": "cf_mapk_comparable_constitutive_kras_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "comparable_across_ages"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-7-claude-code",
      "modelName": "claude-opus-4-7",
      "harness": "claude-code",
      "task": "gse297023_q3_age_retained_mapk_reduced",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 2662,
      "cost": 4.2929682499999995,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "age_retained",
            "role": "gate",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_retained"
            ]
          },
          {
            "name": "identity_is_aging",
            "role": "gate",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_aging_signatures"
            ]
          },
          {
            "name": "nature_state_shift",
            "role": "gate",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_state_expression_shift"
            ]
          },
          {
            "name": "pathway_is_mapk",
            "role": "gate",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "mapk"
            ]
          },
          {
            "name": "pathway_reduced_in_aged",
            "role": "gate",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "reduced_in_aged"
            ]
          },
          {
            "name": "cf_mapk_increased_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_in_aged"
            ]
          },
          {
            "name": "cf_age_erased",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_erased"
            ]
          },
          {
            "name": "cf_pervasive_overcall",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_pervasive_artifactual"
            ]
          },
          {
            "name": "cf_explain_away_aging",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_compositional_or_identity",
            "role": "hard_fail",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_proportion_shift",
              "cell_identity_shift",
              "composition_dominant_state_secondary",
              "both_state_and_composition_equally"
            ]
          },
          {
            "name": "cf_biggest_inflammatory_signal_not_mapk",
            "role": "hard_fail",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "jak_stat",
              "nfkb",
              "tnfa"
            ]
          },
          {
            "name": "cf_senescence_misattribution",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_senescence_signature_specifically"
            ]
          },
          {
            "name": "cf_dismiss_as_batch_artifact",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_batch_or_technical_artifact"
            ]
          },
          {
            "name": "cf_mapk_comparable_constitutive_kras_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "comparable_across_ages"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-7-claude-code",
      "modelName": "claude-opus-4-7",
      "harness": "claude-code",
      "task": "gse297023_q3_age_retained_mapk_reduced",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1592,
      "cost": 3.70865325,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "age_retained",
            "role": "gate",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_retained"
            ]
          },
          {
            "name": "identity_is_aging",
            "role": "gate",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_aging_signatures"
            ]
          },
          {
            "name": "nature_state_shift",
            "role": "gate",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_state_expression_shift"
            ]
          },
          {
            "name": "pathway_is_mapk",
            "role": "gate",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "mapk"
            ]
          },
          {
            "name": "pathway_reduced_in_aged",
            "role": "gate",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "reduced_in_aged"
            ]
          },
          {
            "name": "cf_mapk_increased_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_in_aged"
            ]
          },
          {
            "name": "cf_age_erased",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_erased"
            ]
          },
          {
            "name": "cf_pervasive_overcall",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_pervasive_artifactual"
            ]
          },
          {
            "name": "cf_explain_away_aging",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_compositional_or_identity",
            "role": "hard_fail",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_proportion_shift",
              "cell_identity_shift",
              "composition_dominant_state_secondary",
              "both_state_and_composition_equally"
            ]
          },
          {
            "name": "cf_biggest_inflammatory_signal_not_mapk",
            "role": "hard_fail",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "jak_stat",
              "nfkb",
              "tnfa"
            ]
          },
          {
            "name": "cf_senescence_misattribution",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_senescence_signature_specifically"
            ]
          },
          {
            "name": "cf_dismiss_as_batch_artifact",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_batch_or_technical_artifact"
            ]
          },
          {
            "name": "cf_mapk_comparable_constitutive_kras_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "comparable_across_ages"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-7-pi",
      "modelName": "claude-opus-4-7",
      "harness": "pi",
      "task": "gse297023_q3_age_retained_mapk_reduced",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 2110,
      "cost": 1.9294904999999998,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "age_retained",
            "role": "gate",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_retained"
            ]
          },
          {
            "name": "identity_is_aging",
            "role": "gate",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_aging_signatures"
            ]
          },
          {
            "name": "nature_state_shift",
            "role": "gate",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_state_expression_shift"
            ]
          },
          {
            "name": "pathway_is_mapk",
            "role": "gate",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "mapk"
            ]
          },
          {
            "name": "pathway_reduced_in_aged",
            "role": "gate",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "reduced_in_aged"
            ]
          },
          {
            "name": "cf_mapk_increased_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_in_aged"
            ]
          },
          {
            "name": "cf_age_erased",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_erased"
            ]
          },
          {
            "name": "cf_pervasive_overcall",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_pervasive_artifactual"
            ]
          },
          {
            "name": "cf_explain_away_aging",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_compositional_or_identity",
            "role": "hard_fail",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_proportion_shift",
              "cell_identity_shift",
              "composition_dominant_state_secondary",
              "both_state_and_composition_equally"
            ]
          },
          {
            "name": "cf_biggest_inflammatory_signal_not_mapk",
            "role": "hard_fail",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "jak_stat",
              "nfkb",
              "tnfa"
            ]
          },
          {
            "name": "cf_senescence_misattribution",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_senescence_signature_specifically"
            ]
          },
          {
            "name": "cf_dismiss_as_batch_artifact",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_batch_or_technical_artifact"
            ]
          },
          {
            "name": "cf_mapk_comparable_constitutive_kras_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "comparable_across_ages"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-7-pi",
      "modelName": "claude-opus-4-7",
      "harness": "pi",
      "task": "gse297023_q3_age_retained_mapk_reduced",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 2,
      "passed": true,
      "score": null,
      "durationS": 1231,
      "cost": 0.859317,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "age_retained",
            "role": "gate",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_retained"
            ]
          },
          {
            "name": "identity_is_aging",
            "role": "gate",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_aging_signatures"
            ]
          },
          {
            "name": "nature_state_shift",
            "role": "gate",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_state_expression_shift"
            ]
          },
          {
            "name": "pathway_is_mapk",
            "role": "gate",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "mapk"
            ]
          },
          {
            "name": "pathway_reduced_in_aged",
            "role": "gate",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "reduced_in_aged"
            ]
          },
          {
            "name": "cf_mapk_increased_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_in_aged"
            ]
          },
          {
            "name": "cf_age_erased",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_erased"
            ]
          },
          {
            "name": "cf_pervasive_overcall",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_pervasive_artifactual"
            ]
          },
          {
            "name": "cf_explain_away_aging",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_compositional_or_identity",
            "role": "hard_fail",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_proportion_shift",
              "cell_identity_shift",
              "composition_dominant_state_secondary",
              "both_state_and_composition_equally"
            ]
          },
          {
            "name": "cf_biggest_inflammatory_signal_not_mapk",
            "role": "hard_fail",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "jak_stat",
              "nfkb",
              "tnfa"
            ]
          },
          {
            "name": "cf_senescence_misattribution",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_senescence_signature_specifically"
            ]
          },
          {
            "name": "cf_dismiss_as_batch_artifact",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_batch_or_technical_artifact"
            ]
          },
          {
            "name": "cf_mapk_comparable_constitutive_kras_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "comparable_across_ages"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-7-pi",
      "modelName": "claude-opus-4-7",
      "harness": "pi",
      "task": "gse297023_q3_age_retained_mapk_reduced",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1001,
      "cost": 1.2246455,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "age_retained",
            "role": "gate",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_retained"
            ]
          },
          {
            "name": "identity_is_aging",
            "role": "gate",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_aging_signatures"
            ]
          },
          {
            "name": "nature_state_shift",
            "role": "gate",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_state_expression_shift"
            ]
          },
          {
            "name": "pathway_is_mapk",
            "role": "gate",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "mapk"
            ]
          },
          {
            "name": "pathway_reduced_in_aged",
            "role": "gate",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "reduced_in_aged"
            ]
          },
          {
            "name": "cf_mapk_increased_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_in_aged"
            ]
          },
          {
            "name": "cf_age_erased",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_erased"
            ]
          },
          {
            "name": "cf_pervasive_overcall",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_pervasive_artifactual"
            ]
          },
          {
            "name": "cf_explain_away_aging",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_compositional_or_identity",
            "role": "hard_fail",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_proportion_shift",
              "cell_identity_shift",
              "composition_dominant_state_secondary",
              "both_state_and_composition_equally"
            ]
          },
          {
            "name": "cf_biggest_inflammatory_signal_not_mapk",
            "role": "hard_fail",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "jak_stat",
              "nfkb",
              "tnfa"
            ]
          },
          {
            "name": "cf_senescence_misattribution",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_senescence_signature_specifically"
            ]
          },
          {
            "name": "cf_dismiss_as_batch_artifact",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_batch_or_technical_artifact"
            ]
          },
          {
            "name": "cf_mapk_comparable_constitutive_kras_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "comparable_across_ages"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-8-claude-code",
      "modelName": "claude-opus-4-8",
      "harness": "claude-code",
      "task": "gse297023_q3_age_retained_mapk_reduced",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 1970,
      "cost": 4.3054517500000005,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "age_retained",
            "role": "gate",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_retained"
            ]
          },
          {
            "name": "identity_is_aging",
            "role": "gate",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_aging_signatures"
            ]
          },
          {
            "name": "nature_state_shift",
            "role": "gate",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_state_expression_shift"
            ]
          },
          {
            "name": "pathway_is_mapk",
            "role": "gate",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "mapk"
            ]
          },
          {
            "name": "pathway_reduced_in_aged",
            "role": "gate",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "reduced_in_aged"
            ]
          },
          {
            "name": "cf_mapk_increased_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_in_aged"
            ]
          },
          {
            "name": "cf_age_erased",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_erased"
            ]
          },
          {
            "name": "cf_pervasive_overcall",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_pervasive_artifactual"
            ]
          },
          {
            "name": "cf_explain_away_aging",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_compositional_or_identity",
            "role": "hard_fail",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_proportion_shift",
              "cell_identity_shift",
              "composition_dominant_state_secondary",
              "both_state_and_composition_equally"
            ]
          },
          {
            "name": "cf_biggest_inflammatory_signal_not_mapk",
            "role": "hard_fail",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "jak_stat",
              "nfkb",
              "tnfa"
            ]
          },
          {
            "name": "cf_senescence_misattribution",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_senescence_signature_specifically"
            ]
          },
          {
            "name": "cf_dismiss_as_batch_artifact",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_batch_or_technical_artifact"
            ]
          },
          {
            "name": "cf_mapk_comparable_constitutive_kras_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "comparable_across_ages"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-8-claude-code",
      "modelName": "claude-opus-4-8",
      "harness": "claude-code",
      "task": "gse297023_q3_age_retained_mapk_reduced",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 2,
      "passed": true,
      "score": null,
      "durationS": 3418,
      "cost": 12.617745000000001,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "age_retained",
            "role": "gate",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_retained"
            ]
          },
          {
            "name": "identity_is_aging",
            "role": "gate",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_aging_signatures"
            ]
          },
          {
            "name": "nature_state_shift",
            "role": "gate",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_state_expression_shift"
            ]
          },
          {
            "name": "pathway_is_mapk",
            "role": "gate",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "mapk"
            ]
          },
          {
            "name": "pathway_reduced_in_aged",
            "role": "gate",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "reduced_in_aged"
            ]
          },
          {
            "name": "cf_mapk_increased_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_in_aged"
            ]
          },
          {
            "name": "cf_age_erased",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_erased"
            ]
          },
          {
            "name": "cf_pervasive_overcall",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_pervasive_artifactual"
            ]
          },
          {
            "name": "cf_explain_away_aging",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_compositional_or_identity",
            "role": "hard_fail",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_proportion_shift",
              "cell_identity_shift",
              "composition_dominant_state_secondary",
              "both_state_and_composition_equally"
            ]
          },
          {
            "name": "cf_biggest_inflammatory_signal_not_mapk",
            "role": "hard_fail",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "jak_stat",
              "nfkb",
              "tnfa"
            ]
          },
          {
            "name": "cf_senescence_misattribution",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_senescence_signature_specifically"
            ]
          },
          {
            "name": "cf_dismiss_as_batch_artifact",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_batch_or_technical_artifact"
            ]
          },
          {
            "name": "cf_mapk_comparable_constitutive_kras_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "comparable_across_ages"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-8-claude-code",
      "modelName": "claude-opus-4-8",
      "harness": "claude-code",
      "task": "gse297023_q3_age_retained_mapk_reduced",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 3,
      "passed": true,
      "score": null,
      "durationS": 2156,
      "cost": 4.947156750000001,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "age_retained",
            "role": "gate",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_retained"
            ]
          },
          {
            "name": "identity_is_aging",
            "role": "gate",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_aging_signatures"
            ]
          },
          {
            "name": "nature_state_shift",
            "role": "gate",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_state_expression_shift"
            ]
          },
          {
            "name": "pathway_is_mapk",
            "role": "gate",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "mapk"
            ]
          },
          {
            "name": "pathway_reduced_in_aged",
            "role": "gate",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "reduced_in_aged"
            ]
          },
          {
            "name": "cf_mapk_increased_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_in_aged"
            ]
          },
          {
            "name": "cf_age_erased",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_erased"
            ]
          },
          {
            "name": "cf_pervasive_overcall",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_pervasive_artifactual"
            ]
          },
          {
            "name": "cf_explain_away_aging",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_compositional_or_identity",
            "role": "hard_fail",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_proportion_shift",
              "cell_identity_shift",
              "composition_dominant_state_secondary",
              "both_state_and_composition_equally"
            ]
          },
          {
            "name": "cf_biggest_inflammatory_signal_not_mapk",
            "role": "hard_fail",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "jak_stat",
              "nfkb",
              "tnfa"
            ]
          },
          {
            "name": "cf_senescence_misattribution",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_senescence_signature_specifically"
            ]
          },
          {
            "name": "cf_dismiss_as_batch_artifact",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_batch_or_technical_artifact"
            ]
          },
          {
            "name": "cf_mapk_comparable_constitutive_kras_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "comparable_across_ages"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-8-pi",
      "modelName": "claude-opus-4-8",
      "harness": "pi",
      "task": "gse297023_q3_age_retained_mapk_reduced",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 1,
      "passed": true,
      "score": null,
      "durationS": 1529,
      "cost": 2.4637205000000004,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "age_retained",
            "role": "gate",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_retained"
            ]
          },
          {
            "name": "identity_is_aging",
            "role": "gate",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_aging_signatures"
            ]
          },
          {
            "name": "nature_state_shift",
            "role": "gate",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_state_expression_shift"
            ]
          },
          {
            "name": "pathway_is_mapk",
            "role": "gate",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "mapk"
            ]
          },
          {
            "name": "pathway_reduced_in_aged",
            "role": "gate",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "reduced_in_aged"
            ]
          },
          {
            "name": "cf_mapk_increased_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_in_aged"
            ]
          },
          {
            "name": "cf_age_erased",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_erased"
            ]
          },
          {
            "name": "cf_pervasive_overcall",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_pervasive_artifactual"
            ]
          },
          {
            "name": "cf_explain_away_aging",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_compositional_or_identity",
            "role": "hard_fail",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_proportion_shift",
              "cell_identity_shift",
              "composition_dominant_state_secondary",
              "both_state_and_composition_equally"
            ]
          },
          {
            "name": "cf_biggest_inflammatory_signal_not_mapk",
            "role": "hard_fail",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "jak_stat",
              "nfkb",
              "tnfa"
            ]
          },
          {
            "name": "cf_senescence_misattribution",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_senescence_signature_specifically"
            ]
          },
          {
            "name": "cf_dismiss_as_batch_artifact",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_batch_or_technical_artifact"
            ]
          },
          {
            "name": "cf_mapk_comparable_constitutive_kras_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "comparable_across_ages"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-8-pi",
      "modelName": "claude-opus-4-8",
      "harness": "pi",
      "task": "gse297023_q3_age_retained_mapk_reduced",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 1151,
      "cost": 1.476149,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "age_retained",
            "role": "gate",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_retained"
            ]
          },
          {
            "name": "identity_is_aging",
            "role": "gate",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_aging_signatures"
            ]
          },
          {
            "name": "nature_state_shift",
            "role": "gate",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_state_expression_shift"
            ]
          },
          {
            "name": "pathway_is_mapk",
            "role": "gate",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "mapk"
            ]
          },
          {
            "name": "pathway_reduced_in_aged",
            "role": "gate",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "reduced_in_aged"
            ]
          },
          {
            "name": "cf_mapk_increased_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_in_aged"
            ]
          },
          {
            "name": "cf_age_erased",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_erased"
            ]
          },
          {
            "name": "cf_pervasive_overcall",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_pervasive_artifactual"
            ]
          },
          {
            "name": "cf_explain_away_aging",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_compositional_or_identity",
            "role": "hard_fail",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_proportion_shift",
              "cell_identity_shift",
              "composition_dominant_state_secondary",
              "both_state_and_composition_equally"
            ]
          },
          {
            "name": "cf_biggest_inflammatory_signal_not_mapk",
            "role": "hard_fail",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "jak_stat",
              "nfkb",
              "tnfa"
            ]
          },
          {
            "name": "cf_senescence_misattribution",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_senescence_signature_specifically"
            ]
          },
          {
            "name": "cf_dismiss_as_batch_artifact",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_batch_or_technical_artifact"
            ]
          },
          {
            "name": "cf_mapk_comparable_constitutive_kras_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "comparable_across_ages"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-8-pi",
      "modelName": "claude-opus-4-8",
      "harness": "pi",
      "task": "gse297023_q3_age_retained_mapk_reduced",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1981,
      "cost": 2.55565825,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "age_retained",
            "role": "gate",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_retained"
            ]
          },
          {
            "name": "identity_is_aging",
            "role": "gate",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_aging_signatures"
            ]
          },
          {
            "name": "nature_state_shift",
            "role": "gate",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_state_expression_shift"
            ]
          },
          {
            "name": "pathway_is_mapk",
            "role": "gate",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "mapk"
            ]
          },
          {
            "name": "pathway_reduced_in_aged",
            "role": "gate",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "reduced_in_aged"
            ]
          },
          {
            "name": "cf_mapk_increased_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_in_aged"
            ]
          },
          {
            "name": "cf_age_erased",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_erased"
            ]
          },
          {
            "name": "cf_pervasive_overcall",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_pervasive_artifactual"
            ]
          },
          {
            "name": "cf_explain_away_aging",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_compositional_or_identity",
            "role": "hard_fail",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_proportion_shift",
              "cell_identity_shift",
              "composition_dominant_state_secondary",
              "both_state_and_composition_equally"
            ]
          },
          {
            "name": "cf_biggest_inflammatory_signal_not_mapk",
            "role": "hard_fail",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "jak_stat",
              "nfkb",
              "tnfa"
            ]
          },
          {
            "name": "cf_senescence_misattribution",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_senescence_signature_specifically"
            ]
          },
          {
            "name": "cf_dismiss_as_batch_artifact",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_batch_or_technical_artifact"
            ]
          },
          {
            "name": "cf_mapk_comparable_constitutive_kras_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "comparable_across_ages"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-4-6-pi",
      "modelName": "claude-sonnet-4-6",
      "harness": "pi",
      "task": "gse297023_q3_age_retained_mapk_reduced",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 4042,
      "cost": 2.04781215,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "age_retained",
            "role": "gate",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_retained"
            ]
          },
          {
            "name": "identity_is_aging",
            "role": "gate",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_aging_signatures"
            ]
          },
          {
            "name": "nature_state_shift",
            "role": "gate",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_state_expression_shift"
            ]
          },
          {
            "name": "pathway_is_mapk",
            "role": "gate",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "mapk"
            ]
          },
          {
            "name": "pathway_reduced_in_aged",
            "role": "gate",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "reduced_in_aged"
            ]
          },
          {
            "name": "cf_mapk_increased_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_in_aged"
            ]
          },
          {
            "name": "cf_age_erased",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_erased"
            ]
          },
          {
            "name": "cf_pervasive_overcall",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_pervasive_artifactual"
            ]
          },
          {
            "name": "cf_explain_away_aging",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_compositional_or_identity",
            "role": "hard_fail",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_proportion_shift",
              "cell_identity_shift",
              "composition_dominant_state_secondary",
              "both_state_and_composition_equally"
            ]
          },
          {
            "name": "cf_biggest_inflammatory_signal_not_mapk",
            "role": "hard_fail",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "jak_stat",
              "nfkb",
              "tnfa"
            ]
          },
          {
            "name": "cf_senescence_misattribution",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_senescence_signature_specifically"
            ]
          },
          {
            "name": "cf_dismiss_as_batch_artifact",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_batch_or_technical_artifact"
            ]
          },
          {
            "name": "cf_mapk_comparable_constitutive_kras_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "comparable_across_ages"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-4-6-pi",
      "modelName": "claude-sonnet-4-6",
      "harness": "pi",
      "task": "gse297023_q3_age_retained_mapk_reduced",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 2306,
      "cost": 1.3744347,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "age_retained",
            "role": "gate",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_retained"
            ]
          },
          {
            "name": "identity_is_aging",
            "role": "gate",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_aging_signatures"
            ]
          },
          {
            "name": "nature_state_shift",
            "role": "gate",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_state_expression_shift"
            ]
          },
          {
            "name": "pathway_is_mapk",
            "role": "gate",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "mapk"
            ]
          },
          {
            "name": "pathway_reduced_in_aged",
            "role": "gate",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "reduced_in_aged"
            ]
          },
          {
            "name": "cf_mapk_increased_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_in_aged"
            ]
          },
          {
            "name": "cf_age_erased",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_erased"
            ]
          },
          {
            "name": "cf_pervasive_overcall",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_pervasive_artifactual"
            ]
          },
          {
            "name": "cf_explain_away_aging",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_compositional_or_identity",
            "role": "hard_fail",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_proportion_shift",
              "cell_identity_shift",
              "composition_dominant_state_secondary",
              "both_state_and_composition_equally"
            ]
          },
          {
            "name": "cf_biggest_inflammatory_signal_not_mapk",
            "role": "hard_fail",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "jak_stat",
              "nfkb",
              "tnfa"
            ]
          },
          {
            "name": "cf_senescence_misattribution",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_senescence_signature_specifically"
            ]
          },
          {
            "name": "cf_dismiss_as_batch_artifact",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_batch_or_technical_artifact"
            ]
          },
          {
            "name": "cf_mapk_comparable_constitutive_kras_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "comparable_across_ages"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-4-6-pi",
      "modelName": "claude-sonnet-4-6",
      "harness": "pi",
      "task": "gse297023_q3_age_retained_mapk_reduced",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 2378,
      "cost": 0.9966496500000002,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "age_retained",
            "role": "gate",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_retained"
            ]
          },
          {
            "name": "identity_is_aging",
            "role": "gate",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_aging_signatures"
            ]
          },
          {
            "name": "nature_state_shift",
            "role": "gate",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_state_expression_shift"
            ]
          },
          {
            "name": "pathway_is_mapk",
            "role": "gate",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "mapk"
            ]
          },
          {
            "name": "pathway_reduced_in_aged",
            "role": "gate",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "reduced_in_aged"
            ]
          },
          {
            "name": "cf_mapk_increased_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_in_aged"
            ]
          },
          {
            "name": "cf_age_erased",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_erased"
            ]
          },
          {
            "name": "cf_pervasive_overcall",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_pervasive_artifactual"
            ]
          },
          {
            "name": "cf_explain_away_aging",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_compositional_or_identity",
            "role": "hard_fail",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_proportion_shift",
              "cell_identity_shift",
              "composition_dominant_state_secondary",
              "both_state_and_composition_equally"
            ]
          },
          {
            "name": "cf_biggest_inflammatory_signal_not_mapk",
            "role": "hard_fail",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "jak_stat",
              "nfkb",
              "tnfa"
            ]
          },
          {
            "name": "cf_senescence_misattribution",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_senescence_signature_specifically"
            ]
          },
          {
            "name": "cf_dismiss_as_batch_artifact",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_batch_or_technical_artifact"
            ]
          },
          {
            "name": "cf_mapk_comparable_constitutive_kras_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "comparable_across_ages"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gemini-3-1-pro-preview-pi",
      "modelName": "gemini-3.1-pro-preview",
      "harness": "pi",
      "task": "gse297023_q3_age_retained_mapk_reduced",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 2584,
      "cost": 1.1052503999999999,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "age_retained",
            "role": "gate",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_retained"
            ]
          },
          {
            "name": "identity_is_aging",
            "role": "gate",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_aging_signatures"
            ]
          },
          {
            "name": "nature_state_shift",
            "role": "gate",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_state_expression_shift"
            ]
          },
          {
            "name": "pathway_is_mapk",
            "role": "gate",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "mapk"
            ]
          },
          {
            "name": "pathway_reduced_in_aged",
            "role": "gate",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "reduced_in_aged"
            ]
          },
          {
            "name": "cf_mapk_increased_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_in_aged"
            ]
          },
          {
            "name": "cf_age_erased",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_erased"
            ]
          },
          {
            "name": "cf_pervasive_overcall",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_pervasive_artifactual"
            ]
          },
          {
            "name": "cf_explain_away_aging",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_compositional_or_identity",
            "role": "hard_fail",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_proportion_shift",
              "cell_identity_shift",
              "composition_dominant_state_secondary",
              "both_state_and_composition_equally"
            ]
          },
          {
            "name": "cf_biggest_inflammatory_signal_not_mapk",
            "role": "hard_fail",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "jak_stat",
              "nfkb",
              "tnfa"
            ]
          },
          {
            "name": "cf_senescence_misattribution",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_senescence_signature_specifically"
            ]
          },
          {
            "name": "cf_dismiss_as_batch_artifact",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_batch_or_technical_artifact"
            ]
          },
          {
            "name": "cf_mapk_comparable_constitutive_kras_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "comparable_across_ages"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gemini-3-1-pro-preview-pi",
      "modelName": "gemini-3.1-pro-preview",
      "harness": "pi",
      "task": "gse297023_q3_age_retained_mapk_reduced",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 2313,
      "cost": 1.3726919999999998,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "age_retained",
            "role": "gate",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_retained"
            ]
          },
          {
            "name": "identity_is_aging",
            "role": "gate",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_aging_signatures"
            ]
          },
          {
            "name": "nature_state_shift",
            "role": "gate",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_state_expression_shift"
            ]
          },
          {
            "name": "pathway_is_mapk",
            "role": "gate",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "mapk"
            ]
          },
          {
            "name": "pathway_reduced_in_aged",
            "role": "gate",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "reduced_in_aged"
            ]
          },
          {
            "name": "cf_mapk_increased_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_in_aged"
            ]
          },
          {
            "name": "cf_age_erased",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_erased"
            ]
          },
          {
            "name": "cf_pervasive_overcall",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_pervasive_artifactual"
            ]
          },
          {
            "name": "cf_explain_away_aging",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_compositional_or_identity",
            "role": "hard_fail",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_proportion_shift",
              "cell_identity_shift",
              "composition_dominant_state_secondary",
              "both_state_and_composition_equally"
            ]
          },
          {
            "name": "cf_biggest_inflammatory_signal_not_mapk",
            "role": "hard_fail",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "jak_stat",
              "nfkb",
              "tnfa"
            ]
          },
          {
            "name": "cf_senescence_misattribution",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_senescence_signature_specifically"
            ]
          },
          {
            "name": "cf_dismiss_as_batch_artifact",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_batch_or_technical_artifact"
            ]
          },
          {
            "name": "cf_mapk_comparable_constitutive_kras_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "comparable_across_ages"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gemini-3-1-pro-preview-pi",
      "modelName": "gemini-3.1-pro-preview",
      "harness": "pi",
      "task": "gse297023_q3_age_retained_mapk_reduced",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 3,
      "passed": true,
      "score": null,
      "durationS": 1191,
      "cost": 0.6352176000000002,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "age_retained",
            "role": "gate",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_retained"
            ]
          },
          {
            "name": "identity_is_aging",
            "role": "gate",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_aging_signatures"
            ]
          },
          {
            "name": "nature_state_shift",
            "role": "gate",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_state_expression_shift"
            ]
          },
          {
            "name": "pathway_is_mapk",
            "role": "gate",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "mapk"
            ]
          },
          {
            "name": "pathway_reduced_in_aged",
            "role": "gate",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "reduced_in_aged"
            ]
          },
          {
            "name": "cf_mapk_increased_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_in_aged"
            ]
          },
          {
            "name": "cf_age_erased",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_erased"
            ]
          },
          {
            "name": "cf_pervasive_overcall",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_pervasive_artifactual"
            ]
          },
          {
            "name": "cf_explain_away_aging",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_compositional_or_identity",
            "role": "hard_fail",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_proportion_shift",
              "cell_identity_shift",
              "composition_dominant_state_secondary",
              "both_state_and_composition_equally"
            ]
          },
          {
            "name": "cf_biggest_inflammatory_signal_not_mapk",
            "role": "hard_fail",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "jak_stat",
              "nfkb",
              "tnfa"
            ]
          },
          {
            "name": "cf_senescence_misattribution",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_senescence_signature_specifically"
            ]
          },
          {
            "name": "cf_dismiss_as_batch_artifact",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_batch_or_technical_artifact"
            ]
          },
          {
            "name": "cf_mapk_comparable_constitutive_kras_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "comparable_across_ages"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gemini-3-5-flash-pi",
      "modelName": "gemini-3.5-flash",
      "harness": "pi",
      "task": "gse297023_q3_age_retained_mapk_reduced",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 1,
      "passed": true,
      "score": null,
      "durationS": 309,
      "cost": 0.9246629999999998,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "age_retained",
            "role": "gate",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_retained"
            ]
          },
          {
            "name": "identity_is_aging",
            "role": "gate",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_aging_signatures"
            ]
          },
          {
            "name": "nature_state_shift",
            "role": "gate",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_state_expression_shift"
            ]
          },
          {
            "name": "pathway_is_mapk",
            "role": "gate",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "mapk"
            ]
          },
          {
            "name": "pathway_reduced_in_aged",
            "role": "gate",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "reduced_in_aged"
            ]
          },
          {
            "name": "cf_mapk_increased_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_in_aged"
            ]
          },
          {
            "name": "cf_age_erased",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_erased"
            ]
          },
          {
            "name": "cf_pervasive_overcall",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_pervasive_artifactual"
            ]
          },
          {
            "name": "cf_explain_away_aging",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_compositional_or_identity",
            "role": "hard_fail",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_proportion_shift",
              "cell_identity_shift",
              "composition_dominant_state_secondary",
              "both_state_and_composition_equally"
            ]
          },
          {
            "name": "cf_biggest_inflammatory_signal_not_mapk",
            "role": "hard_fail",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "jak_stat",
              "nfkb",
              "tnfa"
            ]
          },
          {
            "name": "cf_senescence_misattribution",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_senescence_signature_specifically"
            ]
          },
          {
            "name": "cf_dismiss_as_batch_artifact",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_batch_or_technical_artifact"
            ]
          },
          {
            "name": "cf_mapk_comparable_constitutive_kras_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "comparable_across_ages"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gemini-3-5-flash-pi",
      "modelName": "gemini-3.5-flash",
      "harness": "pi",
      "task": "gse297023_q3_age_retained_mapk_reduced",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 2,
      "passed": true,
      "score": null,
      "durationS": 171,
      "cost": 0.5080909499999999,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "age_retained",
            "role": "gate",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_retained"
            ]
          },
          {
            "name": "identity_is_aging",
            "role": "gate",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_aging_signatures"
            ]
          },
          {
            "name": "nature_state_shift",
            "role": "gate",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_state_expression_shift"
            ]
          },
          {
            "name": "pathway_is_mapk",
            "role": "gate",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "mapk"
            ]
          },
          {
            "name": "pathway_reduced_in_aged",
            "role": "gate",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "reduced_in_aged"
            ]
          },
          {
            "name": "cf_mapk_increased_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_in_aged"
            ]
          },
          {
            "name": "cf_age_erased",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_erased"
            ]
          },
          {
            "name": "cf_pervasive_overcall",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_pervasive_artifactual"
            ]
          },
          {
            "name": "cf_explain_away_aging",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_compositional_or_identity",
            "role": "hard_fail",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_proportion_shift",
              "cell_identity_shift",
              "composition_dominant_state_secondary",
              "both_state_and_composition_equally"
            ]
          },
          {
            "name": "cf_biggest_inflammatory_signal_not_mapk",
            "role": "hard_fail",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "jak_stat",
              "nfkb",
              "tnfa"
            ]
          },
          {
            "name": "cf_senescence_misattribution",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_senescence_signature_specifically"
            ]
          },
          {
            "name": "cf_dismiss_as_batch_artifact",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_batch_or_technical_artifact"
            ]
          },
          {
            "name": "cf_mapk_comparable_constitutive_kras_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "comparable_across_ages"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gemini-3-5-flash-pi",
      "modelName": "gemini-3.5-flash",
      "harness": "pi",
      "task": "gse297023_q3_age_retained_mapk_reduced",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1178,
      "cost": 0.546252,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "age_retained",
            "role": "gate",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_retained"
            ]
          },
          {
            "name": "identity_is_aging",
            "role": "gate",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_aging_signatures"
            ]
          },
          {
            "name": "nature_state_shift",
            "role": "gate",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_state_expression_shift"
            ]
          },
          {
            "name": "pathway_is_mapk",
            "role": "gate",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "mapk"
            ]
          },
          {
            "name": "pathway_reduced_in_aged",
            "role": "gate",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "reduced_in_aged"
            ]
          },
          {
            "name": "cf_mapk_increased_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_in_aged"
            ]
          },
          {
            "name": "cf_age_erased",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_erased"
            ]
          },
          {
            "name": "cf_pervasive_overcall",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_pervasive_artifactual"
            ]
          },
          {
            "name": "cf_explain_away_aging",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_compositional_or_identity",
            "role": "hard_fail",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_proportion_shift",
              "cell_identity_shift",
              "composition_dominant_state_secondary",
              "both_state_and_composition_equally"
            ]
          },
          {
            "name": "cf_biggest_inflammatory_signal_not_mapk",
            "role": "hard_fail",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "jak_stat",
              "nfkb",
              "tnfa"
            ]
          },
          {
            "name": "cf_senescence_misattribution",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_senescence_signature_specifically"
            ]
          },
          {
            "name": "cf_dismiss_as_batch_artifact",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_batch_or_technical_artifact"
            ]
          },
          {
            "name": "cf_mapk_comparable_constitutive_kras_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "comparable_across_ages"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-glm-5p2-pi",
      "modelName": "glm-5p2",
      "harness": "pi",
      "task": "gse297023_q3_age_retained_mapk_reduced",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 5074,
      "cost": 3.622836000000001,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "age_retained",
            "role": "gate",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_retained"
            ]
          },
          {
            "name": "identity_is_aging",
            "role": "gate",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_aging_signatures"
            ]
          },
          {
            "name": "nature_state_shift",
            "role": "gate",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_state_expression_shift"
            ]
          },
          {
            "name": "pathway_is_mapk",
            "role": "gate",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "mapk"
            ]
          },
          {
            "name": "pathway_reduced_in_aged",
            "role": "gate",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "reduced_in_aged"
            ]
          },
          {
            "name": "cf_mapk_increased_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_in_aged"
            ]
          },
          {
            "name": "cf_age_erased",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_erased"
            ]
          },
          {
            "name": "cf_pervasive_overcall",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_pervasive_artifactual"
            ]
          },
          {
            "name": "cf_explain_away_aging",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_compositional_or_identity",
            "role": "hard_fail",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_proportion_shift",
              "cell_identity_shift",
              "composition_dominant_state_secondary",
              "both_state_and_composition_equally"
            ]
          },
          {
            "name": "cf_biggest_inflammatory_signal_not_mapk",
            "role": "hard_fail",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "jak_stat",
              "nfkb",
              "tnfa"
            ]
          },
          {
            "name": "cf_senescence_misattribution",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_senescence_signature_specifically"
            ]
          },
          {
            "name": "cf_dismiss_as_batch_artifact",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_batch_or_technical_artifact"
            ]
          },
          {
            "name": "cf_mapk_comparable_constitutive_kras_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "comparable_across_ages"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-glm-5p2-pi",
      "modelName": "glm-5p2",
      "harness": "pi",
      "task": "gse297023_q3_age_retained_mapk_reduced",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 6938,
      "cost": 5.79281724,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "age_retained",
            "role": "gate",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_retained"
            ]
          },
          {
            "name": "identity_is_aging",
            "role": "gate",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_aging_signatures"
            ]
          },
          {
            "name": "nature_state_shift",
            "role": "gate",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_state_expression_shift"
            ]
          },
          {
            "name": "pathway_is_mapk",
            "role": "gate",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "mapk"
            ]
          },
          {
            "name": "pathway_reduced_in_aged",
            "role": "gate",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "reduced_in_aged"
            ]
          },
          {
            "name": "cf_mapk_increased_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_in_aged"
            ]
          },
          {
            "name": "cf_age_erased",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_erased"
            ]
          },
          {
            "name": "cf_pervasive_overcall",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_pervasive_artifactual"
            ]
          },
          {
            "name": "cf_explain_away_aging",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_compositional_or_identity",
            "role": "hard_fail",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_proportion_shift",
              "cell_identity_shift",
              "composition_dominant_state_secondary",
              "both_state_and_composition_equally"
            ]
          },
          {
            "name": "cf_biggest_inflammatory_signal_not_mapk",
            "role": "hard_fail",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "jak_stat",
              "nfkb",
              "tnfa"
            ]
          },
          {
            "name": "cf_senescence_misattribution",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_senescence_signature_specifically"
            ]
          },
          {
            "name": "cf_dismiss_as_batch_artifact",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_batch_or_technical_artifact"
            ]
          },
          {
            "name": "cf_mapk_comparable_constitutive_kras_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "comparable_across_ages"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-glm-5p2-pi",
      "modelName": "glm-5p2",
      "harness": "pi",
      "task": "gse297023_q3_age_retained_mapk_reduced",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 5189,
      "cost": 2.2448005,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "age_retained",
            "role": "gate",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_retained"
            ]
          },
          {
            "name": "identity_is_aging",
            "role": "gate",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_aging_signatures"
            ]
          },
          {
            "name": "nature_state_shift",
            "role": "gate",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_state_expression_shift"
            ]
          },
          {
            "name": "pathway_is_mapk",
            "role": "gate",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "mapk"
            ]
          },
          {
            "name": "pathway_reduced_in_aged",
            "role": "gate",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "reduced_in_aged"
            ]
          },
          {
            "name": "cf_mapk_increased_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_in_aged"
            ]
          },
          {
            "name": "cf_age_erased",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_erased"
            ]
          },
          {
            "name": "cf_pervasive_overcall",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_pervasive_artifactual"
            ]
          },
          {
            "name": "cf_explain_away_aging",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_compositional_or_identity",
            "role": "hard_fail",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_proportion_shift",
              "cell_identity_shift",
              "composition_dominant_state_secondary",
              "both_state_and_composition_equally"
            ]
          },
          {
            "name": "cf_biggest_inflammatory_signal_not_mapk",
            "role": "hard_fail",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "jak_stat",
              "nfkb",
              "tnfa"
            ]
          },
          {
            "name": "cf_senescence_misattribution",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_senescence_signature_specifically"
            ]
          },
          {
            "name": "cf_dismiss_as_batch_artifact",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_batch_or_technical_artifact"
            ]
          },
          {
            "name": "cf_mapk_comparable_constitutive_kras_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "comparable_across_ages"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-4-codex",
      "modelName": "gpt-5.4",
      "harness": "openai-codex",
      "task": "gse297023_q3_age_retained_mapk_reduced",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 908,
      "cost": 2.154582,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "age_retained",
            "role": "gate",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_retained"
            ]
          },
          {
            "name": "identity_is_aging",
            "role": "gate",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_aging_signatures"
            ]
          },
          {
            "name": "nature_state_shift",
            "role": "gate",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_state_expression_shift"
            ]
          },
          {
            "name": "pathway_is_mapk",
            "role": "gate",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "mapk"
            ]
          },
          {
            "name": "pathway_reduced_in_aged",
            "role": "gate",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "reduced_in_aged"
            ]
          },
          {
            "name": "cf_mapk_increased_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_in_aged"
            ]
          },
          {
            "name": "cf_age_erased",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_erased"
            ]
          },
          {
            "name": "cf_pervasive_overcall",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_pervasive_artifactual"
            ]
          },
          {
            "name": "cf_explain_away_aging",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_compositional_or_identity",
            "role": "hard_fail",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_proportion_shift",
              "cell_identity_shift",
              "composition_dominant_state_secondary",
              "both_state_and_composition_equally"
            ]
          },
          {
            "name": "cf_biggest_inflammatory_signal_not_mapk",
            "role": "hard_fail",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "jak_stat",
              "nfkb",
              "tnfa"
            ]
          },
          {
            "name": "cf_senescence_misattribution",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_senescence_signature_specifically"
            ]
          },
          {
            "name": "cf_dismiss_as_batch_artifact",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_batch_or_technical_artifact"
            ]
          },
          {
            "name": "cf_mapk_comparable_constitutive_kras_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "comparable_across_ages"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-4-codex",
      "modelName": "gpt-5.4",
      "harness": "openai-codex",
      "task": "gse297023_q3_age_retained_mapk_reduced",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 1838,
      "cost": 3.8288415000000002,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "age_retained",
            "role": "gate",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_retained"
            ]
          },
          {
            "name": "identity_is_aging",
            "role": "gate",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_aging_signatures"
            ]
          },
          {
            "name": "nature_state_shift",
            "role": "gate",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_state_expression_shift"
            ]
          },
          {
            "name": "pathway_is_mapk",
            "role": "gate",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "mapk"
            ]
          },
          {
            "name": "pathway_reduced_in_aged",
            "role": "gate",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "reduced_in_aged"
            ]
          },
          {
            "name": "cf_mapk_increased_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_in_aged"
            ]
          },
          {
            "name": "cf_age_erased",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_erased"
            ]
          },
          {
            "name": "cf_pervasive_overcall",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_pervasive_artifactual"
            ]
          },
          {
            "name": "cf_explain_away_aging",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_compositional_or_identity",
            "role": "hard_fail",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_proportion_shift",
              "cell_identity_shift",
              "composition_dominant_state_secondary",
              "both_state_and_composition_equally"
            ]
          },
          {
            "name": "cf_biggest_inflammatory_signal_not_mapk",
            "role": "hard_fail",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "jak_stat",
              "nfkb",
              "tnfa"
            ]
          },
          {
            "name": "cf_senescence_misattribution",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_senescence_signature_specifically"
            ]
          },
          {
            "name": "cf_dismiss_as_batch_artifact",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_batch_or_technical_artifact"
            ]
          },
          {
            "name": "cf_mapk_comparable_constitutive_kras_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "comparable_across_ages"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-4-codex",
      "modelName": "gpt-5.4",
      "harness": "openai-codex",
      "task": "gse297023_q3_age_retained_mapk_reduced",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1520,
      "cost": 3.912985,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "age_retained",
            "role": "gate",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_retained"
            ]
          },
          {
            "name": "identity_is_aging",
            "role": "gate",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_aging_signatures"
            ]
          },
          {
            "name": "nature_state_shift",
            "role": "gate",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_state_expression_shift"
            ]
          },
          {
            "name": "pathway_is_mapk",
            "role": "gate",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "mapk"
            ]
          },
          {
            "name": "pathway_reduced_in_aged",
            "role": "gate",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "reduced_in_aged"
            ]
          },
          {
            "name": "cf_mapk_increased_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_in_aged"
            ]
          },
          {
            "name": "cf_age_erased",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_erased"
            ]
          },
          {
            "name": "cf_pervasive_overcall",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_pervasive_artifactual"
            ]
          },
          {
            "name": "cf_explain_away_aging",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_compositional_or_identity",
            "role": "hard_fail",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_proportion_shift",
              "cell_identity_shift",
              "composition_dominant_state_secondary",
              "both_state_and_composition_equally"
            ]
          },
          {
            "name": "cf_biggest_inflammatory_signal_not_mapk",
            "role": "hard_fail",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "jak_stat",
              "nfkb",
              "tnfa"
            ]
          },
          {
            "name": "cf_senescence_misattribution",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_senescence_signature_specifically"
            ]
          },
          {
            "name": "cf_dismiss_as_batch_artifact",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_batch_or_technical_artifact"
            ]
          },
          {
            "name": "cf_mapk_comparable_constitutive_kras_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "comparable_across_ages"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-4-pi",
      "modelName": "gpt-5.4",
      "harness": "pi",
      "task": "gse297023_q3_age_retained_mapk_reduced",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 2694,
      "cost": 1.8674389999999998,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "age_retained",
            "role": "gate",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_retained"
            ]
          },
          {
            "name": "identity_is_aging",
            "role": "gate",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_aging_signatures"
            ]
          },
          {
            "name": "nature_state_shift",
            "role": "gate",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_state_expression_shift"
            ]
          },
          {
            "name": "pathway_is_mapk",
            "role": "gate",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "mapk"
            ]
          },
          {
            "name": "pathway_reduced_in_aged",
            "role": "gate",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "reduced_in_aged"
            ]
          },
          {
            "name": "cf_mapk_increased_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_in_aged"
            ]
          },
          {
            "name": "cf_age_erased",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_erased"
            ]
          },
          {
            "name": "cf_pervasive_overcall",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_pervasive_artifactual"
            ]
          },
          {
            "name": "cf_explain_away_aging",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_compositional_or_identity",
            "role": "hard_fail",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_proportion_shift",
              "cell_identity_shift",
              "composition_dominant_state_secondary",
              "both_state_and_composition_equally"
            ]
          },
          {
            "name": "cf_biggest_inflammatory_signal_not_mapk",
            "role": "hard_fail",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "jak_stat",
              "nfkb",
              "tnfa"
            ]
          },
          {
            "name": "cf_senescence_misattribution",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_senescence_signature_specifically"
            ]
          },
          {
            "name": "cf_dismiss_as_batch_artifact",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_batch_or_technical_artifact"
            ]
          },
          {
            "name": "cf_mapk_comparable_constitutive_kras_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "comparable_across_ages"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-4-pi",
      "modelName": "gpt-5.4",
      "harness": "pi",
      "task": "gse297023_q3_age_retained_mapk_reduced",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 1889,
      "cost": 1.200808,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "age_retained",
            "role": "gate",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_retained"
            ]
          },
          {
            "name": "identity_is_aging",
            "role": "gate",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_aging_signatures"
            ]
          },
          {
            "name": "nature_state_shift",
            "role": "gate",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_state_expression_shift"
            ]
          },
          {
            "name": "pathway_is_mapk",
            "role": "gate",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "mapk"
            ]
          },
          {
            "name": "pathway_reduced_in_aged",
            "role": "gate",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "reduced_in_aged"
            ]
          },
          {
            "name": "cf_mapk_increased_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_in_aged"
            ]
          },
          {
            "name": "cf_age_erased",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_erased"
            ]
          },
          {
            "name": "cf_pervasive_overcall",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_pervasive_artifactual"
            ]
          },
          {
            "name": "cf_explain_away_aging",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_compositional_or_identity",
            "role": "hard_fail",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_proportion_shift",
              "cell_identity_shift",
              "composition_dominant_state_secondary",
              "both_state_and_composition_equally"
            ]
          },
          {
            "name": "cf_biggest_inflammatory_signal_not_mapk",
            "role": "hard_fail",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "jak_stat",
              "nfkb",
              "tnfa"
            ]
          },
          {
            "name": "cf_senescence_misattribution",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_senescence_signature_specifically"
            ]
          },
          {
            "name": "cf_dismiss_as_batch_artifact",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_batch_or_technical_artifact"
            ]
          },
          {
            "name": "cf_mapk_comparable_constitutive_kras_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "comparable_across_ages"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-4-pi",
      "modelName": "gpt-5.4",
      "harness": "pi",
      "task": "gse297023_q3_age_retained_mapk_reduced",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 4671,
      "cost": 1.461338,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "age_retained",
            "role": "gate",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_retained"
            ]
          },
          {
            "name": "identity_is_aging",
            "role": "gate",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_aging_signatures"
            ]
          },
          {
            "name": "nature_state_shift",
            "role": "gate",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_state_expression_shift"
            ]
          },
          {
            "name": "pathway_is_mapk",
            "role": "gate",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "mapk"
            ]
          },
          {
            "name": "pathway_reduced_in_aged",
            "role": "gate",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "reduced_in_aged"
            ]
          },
          {
            "name": "cf_mapk_increased_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_in_aged"
            ]
          },
          {
            "name": "cf_age_erased",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_erased"
            ]
          },
          {
            "name": "cf_pervasive_overcall",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_pervasive_artifactual"
            ]
          },
          {
            "name": "cf_explain_away_aging",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_compositional_or_identity",
            "role": "hard_fail",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_proportion_shift",
              "cell_identity_shift",
              "composition_dominant_state_secondary",
              "both_state_and_composition_equally"
            ]
          },
          {
            "name": "cf_biggest_inflammatory_signal_not_mapk",
            "role": "hard_fail",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "jak_stat",
              "nfkb",
              "tnfa"
            ]
          },
          {
            "name": "cf_senescence_misattribution",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_senescence_signature_specifically"
            ]
          },
          {
            "name": "cf_dismiss_as_batch_artifact",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_batch_or_technical_artifact"
            ]
          },
          {
            "name": "cf_mapk_comparable_constitutive_kras_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "comparable_across_ages"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-5-codex",
      "modelName": "gpt-5.5",
      "harness": "openai-codex",
      "task": "gse297023_q3_age_retained_mapk_reduced",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 897,
      "cost": 2.124029,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "age_retained",
            "role": "gate",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_retained"
            ]
          },
          {
            "name": "identity_is_aging",
            "role": "gate",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_aging_signatures"
            ]
          },
          {
            "name": "nature_state_shift",
            "role": "gate",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_state_expression_shift"
            ]
          },
          {
            "name": "pathway_is_mapk",
            "role": "gate",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "mapk"
            ]
          },
          {
            "name": "pathway_reduced_in_aged",
            "role": "gate",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "reduced_in_aged"
            ]
          },
          {
            "name": "cf_mapk_increased_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_in_aged"
            ]
          },
          {
            "name": "cf_age_erased",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_erased"
            ]
          },
          {
            "name": "cf_pervasive_overcall",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_pervasive_artifactual"
            ]
          },
          {
            "name": "cf_explain_away_aging",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_compositional_or_identity",
            "role": "hard_fail",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_proportion_shift",
              "cell_identity_shift",
              "composition_dominant_state_secondary",
              "both_state_and_composition_equally"
            ]
          },
          {
            "name": "cf_biggest_inflammatory_signal_not_mapk",
            "role": "hard_fail",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "jak_stat",
              "nfkb",
              "tnfa"
            ]
          },
          {
            "name": "cf_senescence_misattribution",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_senescence_signature_specifically"
            ]
          },
          {
            "name": "cf_dismiss_as_batch_artifact",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_batch_or_technical_artifact"
            ]
          },
          {
            "name": "cf_mapk_comparable_constitutive_kras_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "comparable_across_ages"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-5-codex",
      "modelName": "gpt-5.5",
      "harness": "openai-codex",
      "task": "gse297023_q3_age_retained_mapk_reduced",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 997,
      "cost": 2.8428950000000004,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "age_retained",
            "role": "gate",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_retained"
            ]
          },
          {
            "name": "identity_is_aging",
            "role": "gate",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_aging_signatures"
            ]
          },
          {
            "name": "nature_state_shift",
            "role": "gate",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_state_expression_shift"
            ]
          },
          {
            "name": "pathway_is_mapk",
            "role": "gate",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "mapk"
            ]
          },
          {
            "name": "pathway_reduced_in_aged",
            "role": "gate",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "reduced_in_aged"
            ]
          },
          {
            "name": "cf_mapk_increased_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_in_aged"
            ]
          },
          {
            "name": "cf_age_erased",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_erased"
            ]
          },
          {
            "name": "cf_pervasive_overcall",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_pervasive_artifactual"
            ]
          },
          {
            "name": "cf_explain_away_aging",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_compositional_or_identity",
            "role": "hard_fail",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_proportion_shift",
              "cell_identity_shift",
              "composition_dominant_state_secondary",
              "both_state_and_composition_equally"
            ]
          },
          {
            "name": "cf_biggest_inflammatory_signal_not_mapk",
            "role": "hard_fail",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "jak_stat",
              "nfkb",
              "tnfa"
            ]
          },
          {
            "name": "cf_senescence_misattribution",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_senescence_signature_specifically"
            ]
          },
          {
            "name": "cf_dismiss_as_batch_artifact",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_batch_or_technical_artifact"
            ]
          },
          {
            "name": "cf_mapk_comparable_constitutive_kras_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "comparable_across_ages"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-5-codex",
      "modelName": "gpt-5.5",
      "harness": "openai-codex",
      "task": "gse297023_q3_age_retained_mapk_reduced",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 930,
      "cost": 2.034874,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "age_retained",
            "role": "gate",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_retained"
            ]
          },
          {
            "name": "identity_is_aging",
            "role": "gate",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_aging_signatures"
            ]
          },
          {
            "name": "nature_state_shift",
            "role": "gate",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_state_expression_shift"
            ]
          },
          {
            "name": "pathway_is_mapk",
            "role": "gate",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "mapk"
            ]
          },
          {
            "name": "pathway_reduced_in_aged",
            "role": "gate",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "reduced_in_aged"
            ]
          },
          {
            "name": "cf_mapk_increased_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_in_aged"
            ]
          },
          {
            "name": "cf_age_erased",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_erased"
            ]
          },
          {
            "name": "cf_pervasive_overcall",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_pervasive_artifactual"
            ]
          },
          {
            "name": "cf_explain_away_aging",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_compositional_or_identity",
            "role": "hard_fail",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_proportion_shift",
              "cell_identity_shift",
              "composition_dominant_state_secondary",
              "both_state_and_composition_equally"
            ]
          },
          {
            "name": "cf_biggest_inflammatory_signal_not_mapk",
            "role": "hard_fail",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "jak_stat",
              "nfkb",
              "tnfa"
            ]
          },
          {
            "name": "cf_senescence_misattribution",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_senescence_signature_specifically"
            ]
          },
          {
            "name": "cf_dismiss_as_batch_artifact",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_batch_or_technical_artifact"
            ]
          },
          {
            "name": "cf_mapk_comparable_constitutive_kras_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "comparable_across_ages"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-5-pi",
      "modelName": "gpt-5.5",
      "harness": "pi",
      "task": "gse297023_q3_age_retained_mapk_reduced",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 1978,
      "cost": 2.121997,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "age_retained",
            "role": "gate",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_retained"
            ]
          },
          {
            "name": "identity_is_aging",
            "role": "gate",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_aging_signatures"
            ]
          },
          {
            "name": "nature_state_shift",
            "role": "gate",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_state_expression_shift"
            ]
          },
          {
            "name": "pathway_is_mapk",
            "role": "gate",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "mapk"
            ]
          },
          {
            "name": "pathway_reduced_in_aged",
            "role": "gate",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "reduced_in_aged"
            ]
          },
          {
            "name": "cf_mapk_increased_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_in_aged"
            ]
          },
          {
            "name": "cf_age_erased",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_erased"
            ]
          },
          {
            "name": "cf_pervasive_overcall",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_pervasive_artifactual"
            ]
          },
          {
            "name": "cf_explain_away_aging",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_compositional_or_identity",
            "role": "hard_fail",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_proportion_shift",
              "cell_identity_shift",
              "composition_dominant_state_secondary",
              "both_state_and_composition_equally"
            ]
          },
          {
            "name": "cf_biggest_inflammatory_signal_not_mapk",
            "role": "hard_fail",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "jak_stat",
              "nfkb",
              "tnfa"
            ]
          },
          {
            "name": "cf_senescence_misattribution",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_senescence_signature_specifically"
            ]
          },
          {
            "name": "cf_dismiss_as_batch_artifact",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_batch_or_technical_artifact"
            ]
          },
          {
            "name": "cf_mapk_comparable_constitutive_kras_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "comparable_across_ages"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-5-pi",
      "modelName": "gpt-5.5",
      "harness": "pi",
      "task": "gse297023_q3_age_retained_mapk_reduced",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 2,
      "passed": true,
      "score": null,
      "durationS": 910,
      "cost": 2.005228,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "age_retained",
            "role": "gate",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_retained"
            ]
          },
          {
            "name": "identity_is_aging",
            "role": "gate",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_aging_signatures"
            ]
          },
          {
            "name": "nature_state_shift",
            "role": "gate",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_state_expression_shift"
            ]
          },
          {
            "name": "pathway_is_mapk",
            "role": "gate",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "mapk"
            ]
          },
          {
            "name": "pathway_reduced_in_aged",
            "role": "gate",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "reduced_in_aged"
            ]
          },
          {
            "name": "cf_mapk_increased_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_in_aged"
            ]
          },
          {
            "name": "cf_age_erased",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_erased"
            ]
          },
          {
            "name": "cf_pervasive_overcall",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_pervasive_artifactual"
            ]
          },
          {
            "name": "cf_explain_away_aging",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_compositional_or_identity",
            "role": "hard_fail",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_proportion_shift",
              "cell_identity_shift",
              "composition_dominant_state_secondary",
              "both_state_and_composition_equally"
            ]
          },
          {
            "name": "cf_biggest_inflammatory_signal_not_mapk",
            "role": "hard_fail",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "jak_stat",
              "nfkb",
              "tnfa"
            ]
          },
          {
            "name": "cf_senescence_misattribution",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_senescence_signature_specifically"
            ]
          },
          {
            "name": "cf_dismiss_as_batch_artifact",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_batch_or_technical_artifact"
            ]
          },
          {
            "name": "cf_mapk_comparable_constitutive_kras_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "comparable_across_ages"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-5-pi",
      "modelName": "gpt-5.5",
      "harness": "pi",
      "task": "gse297023_q3_age_retained_mapk_reduced",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1306,
      "cost": 1.944693,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "age_retained",
            "role": "gate",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_retained"
            ]
          },
          {
            "name": "identity_is_aging",
            "role": "gate",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_aging_signatures"
            ]
          },
          {
            "name": "nature_state_shift",
            "role": "gate",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_state_expression_shift"
            ]
          },
          {
            "name": "pathway_is_mapk",
            "role": "gate",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "mapk"
            ]
          },
          {
            "name": "pathway_reduced_in_aged",
            "role": "gate",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "reduced_in_aged"
            ]
          },
          {
            "name": "cf_mapk_increased_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_in_aged"
            ]
          },
          {
            "name": "cf_age_erased",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_erased"
            ]
          },
          {
            "name": "cf_pervasive_overcall",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_pervasive_artifactual"
            ]
          },
          {
            "name": "cf_explain_away_aging",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_compositional_or_identity",
            "role": "hard_fail",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_proportion_shift",
              "cell_identity_shift",
              "composition_dominant_state_secondary",
              "both_state_and_composition_equally"
            ]
          },
          {
            "name": "cf_biggest_inflammatory_signal_not_mapk",
            "role": "hard_fail",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "jak_stat",
              "nfkb",
              "tnfa"
            ]
          },
          {
            "name": "cf_senescence_misattribution",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_senescence_signature_specifically"
            ]
          },
          {
            "name": "cf_dismiss_as_batch_artifact",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_batch_or_technical_artifact"
            ]
          },
          {
            "name": "cf_mapk_comparable_constitutive_kras_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "comparable_across_ages"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-20-beta-reasoning-pi",
      "modelName": "grok-4.20-beta-0309-reasoning",
      "harness": "pi",
      "task": "gse297023_q3_age_retained_mapk_reduced",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 1,
      "passed": true,
      "score": null,
      "durationS": 698,
      "cost": 0.05999260000000001,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "age_retained",
            "role": "gate",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_retained"
            ]
          },
          {
            "name": "identity_is_aging",
            "role": "gate",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_aging_signatures"
            ]
          },
          {
            "name": "nature_state_shift",
            "role": "gate",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_state_expression_shift"
            ]
          },
          {
            "name": "pathway_is_mapk",
            "role": "gate",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "mapk"
            ]
          },
          {
            "name": "pathway_reduced_in_aged",
            "role": "gate",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "reduced_in_aged"
            ]
          },
          {
            "name": "cf_mapk_increased_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_in_aged"
            ]
          },
          {
            "name": "cf_age_erased",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_erased"
            ]
          },
          {
            "name": "cf_pervasive_overcall",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_pervasive_artifactual"
            ]
          },
          {
            "name": "cf_explain_away_aging",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_compositional_or_identity",
            "role": "hard_fail",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_proportion_shift",
              "cell_identity_shift",
              "composition_dominant_state_secondary",
              "both_state_and_composition_equally"
            ]
          },
          {
            "name": "cf_biggest_inflammatory_signal_not_mapk",
            "role": "hard_fail",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "jak_stat",
              "nfkb",
              "tnfa"
            ]
          },
          {
            "name": "cf_senescence_misattribution",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_senescence_signature_specifically"
            ]
          },
          {
            "name": "cf_dismiss_as_batch_artifact",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_batch_or_technical_artifact"
            ]
          },
          {
            "name": "cf_mapk_comparable_constitutive_kras_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "comparable_across_ages"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-20-beta-reasoning-pi",
      "modelName": "grok-4.20-beta-0309-reasoning",
      "harness": "pi",
      "task": "gse297023_q3_age_retained_mapk_reduced",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 1368,
      "cost": 0.032993850000000005,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "age_retained",
            "role": "gate",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_retained"
            ]
          },
          {
            "name": "identity_is_aging",
            "role": "gate",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_aging_signatures"
            ]
          },
          {
            "name": "nature_state_shift",
            "role": "gate",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_state_expression_shift"
            ]
          },
          {
            "name": "pathway_is_mapk",
            "role": "gate",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "mapk"
            ]
          },
          {
            "name": "pathway_reduced_in_aged",
            "role": "gate",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "reduced_in_aged"
            ]
          },
          {
            "name": "cf_mapk_increased_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_in_aged"
            ]
          },
          {
            "name": "cf_age_erased",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_erased"
            ]
          },
          {
            "name": "cf_pervasive_overcall",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_pervasive_artifactual"
            ]
          },
          {
            "name": "cf_explain_away_aging",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_compositional_or_identity",
            "role": "hard_fail",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_proportion_shift",
              "cell_identity_shift",
              "composition_dominant_state_secondary",
              "both_state_and_composition_equally"
            ]
          },
          {
            "name": "cf_biggest_inflammatory_signal_not_mapk",
            "role": "hard_fail",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "jak_stat",
              "nfkb",
              "tnfa"
            ]
          },
          {
            "name": "cf_senescence_misattribution",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_senescence_signature_specifically"
            ]
          },
          {
            "name": "cf_dismiss_as_batch_artifact",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_batch_or_technical_artifact"
            ]
          },
          {
            "name": "cf_mapk_comparable_constitutive_kras_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "comparable_across_ages"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-20-beta-reasoning-pi",
      "modelName": "grok-4.20-beta-0309-reasoning",
      "harness": "pi",
      "task": "gse297023_q3_age_retained_mapk_reduced",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1499,
      "cost": 0.03602495000000001,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "age_retained",
            "role": "gate",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_retained"
            ]
          },
          {
            "name": "identity_is_aging",
            "role": "gate",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_aging_signatures"
            ]
          },
          {
            "name": "nature_state_shift",
            "role": "gate",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_state_expression_shift"
            ]
          },
          {
            "name": "pathway_is_mapk",
            "role": "gate",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "mapk"
            ]
          },
          {
            "name": "pathway_reduced_in_aged",
            "role": "gate",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "reduced_in_aged"
            ]
          },
          {
            "name": "cf_mapk_increased_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_in_aged"
            ]
          },
          {
            "name": "cf_age_erased",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_erased"
            ]
          },
          {
            "name": "cf_pervasive_overcall",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_pervasive_artifactual"
            ]
          },
          {
            "name": "cf_explain_away_aging",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_compositional_or_identity",
            "role": "hard_fail",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_proportion_shift",
              "cell_identity_shift",
              "composition_dominant_state_secondary",
              "both_state_and_composition_equally"
            ]
          },
          {
            "name": "cf_biggest_inflammatory_signal_not_mapk",
            "role": "hard_fail",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "jak_stat",
              "nfkb",
              "tnfa"
            ]
          },
          {
            "name": "cf_senescence_misattribution",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_senescence_signature_specifically"
            ]
          },
          {
            "name": "cf_dismiss_as_batch_artifact",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_batch_or_technical_artifact"
            ]
          },
          {
            "name": "cf_mapk_comparable_constitutive_kras_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "comparable_across_ages"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-3-pi",
      "modelName": "grok-4.3",
      "harness": "pi",
      "task": "gse297023_q3_age_retained_mapk_reduced",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 230,
      "cost": 0.0070440500000000005,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "age_retained",
            "role": "gate",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_retained"
            ]
          },
          {
            "name": "identity_is_aging",
            "role": "gate",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_aging_signatures"
            ]
          },
          {
            "name": "nature_state_shift",
            "role": "gate",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_state_expression_shift"
            ]
          },
          {
            "name": "pathway_is_mapk",
            "role": "gate",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "mapk"
            ]
          },
          {
            "name": "pathway_reduced_in_aged",
            "role": "gate",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "reduced_in_aged"
            ]
          },
          {
            "name": "cf_mapk_increased_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_in_aged"
            ]
          },
          {
            "name": "cf_age_erased",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_erased"
            ]
          },
          {
            "name": "cf_pervasive_overcall",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_pervasive_artifactual"
            ]
          },
          {
            "name": "cf_explain_away_aging",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_compositional_or_identity",
            "role": "hard_fail",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_proportion_shift",
              "cell_identity_shift",
              "composition_dominant_state_secondary",
              "both_state_and_composition_equally"
            ]
          },
          {
            "name": "cf_biggest_inflammatory_signal_not_mapk",
            "role": "hard_fail",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "jak_stat",
              "nfkb",
              "tnfa"
            ]
          },
          {
            "name": "cf_senescence_misattribution",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_senescence_signature_specifically"
            ]
          },
          {
            "name": "cf_dismiss_as_batch_artifact",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_batch_or_technical_artifact"
            ]
          },
          {
            "name": "cf_mapk_comparable_constitutive_kras_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "comparable_across_ages"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-3-pi",
      "modelName": "grok-4.3",
      "harness": "pi",
      "task": "gse297023_q3_age_retained_mapk_reduced",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 91,
      "cost": 0.00777395,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "age_retained",
            "role": "gate",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_retained"
            ]
          },
          {
            "name": "identity_is_aging",
            "role": "gate",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_aging_signatures"
            ]
          },
          {
            "name": "nature_state_shift",
            "role": "gate",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_state_expression_shift"
            ]
          },
          {
            "name": "pathway_is_mapk",
            "role": "gate",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "mapk"
            ]
          },
          {
            "name": "pathway_reduced_in_aged",
            "role": "gate",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "reduced_in_aged"
            ]
          },
          {
            "name": "cf_mapk_increased_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_in_aged"
            ]
          },
          {
            "name": "cf_age_erased",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_erased"
            ]
          },
          {
            "name": "cf_pervasive_overcall",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_pervasive_artifactual"
            ]
          },
          {
            "name": "cf_explain_away_aging",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_compositional_or_identity",
            "role": "hard_fail",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_proportion_shift",
              "cell_identity_shift",
              "composition_dominant_state_secondary",
              "both_state_and_composition_equally"
            ]
          },
          {
            "name": "cf_biggest_inflammatory_signal_not_mapk",
            "role": "hard_fail",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "jak_stat",
              "nfkb",
              "tnfa"
            ]
          },
          {
            "name": "cf_senescence_misattribution",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_senescence_signature_specifically"
            ]
          },
          {
            "name": "cf_dismiss_as_batch_artifact",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_batch_or_technical_artifact"
            ]
          },
          {
            "name": "cf_mapk_comparable_constitutive_kras_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "comparable_across_ages"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-3-pi",
      "modelName": "grok-4.3",
      "harness": "pi",
      "task": "gse297023_q3_age_retained_mapk_reduced",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 359,
      "cost": 0.012115300000000002,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "age_retained",
            "role": "gate",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_retained"
            ]
          },
          {
            "name": "identity_is_aging",
            "role": "gate",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_aging_signatures"
            ]
          },
          {
            "name": "nature_state_shift",
            "role": "gate",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_state_expression_shift"
            ]
          },
          {
            "name": "pathway_is_mapk",
            "role": "gate",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "mapk"
            ]
          },
          {
            "name": "pathway_reduced_in_aged",
            "role": "gate",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "reduced_in_aged"
            ]
          },
          {
            "name": "cf_mapk_increased_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_in_aged"
            ]
          },
          {
            "name": "cf_age_erased",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_erased"
            ]
          },
          {
            "name": "cf_pervasive_overcall",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_pervasive_artifactual"
            ]
          },
          {
            "name": "cf_explain_away_aging",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_compositional_or_identity",
            "role": "hard_fail",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_proportion_shift",
              "cell_identity_shift",
              "composition_dominant_state_secondary",
              "both_state_and_composition_equally"
            ]
          },
          {
            "name": "cf_biggest_inflammatory_signal_not_mapk",
            "role": "hard_fail",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "jak_stat",
              "nfkb",
              "tnfa"
            ]
          },
          {
            "name": "cf_senescence_misattribution",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_senescence_signature_specifically"
            ]
          },
          {
            "name": "cf_dismiss_as_batch_artifact",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_batch_or_technical_artifact"
            ]
          },
          {
            "name": "cf_mapk_comparable_constitutive_kras_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "comparable_across_ages"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-kimi-k2p6-pi",
      "modelName": "kimi-k2p6",
      "harness": "pi",
      "task": "gse297023_q3_age_retained_mapk_reduced",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 1955,
      "cost": 0.69223794,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "age_retained",
            "role": "gate",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_retained"
            ]
          },
          {
            "name": "identity_is_aging",
            "role": "gate",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_aging_signatures"
            ]
          },
          {
            "name": "nature_state_shift",
            "role": "gate",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_state_expression_shift"
            ]
          },
          {
            "name": "pathway_is_mapk",
            "role": "gate",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "mapk"
            ]
          },
          {
            "name": "pathway_reduced_in_aged",
            "role": "gate",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "reduced_in_aged"
            ]
          },
          {
            "name": "cf_mapk_increased_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_in_aged"
            ]
          },
          {
            "name": "cf_age_erased",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_erased"
            ]
          },
          {
            "name": "cf_pervasive_overcall",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_pervasive_artifactual"
            ]
          },
          {
            "name": "cf_explain_away_aging",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_compositional_or_identity",
            "role": "hard_fail",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_proportion_shift",
              "cell_identity_shift",
              "composition_dominant_state_secondary",
              "both_state_and_composition_equally"
            ]
          },
          {
            "name": "cf_biggest_inflammatory_signal_not_mapk",
            "role": "hard_fail",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "jak_stat",
              "nfkb",
              "tnfa"
            ]
          },
          {
            "name": "cf_senescence_misattribution",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_senescence_signature_specifically"
            ]
          },
          {
            "name": "cf_dismiss_as_batch_artifact",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_batch_or_technical_artifact"
            ]
          },
          {
            "name": "cf_mapk_comparable_constitutive_kras_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "comparable_across_ages"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-kimi-k2p6-pi",
      "modelName": "kimi-k2p6",
      "harness": "pi",
      "task": "gse297023_q3_age_retained_mapk_reduced",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 2752,
      "cost": 0.6166063999999999,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "age_retained",
            "role": "gate",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_retained"
            ]
          },
          {
            "name": "identity_is_aging",
            "role": "gate",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_aging_signatures"
            ]
          },
          {
            "name": "nature_state_shift",
            "role": "gate",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_state_expression_shift"
            ]
          },
          {
            "name": "pathway_is_mapk",
            "role": "gate",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "mapk"
            ]
          },
          {
            "name": "pathway_reduced_in_aged",
            "role": "gate",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "reduced_in_aged"
            ]
          },
          {
            "name": "cf_mapk_increased_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_in_aged"
            ]
          },
          {
            "name": "cf_age_erased",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_erased"
            ]
          },
          {
            "name": "cf_pervasive_overcall",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_pervasive_artifactual"
            ]
          },
          {
            "name": "cf_explain_away_aging",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_compositional_or_identity",
            "role": "hard_fail",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_proportion_shift",
              "cell_identity_shift",
              "composition_dominant_state_secondary",
              "both_state_and_composition_equally"
            ]
          },
          {
            "name": "cf_biggest_inflammatory_signal_not_mapk",
            "role": "hard_fail",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "jak_stat",
              "nfkb",
              "tnfa"
            ]
          },
          {
            "name": "cf_senescence_misattribution",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_senescence_signature_specifically"
            ]
          },
          {
            "name": "cf_dismiss_as_batch_artifact",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_batch_or_technical_artifact"
            ]
          },
          {
            "name": "cf_mapk_comparable_constitutive_kras_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "comparable_across_ages"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-kimi-k2p6-pi",
      "modelName": "kimi-k2p6",
      "harness": "pi",
      "task": "gse297023_q3_age_retained_mapk_reduced",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 2878,
      "cost": 1.15769404,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "age_retained",
            "role": "gate",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_retained"
            ]
          },
          {
            "name": "identity_is_aging",
            "role": "gate",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_aging_signatures"
            ]
          },
          {
            "name": "nature_state_shift",
            "role": "gate",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_state_expression_shift"
            ]
          },
          {
            "name": "pathway_is_mapk",
            "role": "gate",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "mapk"
            ]
          },
          {
            "name": "pathway_reduced_in_aged",
            "role": "gate",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "reduced_in_aged"
            ]
          },
          {
            "name": "cf_mapk_increased_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_in_aged"
            ]
          },
          {
            "name": "cf_age_erased",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_erased"
            ]
          },
          {
            "name": "cf_pervasive_overcall",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_pervasive_artifactual"
            ]
          },
          {
            "name": "cf_explain_away_aging",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_compositional_or_identity",
            "role": "hard_fail",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_proportion_shift",
              "cell_identity_shift",
              "composition_dominant_state_secondary",
              "both_state_and_composition_equally"
            ]
          },
          {
            "name": "cf_biggest_inflammatory_signal_not_mapk",
            "role": "hard_fail",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "jak_stat",
              "nfkb",
              "tnfa"
            ]
          },
          {
            "name": "cf_senescence_misattribution",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_senescence_signature_specifically"
            ]
          },
          {
            "name": "cf_dismiss_as_batch_artifact",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_batch_or_technical_artifact"
            ]
          },
          {
            "name": "cf_mapk_comparable_constitutive_kras_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "comparable_across_ages"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-6-claude-code",
      "modelName": "claude-opus-4-6",
      "harness": "claude-code",
      "task": "gse297023_q5_tme_age_dependent_remodeling",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 2842,
      "cost": 1.82511925,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "perturbation_is_pten",
            "role": "gate",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "pten"
            ]
          },
          {
            "name": "tme_non_cell_autonomous",
            "role": "gate",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "substantial_non_cell_autonomous_remodeling"
            ]
          },
          {
            "name": "tme_state_not_composition",
            "role": "gate",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "tme_age_dependent",
            "role": "gate",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_dependent"
            ]
          },
          {
            "name": "tme_response_broader_in_aged",
            "role": "gate",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_aged"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "most_responsive_noncancer_cell_types",
            "kind": "list",
            "matchKey": "cell_type",
            "k": 2,
            "expected": [
              {
                "key": "alveolar_macrophage",
                "fields": {}
              },
              {
                "key": "endothelial",
                "fields": {}
              }
            ]
          },
          {
            "name": "cf_cell_intrinsic_only",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "negligible_cell_intrinsic_only_effect",
              "remodeling_unresolved",
              "modest_remodeling_few_cell_types"
            ]
          },
          {
            "name": "cf_remodeling_is_contamination",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "apparent_remodeling_reflects_cancer_contamination"
            ]
          },
          {
            "name": "cf_perturbation_design_misread",
            "role": "hard_fail",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "multiple_tumor_suppressors_pooled",
              "no_additional_gene_difference_kras_only",
              "rb1_cdkn2a_double",
              "pcna_essential_gene_control",
              "difference_is_oncogene_not_suppressor"
            ]
          },
          {
            "name": "cf_age_independent_tme",
            "role": "hard_fail",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_independent"
            ]
          },
          {
            "name": "cf_compositional_shift",
            "role": "hard_fail",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_frequency_shift",
              "compositional_immune_infiltration",
              "composition_dominant_state_secondary",
              "both_state_and_composition"
            ]
          },
          {
            "name": "cf_epithelial_or_cancer_substitution",
            "role": "hard_fail",
            "field": "$.most_responsive_noncancer_cell_types[*].cell_type",
            "kind": "avoid",
            "expected": [
              "AT2_cell",
              "AT1_cell",
              "cancer_cell",
              "ciliated",
              "secretory",
              "club_cell",
              "basal_cell",
              "goblet_cell"
            ]
          },
          {
            "name": "cf_tme_breadth_broader_in_young",
            "role": "hard_fail",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_young"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-6-claude-code",
      "modelName": "claude-opus-4-6",
      "harness": "claude-code",
      "task": "gse297023_q5_tme_age_dependent_remodeling",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 4455,
      "cost": 2.7290807499999996,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "perturbation_is_pten",
            "role": "gate",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "pten"
            ]
          },
          {
            "name": "tme_non_cell_autonomous",
            "role": "gate",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "substantial_non_cell_autonomous_remodeling"
            ]
          },
          {
            "name": "tme_state_not_composition",
            "role": "gate",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "tme_age_dependent",
            "role": "gate",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_dependent"
            ]
          },
          {
            "name": "tme_response_broader_in_aged",
            "role": "gate",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_aged"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "most_responsive_noncancer_cell_types",
            "kind": "list",
            "matchKey": "cell_type",
            "k": 2,
            "expected": [
              {
                "key": "alveolar_macrophage",
                "fields": {}
              },
              {
                "key": "endothelial",
                "fields": {}
              }
            ]
          },
          {
            "name": "cf_cell_intrinsic_only",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "negligible_cell_intrinsic_only_effect",
              "remodeling_unresolved",
              "modest_remodeling_few_cell_types"
            ]
          },
          {
            "name": "cf_remodeling_is_contamination",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "apparent_remodeling_reflects_cancer_contamination"
            ]
          },
          {
            "name": "cf_perturbation_design_misread",
            "role": "hard_fail",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "multiple_tumor_suppressors_pooled",
              "no_additional_gene_difference_kras_only",
              "rb1_cdkn2a_double",
              "pcna_essential_gene_control",
              "difference_is_oncogene_not_suppressor"
            ]
          },
          {
            "name": "cf_age_independent_tme",
            "role": "hard_fail",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_independent"
            ]
          },
          {
            "name": "cf_compositional_shift",
            "role": "hard_fail",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_frequency_shift",
              "compositional_immune_infiltration",
              "composition_dominant_state_secondary",
              "both_state_and_composition"
            ]
          },
          {
            "name": "cf_epithelial_or_cancer_substitution",
            "role": "hard_fail",
            "field": "$.most_responsive_noncancer_cell_types[*].cell_type",
            "kind": "avoid",
            "expected": [
              "AT2_cell",
              "AT1_cell",
              "cancer_cell",
              "ciliated",
              "secretory",
              "club_cell",
              "basal_cell",
              "goblet_cell"
            ]
          },
          {
            "name": "cf_tme_breadth_broader_in_young",
            "role": "hard_fail",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_young"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-6-claude-code",
      "modelName": "claude-opus-4-6",
      "harness": "claude-code",
      "task": "gse297023_q5_tme_age_dependent_remodeling",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 4117,
      "cost": 3.0741525,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "perturbation_is_pten",
            "role": "gate",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "pten"
            ]
          },
          {
            "name": "tme_non_cell_autonomous",
            "role": "gate",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "substantial_non_cell_autonomous_remodeling"
            ]
          },
          {
            "name": "tme_state_not_composition",
            "role": "gate",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "tme_age_dependent",
            "role": "gate",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_dependent"
            ]
          },
          {
            "name": "tme_response_broader_in_aged",
            "role": "gate",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_aged"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "most_responsive_noncancer_cell_types",
            "kind": "list",
            "matchKey": "cell_type",
            "k": 2,
            "expected": [
              {
                "key": "alveolar_macrophage",
                "fields": {}
              },
              {
                "key": "endothelial",
                "fields": {}
              }
            ]
          },
          {
            "name": "cf_cell_intrinsic_only",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "negligible_cell_intrinsic_only_effect",
              "remodeling_unresolved",
              "modest_remodeling_few_cell_types"
            ]
          },
          {
            "name": "cf_remodeling_is_contamination",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "apparent_remodeling_reflects_cancer_contamination"
            ]
          },
          {
            "name": "cf_perturbation_design_misread",
            "role": "hard_fail",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "multiple_tumor_suppressors_pooled",
              "no_additional_gene_difference_kras_only",
              "rb1_cdkn2a_double",
              "pcna_essential_gene_control",
              "difference_is_oncogene_not_suppressor"
            ]
          },
          {
            "name": "cf_age_independent_tme",
            "role": "hard_fail",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_independent"
            ]
          },
          {
            "name": "cf_compositional_shift",
            "role": "hard_fail",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_frequency_shift",
              "compositional_immune_infiltration",
              "composition_dominant_state_secondary",
              "both_state_and_composition"
            ]
          },
          {
            "name": "cf_epithelial_or_cancer_substitution",
            "role": "hard_fail",
            "field": "$.most_responsive_noncancer_cell_types[*].cell_type",
            "kind": "avoid",
            "expected": [
              "AT2_cell",
              "AT1_cell",
              "cancer_cell",
              "ciliated",
              "secretory",
              "club_cell",
              "basal_cell",
              "goblet_cell"
            ]
          },
          {
            "name": "cf_tme_breadth_broader_in_young",
            "role": "hard_fail",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_young"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-6-pi",
      "modelName": "claude-opus-4-6",
      "harness": "pi",
      "task": "gse297023_q5_tme_age_dependent_remodeling",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 3798,
      "cost": 2.4309835,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "perturbation_is_pten",
            "role": "gate",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "pten"
            ]
          },
          {
            "name": "tme_non_cell_autonomous",
            "role": "gate",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "substantial_non_cell_autonomous_remodeling"
            ]
          },
          {
            "name": "tme_state_not_composition",
            "role": "gate",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "tme_age_dependent",
            "role": "gate",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_dependent"
            ]
          },
          {
            "name": "tme_response_broader_in_aged",
            "role": "gate",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_aged"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "most_responsive_noncancer_cell_types",
            "kind": "list",
            "matchKey": "cell_type",
            "k": 2,
            "expected": [
              {
                "key": "alveolar_macrophage",
                "fields": {}
              },
              {
                "key": "endothelial",
                "fields": {}
              }
            ]
          },
          {
            "name": "cf_cell_intrinsic_only",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "negligible_cell_intrinsic_only_effect",
              "remodeling_unresolved",
              "modest_remodeling_few_cell_types"
            ]
          },
          {
            "name": "cf_remodeling_is_contamination",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "apparent_remodeling_reflects_cancer_contamination"
            ]
          },
          {
            "name": "cf_perturbation_design_misread",
            "role": "hard_fail",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "multiple_tumor_suppressors_pooled",
              "no_additional_gene_difference_kras_only",
              "rb1_cdkn2a_double",
              "pcna_essential_gene_control",
              "difference_is_oncogene_not_suppressor"
            ]
          },
          {
            "name": "cf_age_independent_tme",
            "role": "hard_fail",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_independent"
            ]
          },
          {
            "name": "cf_compositional_shift",
            "role": "hard_fail",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_frequency_shift",
              "compositional_immune_infiltration",
              "composition_dominant_state_secondary",
              "both_state_and_composition"
            ]
          },
          {
            "name": "cf_epithelial_or_cancer_substitution",
            "role": "hard_fail",
            "field": "$.most_responsive_noncancer_cell_types[*].cell_type",
            "kind": "avoid",
            "expected": [
              "AT2_cell",
              "AT1_cell",
              "cancer_cell",
              "ciliated",
              "secretory",
              "club_cell",
              "basal_cell",
              "goblet_cell"
            ]
          },
          {
            "name": "cf_tme_breadth_broader_in_young",
            "role": "hard_fail",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_young"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-6-pi",
      "modelName": "claude-opus-4-6",
      "harness": "pi",
      "task": "gse297023_q5_tme_age_dependent_remodeling",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 3485,
      "cost": 3.1791910000000008,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "perturbation_is_pten",
            "role": "gate",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "pten"
            ]
          },
          {
            "name": "tme_non_cell_autonomous",
            "role": "gate",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "substantial_non_cell_autonomous_remodeling"
            ]
          },
          {
            "name": "tme_state_not_composition",
            "role": "gate",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "tme_age_dependent",
            "role": "gate",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_dependent"
            ]
          },
          {
            "name": "tme_response_broader_in_aged",
            "role": "gate",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_aged"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "most_responsive_noncancer_cell_types",
            "kind": "list",
            "matchKey": "cell_type",
            "k": 2,
            "expected": [
              {
                "key": "alveolar_macrophage",
                "fields": {}
              },
              {
                "key": "endothelial",
                "fields": {}
              }
            ]
          },
          {
            "name": "cf_cell_intrinsic_only",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "negligible_cell_intrinsic_only_effect",
              "remodeling_unresolved",
              "modest_remodeling_few_cell_types"
            ]
          },
          {
            "name": "cf_remodeling_is_contamination",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "apparent_remodeling_reflects_cancer_contamination"
            ]
          },
          {
            "name": "cf_perturbation_design_misread",
            "role": "hard_fail",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "multiple_tumor_suppressors_pooled",
              "no_additional_gene_difference_kras_only",
              "rb1_cdkn2a_double",
              "pcna_essential_gene_control",
              "difference_is_oncogene_not_suppressor"
            ]
          },
          {
            "name": "cf_age_independent_tme",
            "role": "hard_fail",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_independent"
            ]
          },
          {
            "name": "cf_compositional_shift",
            "role": "hard_fail",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_frequency_shift",
              "compositional_immune_infiltration",
              "composition_dominant_state_secondary",
              "both_state_and_composition"
            ]
          },
          {
            "name": "cf_epithelial_or_cancer_substitution",
            "role": "hard_fail",
            "field": "$.most_responsive_noncancer_cell_types[*].cell_type",
            "kind": "avoid",
            "expected": [
              "AT2_cell",
              "AT1_cell",
              "cancer_cell",
              "ciliated",
              "secretory",
              "club_cell",
              "basal_cell",
              "goblet_cell"
            ]
          },
          {
            "name": "cf_tme_breadth_broader_in_young",
            "role": "hard_fail",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_young"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-6-pi",
      "modelName": "claude-opus-4-6",
      "harness": "pi",
      "task": "gse297023_q5_tme_age_dependent_remodeling",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 4452,
      "cost": 3.24993125,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "perturbation_is_pten",
            "role": "gate",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "pten"
            ]
          },
          {
            "name": "tme_non_cell_autonomous",
            "role": "gate",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "substantial_non_cell_autonomous_remodeling"
            ]
          },
          {
            "name": "tme_state_not_composition",
            "role": "gate",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "tme_age_dependent",
            "role": "gate",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_dependent"
            ]
          },
          {
            "name": "tme_response_broader_in_aged",
            "role": "gate",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_aged"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "most_responsive_noncancer_cell_types",
            "kind": "list",
            "matchKey": "cell_type",
            "k": 2,
            "expected": [
              {
                "key": "alveolar_macrophage",
                "fields": {}
              },
              {
                "key": "endothelial",
                "fields": {}
              }
            ]
          },
          {
            "name": "cf_cell_intrinsic_only",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "negligible_cell_intrinsic_only_effect",
              "remodeling_unresolved",
              "modest_remodeling_few_cell_types"
            ]
          },
          {
            "name": "cf_remodeling_is_contamination",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "apparent_remodeling_reflects_cancer_contamination"
            ]
          },
          {
            "name": "cf_perturbation_design_misread",
            "role": "hard_fail",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "multiple_tumor_suppressors_pooled",
              "no_additional_gene_difference_kras_only",
              "rb1_cdkn2a_double",
              "pcna_essential_gene_control",
              "difference_is_oncogene_not_suppressor"
            ]
          },
          {
            "name": "cf_age_independent_tme",
            "role": "hard_fail",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_independent"
            ]
          },
          {
            "name": "cf_compositional_shift",
            "role": "hard_fail",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_frequency_shift",
              "compositional_immune_infiltration",
              "composition_dominant_state_secondary",
              "both_state_and_composition"
            ]
          },
          {
            "name": "cf_epithelial_or_cancer_substitution",
            "role": "hard_fail",
            "field": "$.most_responsive_noncancer_cell_types[*].cell_type",
            "kind": "avoid",
            "expected": [
              "AT2_cell",
              "AT1_cell",
              "cancer_cell",
              "ciliated",
              "secretory",
              "club_cell",
              "basal_cell",
              "goblet_cell"
            ]
          },
          {
            "name": "cf_tme_breadth_broader_in_young",
            "role": "hard_fail",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_young"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-7-claude-code",
      "modelName": "claude-opus-4-7",
      "harness": "claude-code",
      "task": "gse297023_q5_tme_age_dependent_remodeling",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 2170,
      "cost": 4.718020750000001,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "perturbation_is_pten",
            "role": "gate",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "pten"
            ]
          },
          {
            "name": "tme_non_cell_autonomous",
            "role": "gate",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "substantial_non_cell_autonomous_remodeling"
            ]
          },
          {
            "name": "tme_state_not_composition",
            "role": "gate",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "tme_age_dependent",
            "role": "gate",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_dependent"
            ]
          },
          {
            "name": "tme_response_broader_in_aged",
            "role": "gate",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_aged"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "most_responsive_noncancer_cell_types",
            "kind": "list",
            "matchKey": "cell_type",
            "k": 2,
            "expected": [
              {
                "key": "alveolar_macrophage",
                "fields": {}
              },
              {
                "key": "endothelial",
                "fields": {}
              }
            ]
          },
          {
            "name": "cf_cell_intrinsic_only",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "negligible_cell_intrinsic_only_effect",
              "remodeling_unresolved",
              "modest_remodeling_few_cell_types"
            ]
          },
          {
            "name": "cf_remodeling_is_contamination",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "apparent_remodeling_reflects_cancer_contamination"
            ]
          },
          {
            "name": "cf_perturbation_design_misread",
            "role": "hard_fail",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "multiple_tumor_suppressors_pooled",
              "no_additional_gene_difference_kras_only",
              "rb1_cdkn2a_double",
              "pcna_essential_gene_control",
              "difference_is_oncogene_not_suppressor"
            ]
          },
          {
            "name": "cf_age_independent_tme",
            "role": "hard_fail",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_independent"
            ]
          },
          {
            "name": "cf_compositional_shift",
            "role": "hard_fail",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_frequency_shift",
              "compositional_immune_infiltration",
              "composition_dominant_state_secondary",
              "both_state_and_composition"
            ]
          },
          {
            "name": "cf_epithelial_or_cancer_substitution",
            "role": "hard_fail",
            "field": "$.most_responsive_noncancer_cell_types[*].cell_type",
            "kind": "avoid",
            "expected": [
              "AT2_cell",
              "AT1_cell",
              "cancer_cell",
              "ciliated",
              "secretory",
              "club_cell",
              "basal_cell",
              "goblet_cell"
            ]
          },
          {
            "name": "cf_tme_breadth_broader_in_young",
            "role": "hard_fail",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_young"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-7-claude-code",
      "modelName": "claude-opus-4-7",
      "harness": "claude-code",
      "task": "gse297023_q5_tme_age_dependent_remodeling",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 1550,
      "cost": 5.962710000000001,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "perturbation_is_pten",
            "role": "gate",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "pten"
            ]
          },
          {
            "name": "tme_non_cell_autonomous",
            "role": "gate",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "substantial_non_cell_autonomous_remodeling"
            ]
          },
          {
            "name": "tme_state_not_composition",
            "role": "gate",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "tme_age_dependent",
            "role": "gate",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_dependent"
            ]
          },
          {
            "name": "tme_response_broader_in_aged",
            "role": "gate",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_aged"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "most_responsive_noncancer_cell_types",
            "kind": "list",
            "matchKey": "cell_type",
            "k": 2,
            "expected": [
              {
                "key": "alveolar_macrophage",
                "fields": {}
              },
              {
                "key": "endothelial",
                "fields": {}
              }
            ]
          },
          {
            "name": "cf_cell_intrinsic_only",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "negligible_cell_intrinsic_only_effect",
              "remodeling_unresolved",
              "modest_remodeling_few_cell_types"
            ]
          },
          {
            "name": "cf_remodeling_is_contamination",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "apparent_remodeling_reflects_cancer_contamination"
            ]
          },
          {
            "name": "cf_perturbation_design_misread",
            "role": "hard_fail",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "multiple_tumor_suppressors_pooled",
              "no_additional_gene_difference_kras_only",
              "rb1_cdkn2a_double",
              "pcna_essential_gene_control",
              "difference_is_oncogene_not_suppressor"
            ]
          },
          {
            "name": "cf_age_independent_tme",
            "role": "hard_fail",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_independent"
            ]
          },
          {
            "name": "cf_compositional_shift",
            "role": "hard_fail",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_frequency_shift",
              "compositional_immune_infiltration",
              "composition_dominant_state_secondary",
              "both_state_and_composition"
            ]
          },
          {
            "name": "cf_epithelial_or_cancer_substitution",
            "role": "hard_fail",
            "field": "$.most_responsive_noncancer_cell_types[*].cell_type",
            "kind": "avoid",
            "expected": [
              "AT2_cell",
              "AT1_cell",
              "cancer_cell",
              "ciliated",
              "secretory",
              "club_cell",
              "basal_cell",
              "goblet_cell"
            ]
          },
          {
            "name": "cf_tme_breadth_broader_in_young",
            "role": "hard_fail",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_young"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-7-claude-code",
      "modelName": "claude-opus-4-7",
      "harness": "claude-code",
      "task": "gse297023_q5_tme_age_dependent_remodeling",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 924,
      "cost": 1.30830425,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "perturbation_is_pten",
            "role": "gate",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "pten"
            ]
          },
          {
            "name": "tme_non_cell_autonomous",
            "role": "gate",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "substantial_non_cell_autonomous_remodeling"
            ]
          },
          {
            "name": "tme_state_not_composition",
            "role": "gate",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "tme_age_dependent",
            "role": "gate",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_dependent"
            ]
          },
          {
            "name": "tme_response_broader_in_aged",
            "role": "gate",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_aged"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "most_responsive_noncancer_cell_types",
            "kind": "list",
            "matchKey": "cell_type",
            "k": 2,
            "expected": [
              {
                "key": "alveolar_macrophage",
                "fields": {}
              },
              {
                "key": "endothelial",
                "fields": {}
              }
            ]
          },
          {
            "name": "cf_cell_intrinsic_only",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "negligible_cell_intrinsic_only_effect",
              "remodeling_unresolved",
              "modest_remodeling_few_cell_types"
            ]
          },
          {
            "name": "cf_remodeling_is_contamination",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "apparent_remodeling_reflects_cancer_contamination"
            ]
          },
          {
            "name": "cf_perturbation_design_misread",
            "role": "hard_fail",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "multiple_tumor_suppressors_pooled",
              "no_additional_gene_difference_kras_only",
              "rb1_cdkn2a_double",
              "pcna_essential_gene_control",
              "difference_is_oncogene_not_suppressor"
            ]
          },
          {
            "name": "cf_age_independent_tme",
            "role": "hard_fail",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_independent"
            ]
          },
          {
            "name": "cf_compositional_shift",
            "role": "hard_fail",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_frequency_shift",
              "compositional_immune_infiltration",
              "composition_dominant_state_secondary",
              "both_state_and_composition"
            ]
          },
          {
            "name": "cf_epithelial_or_cancer_substitution",
            "role": "hard_fail",
            "field": "$.most_responsive_noncancer_cell_types[*].cell_type",
            "kind": "avoid",
            "expected": [
              "AT2_cell",
              "AT1_cell",
              "cancer_cell",
              "ciliated",
              "secretory",
              "club_cell",
              "basal_cell",
              "goblet_cell"
            ]
          },
          {
            "name": "cf_tme_breadth_broader_in_young",
            "role": "hard_fail",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_young"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-7-pi",
      "modelName": "claude-opus-4-7",
      "harness": "pi",
      "task": "gse297023_q5_tme_age_dependent_remodeling",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 4646,
      "cost": 3.2202632499999995,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "perturbation_is_pten",
            "role": "gate",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "pten"
            ]
          },
          {
            "name": "tme_non_cell_autonomous",
            "role": "gate",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "substantial_non_cell_autonomous_remodeling"
            ]
          },
          {
            "name": "tme_state_not_composition",
            "role": "gate",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "tme_age_dependent",
            "role": "gate",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_dependent"
            ]
          },
          {
            "name": "tme_response_broader_in_aged",
            "role": "gate",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_aged"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "most_responsive_noncancer_cell_types",
            "kind": "list",
            "matchKey": "cell_type",
            "k": 2,
            "expected": [
              {
                "key": "alveolar_macrophage",
                "fields": {}
              },
              {
                "key": "endothelial",
                "fields": {}
              }
            ]
          },
          {
            "name": "cf_cell_intrinsic_only",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "negligible_cell_intrinsic_only_effect",
              "remodeling_unresolved",
              "modest_remodeling_few_cell_types"
            ]
          },
          {
            "name": "cf_remodeling_is_contamination",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "apparent_remodeling_reflects_cancer_contamination"
            ]
          },
          {
            "name": "cf_perturbation_design_misread",
            "role": "hard_fail",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "multiple_tumor_suppressors_pooled",
              "no_additional_gene_difference_kras_only",
              "rb1_cdkn2a_double",
              "pcna_essential_gene_control",
              "difference_is_oncogene_not_suppressor"
            ]
          },
          {
            "name": "cf_age_independent_tme",
            "role": "hard_fail",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_independent"
            ]
          },
          {
            "name": "cf_compositional_shift",
            "role": "hard_fail",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_frequency_shift",
              "compositional_immune_infiltration",
              "composition_dominant_state_secondary",
              "both_state_and_composition"
            ]
          },
          {
            "name": "cf_epithelial_or_cancer_substitution",
            "role": "hard_fail",
            "field": "$.most_responsive_noncancer_cell_types[*].cell_type",
            "kind": "avoid",
            "expected": [
              "AT2_cell",
              "AT1_cell",
              "cancer_cell",
              "ciliated",
              "secretory",
              "club_cell",
              "basal_cell",
              "goblet_cell"
            ]
          },
          {
            "name": "cf_tme_breadth_broader_in_young",
            "role": "hard_fail",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_young"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-7-pi",
      "modelName": "claude-opus-4-7",
      "harness": "pi",
      "task": "gse297023_q5_tme_age_dependent_remodeling",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 1800,
      "cost": 1.6026437500000001,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "perturbation_is_pten",
            "role": "gate",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "pten"
            ]
          },
          {
            "name": "tme_non_cell_autonomous",
            "role": "gate",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "substantial_non_cell_autonomous_remodeling"
            ]
          },
          {
            "name": "tme_state_not_composition",
            "role": "gate",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "tme_age_dependent",
            "role": "gate",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_dependent"
            ]
          },
          {
            "name": "tme_response_broader_in_aged",
            "role": "gate",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_aged"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "most_responsive_noncancer_cell_types",
            "kind": "list",
            "matchKey": "cell_type",
            "k": 2,
            "expected": [
              {
                "key": "alveolar_macrophage",
                "fields": {}
              },
              {
                "key": "endothelial",
                "fields": {}
              }
            ]
          },
          {
            "name": "cf_cell_intrinsic_only",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "negligible_cell_intrinsic_only_effect",
              "remodeling_unresolved",
              "modest_remodeling_few_cell_types"
            ]
          },
          {
            "name": "cf_remodeling_is_contamination",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "apparent_remodeling_reflects_cancer_contamination"
            ]
          },
          {
            "name": "cf_perturbation_design_misread",
            "role": "hard_fail",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "multiple_tumor_suppressors_pooled",
              "no_additional_gene_difference_kras_only",
              "rb1_cdkn2a_double",
              "pcna_essential_gene_control",
              "difference_is_oncogene_not_suppressor"
            ]
          },
          {
            "name": "cf_age_independent_tme",
            "role": "hard_fail",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_independent"
            ]
          },
          {
            "name": "cf_compositional_shift",
            "role": "hard_fail",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_frequency_shift",
              "compositional_immune_infiltration",
              "composition_dominant_state_secondary",
              "both_state_and_composition"
            ]
          },
          {
            "name": "cf_epithelial_or_cancer_substitution",
            "role": "hard_fail",
            "field": "$.most_responsive_noncancer_cell_types[*].cell_type",
            "kind": "avoid",
            "expected": [
              "AT2_cell",
              "AT1_cell",
              "cancer_cell",
              "ciliated",
              "secretory",
              "club_cell",
              "basal_cell",
              "goblet_cell"
            ]
          },
          {
            "name": "cf_tme_breadth_broader_in_young",
            "role": "hard_fail",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_young"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-7-pi",
      "modelName": "claude-opus-4-7",
      "harness": "pi",
      "task": "gse297023_q5_tme_age_dependent_remodeling",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 5005,
      "cost": 3.686864999999999,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "perturbation_is_pten",
            "role": "gate",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "pten"
            ]
          },
          {
            "name": "tme_non_cell_autonomous",
            "role": "gate",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "substantial_non_cell_autonomous_remodeling"
            ]
          },
          {
            "name": "tme_state_not_composition",
            "role": "gate",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "tme_age_dependent",
            "role": "gate",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_dependent"
            ]
          },
          {
            "name": "tme_response_broader_in_aged",
            "role": "gate",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_aged"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "most_responsive_noncancer_cell_types",
            "kind": "list",
            "matchKey": "cell_type",
            "k": 2,
            "expected": [
              {
                "key": "alveolar_macrophage",
                "fields": {}
              },
              {
                "key": "endothelial",
                "fields": {}
              }
            ]
          },
          {
            "name": "cf_cell_intrinsic_only",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "negligible_cell_intrinsic_only_effect",
              "remodeling_unresolved",
              "modest_remodeling_few_cell_types"
            ]
          },
          {
            "name": "cf_remodeling_is_contamination",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "apparent_remodeling_reflects_cancer_contamination"
            ]
          },
          {
            "name": "cf_perturbation_design_misread",
            "role": "hard_fail",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "multiple_tumor_suppressors_pooled",
              "no_additional_gene_difference_kras_only",
              "rb1_cdkn2a_double",
              "pcna_essential_gene_control",
              "difference_is_oncogene_not_suppressor"
            ]
          },
          {
            "name": "cf_age_independent_tme",
            "role": "hard_fail",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_independent"
            ]
          },
          {
            "name": "cf_compositional_shift",
            "role": "hard_fail",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_frequency_shift",
              "compositional_immune_infiltration",
              "composition_dominant_state_secondary",
              "both_state_and_composition"
            ]
          },
          {
            "name": "cf_epithelial_or_cancer_substitution",
            "role": "hard_fail",
            "field": "$.most_responsive_noncancer_cell_types[*].cell_type",
            "kind": "avoid",
            "expected": [
              "AT2_cell",
              "AT1_cell",
              "cancer_cell",
              "ciliated",
              "secretory",
              "club_cell",
              "basal_cell",
              "goblet_cell"
            ]
          },
          {
            "name": "cf_tme_breadth_broader_in_young",
            "role": "hard_fail",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_young"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-8-claude-code",
      "modelName": "claude-opus-4-8",
      "harness": "claude-code",
      "task": "gse297023_q5_tme_age_dependent_remodeling",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 600,
      "cost": 2.3876910000000002,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "perturbation_is_pten",
            "role": "gate",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "pten"
            ]
          },
          {
            "name": "tme_non_cell_autonomous",
            "role": "gate",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "substantial_non_cell_autonomous_remodeling"
            ]
          },
          {
            "name": "tme_state_not_composition",
            "role": "gate",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "tme_age_dependent",
            "role": "gate",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_dependent"
            ]
          },
          {
            "name": "tme_response_broader_in_aged",
            "role": "gate",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_aged"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "most_responsive_noncancer_cell_types",
            "kind": "list",
            "matchKey": "cell_type",
            "k": 2,
            "expected": [
              {
                "key": "alveolar_macrophage",
                "fields": {}
              },
              {
                "key": "endothelial",
                "fields": {}
              }
            ]
          },
          {
            "name": "cf_cell_intrinsic_only",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "negligible_cell_intrinsic_only_effect",
              "remodeling_unresolved",
              "modest_remodeling_few_cell_types"
            ]
          },
          {
            "name": "cf_remodeling_is_contamination",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "apparent_remodeling_reflects_cancer_contamination"
            ]
          },
          {
            "name": "cf_perturbation_design_misread",
            "role": "hard_fail",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "multiple_tumor_suppressors_pooled",
              "no_additional_gene_difference_kras_only",
              "rb1_cdkn2a_double",
              "pcna_essential_gene_control",
              "difference_is_oncogene_not_suppressor"
            ]
          },
          {
            "name": "cf_age_independent_tme",
            "role": "hard_fail",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_independent"
            ]
          },
          {
            "name": "cf_compositional_shift",
            "role": "hard_fail",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_frequency_shift",
              "compositional_immune_infiltration",
              "composition_dominant_state_secondary",
              "both_state_and_composition"
            ]
          },
          {
            "name": "cf_epithelial_or_cancer_substitution",
            "role": "hard_fail",
            "field": "$.most_responsive_noncancer_cell_types[*].cell_type",
            "kind": "avoid",
            "expected": [
              "AT2_cell",
              "AT1_cell",
              "cancer_cell",
              "ciliated",
              "secretory",
              "club_cell",
              "basal_cell",
              "goblet_cell"
            ]
          },
          {
            "name": "cf_tme_breadth_broader_in_young",
            "role": "hard_fail",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_young"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-8-claude-code",
      "modelName": "claude-opus-4-8",
      "harness": "claude-code",
      "task": "gse297023_q5_tme_age_dependent_remodeling",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 634,
      "cost": 2.0688617499999995,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "perturbation_is_pten",
            "role": "gate",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "pten"
            ]
          },
          {
            "name": "tme_non_cell_autonomous",
            "role": "gate",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "substantial_non_cell_autonomous_remodeling"
            ]
          },
          {
            "name": "tme_state_not_composition",
            "role": "gate",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "tme_age_dependent",
            "role": "gate",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_dependent"
            ]
          },
          {
            "name": "tme_response_broader_in_aged",
            "role": "gate",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_aged"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "most_responsive_noncancer_cell_types",
            "kind": "list",
            "matchKey": "cell_type",
            "k": 2,
            "expected": [
              {
                "key": "alveolar_macrophage",
                "fields": {}
              },
              {
                "key": "endothelial",
                "fields": {}
              }
            ]
          },
          {
            "name": "cf_cell_intrinsic_only",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "negligible_cell_intrinsic_only_effect",
              "remodeling_unresolved",
              "modest_remodeling_few_cell_types"
            ]
          },
          {
            "name": "cf_remodeling_is_contamination",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "apparent_remodeling_reflects_cancer_contamination"
            ]
          },
          {
            "name": "cf_perturbation_design_misread",
            "role": "hard_fail",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "multiple_tumor_suppressors_pooled",
              "no_additional_gene_difference_kras_only",
              "rb1_cdkn2a_double",
              "pcna_essential_gene_control",
              "difference_is_oncogene_not_suppressor"
            ]
          },
          {
            "name": "cf_age_independent_tme",
            "role": "hard_fail",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_independent"
            ]
          },
          {
            "name": "cf_compositional_shift",
            "role": "hard_fail",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_frequency_shift",
              "compositional_immune_infiltration",
              "composition_dominant_state_secondary",
              "both_state_and_composition"
            ]
          },
          {
            "name": "cf_epithelial_or_cancer_substitution",
            "role": "hard_fail",
            "field": "$.most_responsive_noncancer_cell_types[*].cell_type",
            "kind": "avoid",
            "expected": [
              "AT2_cell",
              "AT1_cell",
              "cancer_cell",
              "ciliated",
              "secretory",
              "club_cell",
              "basal_cell",
              "goblet_cell"
            ]
          },
          {
            "name": "cf_tme_breadth_broader_in_young",
            "role": "hard_fail",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_young"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-8-claude-code",
      "modelName": "claude-opus-4-8",
      "harness": "claude-code",
      "task": "gse297023_q5_tme_age_dependent_remodeling",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1451,
      "cost": 5.516762000000001,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "perturbation_is_pten",
            "role": "gate",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "pten"
            ]
          },
          {
            "name": "tme_non_cell_autonomous",
            "role": "gate",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "substantial_non_cell_autonomous_remodeling"
            ]
          },
          {
            "name": "tme_state_not_composition",
            "role": "gate",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "tme_age_dependent",
            "role": "gate",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_dependent"
            ]
          },
          {
            "name": "tme_response_broader_in_aged",
            "role": "gate",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_aged"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "most_responsive_noncancer_cell_types",
            "kind": "list",
            "matchKey": "cell_type",
            "k": 2,
            "expected": [
              {
                "key": "alveolar_macrophage",
                "fields": {}
              },
              {
                "key": "endothelial",
                "fields": {}
              }
            ]
          },
          {
            "name": "cf_cell_intrinsic_only",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "negligible_cell_intrinsic_only_effect",
              "remodeling_unresolved",
              "modest_remodeling_few_cell_types"
            ]
          },
          {
            "name": "cf_remodeling_is_contamination",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "apparent_remodeling_reflects_cancer_contamination"
            ]
          },
          {
            "name": "cf_perturbation_design_misread",
            "role": "hard_fail",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "multiple_tumor_suppressors_pooled",
              "no_additional_gene_difference_kras_only",
              "rb1_cdkn2a_double",
              "pcna_essential_gene_control",
              "difference_is_oncogene_not_suppressor"
            ]
          },
          {
            "name": "cf_age_independent_tme",
            "role": "hard_fail",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_independent"
            ]
          },
          {
            "name": "cf_compositional_shift",
            "role": "hard_fail",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_frequency_shift",
              "compositional_immune_infiltration",
              "composition_dominant_state_secondary",
              "both_state_and_composition"
            ]
          },
          {
            "name": "cf_epithelial_or_cancer_substitution",
            "role": "hard_fail",
            "field": "$.most_responsive_noncancer_cell_types[*].cell_type",
            "kind": "avoid",
            "expected": [
              "AT2_cell",
              "AT1_cell",
              "cancer_cell",
              "ciliated",
              "secretory",
              "club_cell",
              "basal_cell",
              "goblet_cell"
            ]
          },
          {
            "name": "cf_tme_breadth_broader_in_young",
            "role": "hard_fail",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_young"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-8-pi",
      "modelName": "claude-opus-4-8",
      "harness": "pi",
      "task": "gse297023_q5_tme_age_dependent_remodeling",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 3244,
      "cost": 2.5414135000000004,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "perturbation_is_pten",
            "role": "gate",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "pten"
            ]
          },
          {
            "name": "tme_non_cell_autonomous",
            "role": "gate",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "substantial_non_cell_autonomous_remodeling"
            ]
          },
          {
            "name": "tme_state_not_composition",
            "role": "gate",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "tme_age_dependent",
            "role": "gate",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_dependent"
            ]
          },
          {
            "name": "tme_response_broader_in_aged",
            "role": "gate",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_aged"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "most_responsive_noncancer_cell_types",
            "kind": "list",
            "matchKey": "cell_type",
            "k": 2,
            "expected": [
              {
                "key": "alveolar_macrophage",
                "fields": {}
              },
              {
                "key": "endothelial",
                "fields": {}
              }
            ]
          },
          {
            "name": "cf_cell_intrinsic_only",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "negligible_cell_intrinsic_only_effect",
              "remodeling_unresolved",
              "modest_remodeling_few_cell_types"
            ]
          },
          {
            "name": "cf_remodeling_is_contamination",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "apparent_remodeling_reflects_cancer_contamination"
            ]
          },
          {
            "name": "cf_perturbation_design_misread",
            "role": "hard_fail",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "multiple_tumor_suppressors_pooled",
              "no_additional_gene_difference_kras_only",
              "rb1_cdkn2a_double",
              "pcna_essential_gene_control",
              "difference_is_oncogene_not_suppressor"
            ]
          },
          {
            "name": "cf_age_independent_tme",
            "role": "hard_fail",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_independent"
            ]
          },
          {
            "name": "cf_compositional_shift",
            "role": "hard_fail",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_frequency_shift",
              "compositional_immune_infiltration",
              "composition_dominant_state_secondary",
              "both_state_and_composition"
            ]
          },
          {
            "name": "cf_epithelial_or_cancer_substitution",
            "role": "hard_fail",
            "field": "$.most_responsive_noncancer_cell_types[*].cell_type",
            "kind": "avoid",
            "expected": [
              "AT2_cell",
              "AT1_cell",
              "cancer_cell",
              "ciliated",
              "secretory",
              "club_cell",
              "basal_cell",
              "goblet_cell"
            ]
          },
          {
            "name": "cf_tme_breadth_broader_in_young",
            "role": "hard_fail",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_young"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-8-pi",
      "modelName": "claude-opus-4-8",
      "harness": "pi",
      "task": "gse297023_q5_tme_age_dependent_remodeling",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 6769,
      "cost": 4.441061250000001,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "perturbation_is_pten",
            "role": "gate",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "pten"
            ]
          },
          {
            "name": "tme_non_cell_autonomous",
            "role": "gate",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "substantial_non_cell_autonomous_remodeling"
            ]
          },
          {
            "name": "tme_state_not_composition",
            "role": "gate",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "tme_age_dependent",
            "role": "gate",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_dependent"
            ]
          },
          {
            "name": "tme_response_broader_in_aged",
            "role": "gate",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_aged"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "most_responsive_noncancer_cell_types",
            "kind": "list",
            "matchKey": "cell_type",
            "k": 2,
            "expected": [
              {
                "key": "alveolar_macrophage",
                "fields": {}
              },
              {
                "key": "endothelial",
                "fields": {}
              }
            ]
          },
          {
            "name": "cf_cell_intrinsic_only",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "negligible_cell_intrinsic_only_effect",
              "remodeling_unresolved",
              "modest_remodeling_few_cell_types"
            ]
          },
          {
            "name": "cf_remodeling_is_contamination",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "apparent_remodeling_reflects_cancer_contamination"
            ]
          },
          {
            "name": "cf_perturbation_design_misread",
            "role": "hard_fail",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "multiple_tumor_suppressors_pooled",
              "no_additional_gene_difference_kras_only",
              "rb1_cdkn2a_double",
              "pcna_essential_gene_control",
              "difference_is_oncogene_not_suppressor"
            ]
          },
          {
            "name": "cf_age_independent_tme",
            "role": "hard_fail",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_independent"
            ]
          },
          {
            "name": "cf_compositional_shift",
            "role": "hard_fail",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_frequency_shift",
              "compositional_immune_infiltration",
              "composition_dominant_state_secondary",
              "both_state_and_composition"
            ]
          },
          {
            "name": "cf_epithelial_or_cancer_substitution",
            "role": "hard_fail",
            "field": "$.most_responsive_noncancer_cell_types[*].cell_type",
            "kind": "avoid",
            "expected": [
              "AT2_cell",
              "AT1_cell",
              "cancer_cell",
              "ciliated",
              "secretory",
              "club_cell",
              "basal_cell",
              "goblet_cell"
            ]
          },
          {
            "name": "cf_tme_breadth_broader_in_young",
            "role": "hard_fail",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_young"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-8-pi",
      "modelName": "claude-opus-4-8",
      "harness": "pi",
      "task": "gse297023_q5_tme_age_dependent_remodeling",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 4351,
      "cost": 6.4718675,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "perturbation_is_pten",
            "role": "gate",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "pten"
            ]
          },
          {
            "name": "tme_non_cell_autonomous",
            "role": "gate",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "substantial_non_cell_autonomous_remodeling"
            ]
          },
          {
            "name": "tme_state_not_composition",
            "role": "gate",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "tme_age_dependent",
            "role": "gate",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_dependent"
            ]
          },
          {
            "name": "tme_response_broader_in_aged",
            "role": "gate",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_aged"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "most_responsive_noncancer_cell_types",
            "kind": "list",
            "matchKey": "cell_type",
            "k": 2,
            "expected": [
              {
                "key": "alveolar_macrophage",
                "fields": {}
              },
              {
                "key": "endothelial",
                "fields": {}
              }
            ]
          },
          {
            "name": "cf_cell_intrinsic_only",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "negligible_cell_intrinsic_only_effect",
              "remodeling_unresolved",
              "modest_remodeling_few_cell_types"
            ]
          },
          {
            "name": "cf_remodeling_is_contamination",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "apparent_remodeling_reflects_cancer_contamination"
            ]
          },
          {
            "name": "cf_perturbation_design_misread",
            "role": "hard_fail",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "multiple_tumor_suppressors_pooled",
              "no_additional_gene_difference_kras_only",
              "rb1_cdkn2a_double",
              "pcna_essential_gene_control",
              "difference_is_oncogene_not_suppressor"
            ]
          },
          {
            "name": "cf_age_independent_tme",
            "role": "hard_fail",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_independent"
            ]
          },
          {
            "name": "cf_compositional_shift",
            "role": "hard_fail",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_frequency_shift",
              "compositional_immune_infiltration",
              "composition_dominant_state_secondary",
              "both_state_and_composition"
            ]
          },
          {
            "name": "cf_epithelial_or_cancer_substitution",
            "role": "hard_fail",
            "field": "$.most_responsive_noncancer_cell_types[*].cell_type",
            "kind": "avoid",
            "expected": [
              "AT2_cell",
              "AT1_cell",
              "cancer_cell",
              "ciliated",
              "secretory",
              "club_cell",
              "basal_cell",
              "goblet_cell"
            ]
          },
          {
            "name": "cf_tme_breadth_broader_in_young",
            "role": "hard_fail",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_young"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-4-6-pi",
      "modelName": "claude-sonnet-4-6",
      "harness": "pi",
      "task": "gse297023_q5_tme_age_dependent_remodeling",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 3046,
      "cost": 1.822098,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "perturbation_is_pten",
            "role": "gate",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "pten"
            ]
          },
          {
            "name": "tme_non_cell_autonomous",
            "role": "gate",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "substantial_non_cell_autonomous_remodeling"
            ]
          },
          {
            "name": "tme_state_not_composition",
            "role": "gate",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "tme_age_dependent",
            "role": "gate",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_dependent"
            ]
          },
          {
            "name": "tme_response_broader_in_aged",
            "role": "gate",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_aged"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "most_responsive_noncancer_cell_types",
            "kind": "list",
            "matchKey": "cell_type",
            "k": 2,
            "expected": [
              {
                "key": "alveolar_macrophage",
                "fields": {}
              },
              {
                "key": "endothelial",
                "fields": {}
              }
            ]
          },
          {
            "name": "cf_cell_intrinsic_only",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "negligible_cell_intrinsic_only_effect",
              "remodeling_unresolved",
              "modest_remodeling_few_cell_types"
            ]
          },
          {
            "name": "cf_remodeling_is_contamination",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "apparent_remodeling_reflects_cancer_contamination"
            ]
          },
          {
            "name": "cf_perturbation_design_misread",
            "role": "hard_fail",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "multiple_tumor_suppressors_pooled",
              "no_additional_gene_difference_kras_only",
              "rb1_cdkn2a_double",
              "pcna_essential_gene_control",
              "difference_is_oncogene_not_suppressor"
            ]
          },
          {
            "name": "cf_age_independent_tme",
            "role": "hard_fail",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_independent"
            ]
          },
          {
            "name": "cf_compositional_shift",
            "role": "hard_fail",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_frequency_shift",
              "compositional_immune_infiltration",
              "composition_dominant_state_secondary",
              "both_state_and_composition"
            ]
          },
          {
            "name": "cf_epithelial_or_cancer_substitution",
            "role": "hard_fail",
            "field": "$.most_responsive_noncancer_cell_types[*].cell_type",
            "kind": "avoid",
            "expected": [
              "AT2_cell",
              "AT1_cell",
              "cancer_cell",
              "ciliated",
              "secretory",
              "club_cell",
              "basal_cell",
              "goblet_cell"
            ]
          },
          {
            "name": "cf_tme_breadth_broader_in_young",
            "role": "hard_fail",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_young"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-4-6-pi",
      "modelName": "claude-sonnet-4-6",
      "harness": "pi",
      "task": "gse297023_q5_tme_age_dependent_remodeling",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 3811,
      "cost": 2.6872026,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "perturbation_is_pten",
            "role": "gate",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "pten"
            ]
          },
          {
            "name": "tme_non_cell_autonomous",
            "role": "gate",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "substantial_non_cell_autonomous_remodeling"
            ]
          },
          {
            "name": "tme_state_not_composition",
            "role": "gate",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "tme_age_dependent",
            "role": "gate",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_dependent"
            ]
          },
          {
            "name": "tme_response_broader_in_aged",
            "role": "gate",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_aged"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "most_responsive_noncancer_cell_types",
            "kind": "list",
            "matchKey": "cell_type",
            "k": 2,
            "expected": [
              {
                "key": "alveolar_macrophage",
                "fields": {}
              },
              {
                "key": "endothelial",
                "fields": {}
              }
            ]
          },
          {
            "name": "cf_cell_intrinsic_only",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "negligible_cell_intrinsic_only_effect",
              "remodeling_unresolved",
              "modest_remodeling_few_cell_types"
            ]
          },
          {
            "name": "cf_remodeling_is_contamination",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "apparent_remodeling_reflects_cancer_contamination"
            ]
          },
          {
            "name": "cf_perturbation_design_misread",
            "role": "hard_fail",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "multiple_tumor_suppressors_pooled",
              "no_additional_gene_difference_kras_only",
              "rb1_cdkn2a_double",
              "pcna_essential_gene_control",
              "difference_is_oncogene_not_suppressor"
            ]
          },
          {
            "name": "cf_age_independent_tme",
            "role": "hard_fail",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_independent"
            ]
          },
          {
            "name": "cf_compositional_shift",
            "role": "hard_fail",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_frequency_shift",
              "compositional_immune_infiltration",
              "composition_dominant_state_secondary",
              "both_state_and_composition"
            ]
          },
          {
            "name": "cf_epithelial_or_cancer_substitution",
            "role": "hard_fail",
            "field": "$.most_responsive_noncancer_cell_types[*].cell_type",
            "kind": "avoid",
            "expected": [
              "AT2_cell",
              "AT1_cell",
              "cancer_cell",
              "ciliated",
              "secretory",
              "club_cell",
              "basal_cell",
              "goblet_cell"
            ]
          },
          {
            "name": "cf_tme_breadth_broader_in_young",
            "role": "hard_fail",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_young"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-4-6-pi",
      "modelName": "claude-sonnet-4-6",
      "harness": "pi",
      "task": "gse297023_q5_tme_age_dependent_remodeling",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 5156,
      "cost": 3.4973484,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "perturbation_is_pten",
            "role": "gate",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "pten"
            ]
          },
          {
            "name": "tme_non_cell_autonomous",
            "role": "gate",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "substantial_non_cell_autonomous_remodeling"
            ]
          },
          {
            "name": "tme_state_not_composition",
            "role": "gate",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "tme_age_dependent",
            "role": "gate",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_dependent"
            ]
          },
          {
            "name": "tme_response_broader_in_aged",
            "role": "gate",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_aged"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "most_responsive_noncancer_cell_types",
            "kind": "list",
            "matchKey": "cell_type",
            "k": 2,
            "expected": [
              {
                "key": "alveolar_macrophage",
                "fields": {}
              },
              {
                "key": "endothelial",
                "fields": {}
              }
            ]
          },
          {
            "name": "cf_cell_intrinsic_only",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "negligible_cell_intrinsic_only_effect",
              "remodeling_unresolved",
              "modest_remodeling_few_cell_types"
            ]
          },
          {
            "name": "cf_remodeling_is_contamination",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "apparent_remodeling_reflects_cancer_contamination"
            ]
          },
          {
            "name": "cf_perturbation_design_misread",
            "role": "hard_fail",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "multiple_tumor_suppressors_pooled",
              "no_additional_gene_difference_kras_only",
              "rb1_cdkn2a_double",
              "pcna_essential_gene_control",
              "difference_is_oncogene_not_suppressor"
            ]
          },
          {
            "name": "cf_age_independent_tme",
            "role": "hard_fail",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_independent"
            ]
          },
          {
            "name": "cf_compositional_shift",
            "role": "hard_fail",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_frequency_shift",
              "compositional_immune_infiltration",
              "composition_dominant_state_secondary",
              "both_state_and_composition"
            ]
          },
          {
            "name": "cf_epithelial_or_cancer_substitution",
            "role": "hard_fail",
            "field": "$.most_responsive_noncancer_cell_types[*].cell_type",
            "kind": "avoid",
            "expected": [
              "AT2_cell",
              "AT1_cell",
              "cancer_cell",
              "ciliated",
              "secretory",
              "club_cell",
              "basal_cell",
              "goblet_cell"
            ]
          },
          {
            "name": "cf_tme_breadth_broader_in_young",
            "role": "hard_fail",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_young"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gemini-3-1-pro-preview-pi",
      "modelName": "gemini-3.1-pro-preview",
      "harness": "pi",
      "task": "gse297023_q5_tme_age_dependent_remodeling",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 1972,
      "cost": 0.6290222,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "perturbation_is_pten",
            "role": "gate",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "pten"
            ]
          },
          {
            "name": "tme_non_cell_autonomous",
            "role": "gate",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "substantial_non_cell_autonomous_remodeling"
            ]
          },
          {
            "name": "tme_state_not_composition",
            "role": "gate",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "tme_age_dependent",
            "role": "gate",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_dependent"
            ]
          },
          {
            "name": "tme_response_broader_in_aged",
            "role": "gate",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_aged"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "most_responsive_noncancer_cell_types",
            "kind": "list",
            "matchKey": "cell_type",
            "k": 2,
            "expected": [
              {
                "key": "alveolar_macrophage",
                "fields": {}
              },
              {
                "key": "endothelial",
                "fields": {}
              }
            ]
          },
          {
            "name": "cf_cell_intrinsic_only",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "negligible_cell_intrinsic_only_effect",
              "remodeling_unresolved",
              "modest_remodeling_few_cell_types"
            ]
          },
          {
            "name": "cf_remodeling_is_contamination",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "apparent_remodeling_reflects_cancer_contamination"
            ]
          },
          {
            "name": "cf_perturbation_design_misread",
            "role": "hard_fail",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "multiple_tumor_suppressors_pooled",
              "no_additional_gene_difference_kras_only",
              "rb1_cdkn2a_double",
              "pcna_essential_gene_control",
              "difference_is_oncogene_not_suppressor"
            ]
          },
          {
            "name": "cf_age_independent_tme",
            "role": "hard_fail",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_independent"
            ]
          },
          {
            "name": "cf_compositional_shift",
            "role": "hard_fail",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_frequency_shift",
              "compositional_immune_infiltration",
              "composition_dominant_state_secondary",
              "both_state_and_composition"
            ]
          },
          {
            "name": "cf_epithelial_or_cancer_substitution",
            "role": "hard_fail",
            "field": "$.most_responsive_noncancer_cell_types[*].cell_type",
            "kind": "avoid",
            "expected": [
              "AT2_cell",
              "AT1_cell",
              "cancer_cell",
              "ciliated",
              "secretory",
              "club_cell",
              "basal_cell",
              "goblet_cell"
            ]
          },
          {
            "name": "cf_tme_breadth_broader_in_young",
            "role": "hard_fail",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_young"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gemini-3-1-pro-preview-pi",
      "modelName": "gemini-3.1-pro-preview",
      "harness": "pi",
      "task": "gse297023_q5_tme_age_dependent_remodeling",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 857,
      "cost": 0.5300908,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "perturbation_is_pten",
            "role": "gate",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "pten"
            ]
          },
          {
            "name": "tme_non_cell_autonomous",
            "role": "gate",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "substantial_non_cell_autonomous_remodeling"
            ]
          },
          {
            "name": "tme_state_not_composition",
            "role": "gate",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "tme_age_dependent",
            "role": "gate",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_dependent"
            ]
          },
          {
            "name": "tme_response_broader_in_aged",
            "role": "gate",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_aged"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "most_responsive_noncancer_cell_types",
            "kind": "list",
            "matchKey": "cell_type",
            "k": 2,
            "expected": [
              {
                "key": "alveolar_macrophage",
                "fields": {}
              },
              {
                "key": "endothelial",
                "fields": {}
              }
            ]
          },
          {
            "name": "cf_cell_intrinsic_only",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "negligible_cell_intrinsic_only_effect",
              "remodeling_unresolved",
              "modest_remodeling_few_cell_types"
            ]
          },
          {
            "name": "cf_remodeling_is_contamination",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "apparent_remodeling_reflects_cancer_contamination"
            ]
          },
          {
            "name": "cf_perturbation_design_misread",
            "role": "hard_fail",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "multiple_tumor_suppressors_pooled",
              "no_additional_gene_difference_kras_only",
              "rb1_cdkn2a_double",
              "pcna_essential_gene_control",
              "difference_is_oncogene_not_suppressor"
            ]
          },
          {
            "name": "cf_age_independent_tme",
            "role": "hard_fail",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_independent"
            ]
          },
          {
            "name": "cf_compositional_shift",
            "role": "hard_fail",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_frequency_shift",
              "compositional_immune_infiltration",
              "composition_dominant_state_secondary",
              "both_state_and_composition"
            ]
          },
          {
            "name": "cf_epithelial_or_cancer_substitution",
            "role": "hard_fail",
            "field": "$.most_responsive_noncancer_cell_types[*].cell_type",
            "kind": "avoid",
            "expected": [
              "AT2_cell",
              "AT1_cell",
              "cancer_cell",
              "ciliated",
              "secretory",
              "club_cell",
              "basal_cell",
              "goblet_cell"
            ]
          },
          {
            "name": "cf_tme_breadth_broader_in_young",
            "role": "hard_fail",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_young"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gemini-3-1-pro-preview-pi",
      "modelName": "gemini-3.1-pro-preview",
      "harness": "pi",
      "task": "gse297023_q5_tme_age_dependent_remodeling",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 2224,
      "cost": 0.6757464000000001,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "perturbation_is_pten",
            "role": "gate",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "pten"
            ]
          },
          {
            "name": "tme_non_cell_autonomous",
            "role": "gate",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "substantial_non_cell_autonomous_remodeling"
            ]
          },
          {
            "name": "tme_state_not_composition",
            "role": "gate",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "tme_age_dependent",
            "role": "gate",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_dependent"
            ]
          },
          {
            "name": "tme_response_broader_in_aged",
            "role": "gate",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_aged"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "most_responsive_noncancer_cell_types",
            "kind": "list",
            "matchKey": "cell_type",
            "k": 2,
            "expected": [
              {
                "key": "alveolar_macrophage",
                "fields": {}
              },
              {
                "key": "endothelial",
                "fields": {}
              }
            ]
          },
          {
            "name": "cf_cell_intrinsic_only",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "negligible_cell_intrinsic_only_effect",
              "remodeling_unresolved",
              "modest_remodeling_few_cell_types"
            ]
          },
          {
            "name": "cf_remodeling_is_contamination",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "apparent_remodeling_reflects_cancer_contamination"
            ]
          },
          {
            "name": "cf_perturbation_design_misread",
            "role": "hard_fail",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "multiple_tumor_suppressors_pooled",
              "no_additional_gene_difference_kras_only",
              "rb1_cdkn2a_double",
              "pcna_essential_gene_control",
              "difference_is_oncogene_not_suppressor"
            ]
          },
          {
            "name": "cf_age_independent_tme",
            "role": "hard_fail",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_independent"
            ]
          },
          {
            "name": "cf_compositional_shift",
            "role": "hard_fail",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_frequency_shift",
              "compositional_immune_infiltration",
              "composition_dominant_state_secondary",
              "both_state_and_composition"
            ]
          },
          {
            "name": "cf_epithelial_or_cancer_substitution",
            "role": "hard_fail",
            "field": "$.most_responsive_noncancer_cell_types[*].cell_type",
            "kind": "avoid",
            "expected": [
              "AT2_cell",
              "AT1_cell",
              "cancer_cell",
              "ciliated",
              "secretory",
              "club_cell",
              "basal_cell",
              "goblet_cell"
            ]
          },
          {
            "name": "cf_tme_breadth_broader_in_young",
            "role": "hard_fail",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_young"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gemini-3-5-flash-pi",
      "modelName": "gemini-3.5-flash",
      "harness": "pi",
      "task": "gse297023_q5_tme_age_dependent_remodeling",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 1,
      "passed": true,
      "score": null,
      "durationS": 1124,
      "cost": 1.2336300000000004,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "perturbation_is_pten",
            "role": "gate",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "pten"
            ]
          },
          {
            "name": "tme_non_cell_autonomous",
            "role": "gate",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "substantial_non_cell_autonomous_remodeling"
            ]
          },
          {
            "name": "tme_state_not_composition",
            "role": "gate",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "tme_age_dependent",
            "role": "gate",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_dependent"
            ]
          },
          {
            "name": "tme_response_broader_in_aged",
            "role": "gate",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_aged"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "most_responsive_noncancer_cell_types",
            "kind": "list",
            "matchKey": "cell_type",
            "k": 2,
            "expected": [
              {
                "key": "alveolar_macrophage",
                "fields": {}
              },
              {
                "key": "endothelial",
                "fields": {}
              }
            ]
          },
          {
            "name": "cf_cell_intrinsic_only",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "negligible_cell_intrinsic_only_effect",
              "remodeling_unresolved",
              "modest_remodeling_few_cell_types"
            ]
          },
          {
            "name": "cf_remodeling_is_contamination",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "apparent_remodeling_reflects_cancer_contamination"
            ]
          },
          {
            "name": "cf_perturbation_design_misread",
            "role": "hard_fail",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "multiple_tumor_suppressors_pooled",
              "no_additional_gene_difference_kras_only",
              "rb1_cdkn2a_double",
              "pcna_essential_gene_control",
              "difference_is_oncogene_not_suppressor"
            ]
          },
          {
            "name": "cf_age_independent_tme",
            "role": "hard_fail",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_independent"
            ]
          },
          {
            "name": "cf_compositional_shift",
            "role": "hard_fail",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_frequency_shift",
              "compositional_immune_infiltration",
              "composition_dominant_state_secondary",
              "both_state_and_composition"
            ]
          },
          {
            "name": "cf_epithelial_or_cancer_substitution",
            "role": "hard_fail",
            "field": "$.most_responsive_noncancer_cell_types[*].cell_type",
            "kind": "avoid",
            "expected": [
              "AT2_cell",
              "AT1_cell",
              "cancer_cell",
              "ciliated",
              "secretory",
              "club_cell",
              "basal_cell",
              "goblet_cell"
            ]
          },
          {
            "name": "cf_tme_breadth_broader_in_young",
            "role": "hard_fail",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_young"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gemini-3-5-flash-pi",
      "modelName": "gemini-3.5-flash",
      "harness": "pi",
      "task": "gse297023_q5_tme_age_dependent_remodeling",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 2,
      "passed": true,
      "score": null,
      "durationS": 1099,
      "cost": 0.7895512500000001,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "perturbation_is_pten",
            "role": "gate",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "pten"
            ]
          },
          {
            "name": "tme_non_cell_autonomous",
            "role": "gate",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "substantial_non_cell_autonomous_remodeling"
            ]
          },
          {
            "name": "tme_state_not_composition",
            "role": "gate",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "tme_age_dependent",
            "role": "gate",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_dependent"
            ]
          },
          {
            "name": "tme_response_broader_in_aged",
            "role": "gate",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_aged"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "most_responsive_noncancer_cell_types",
            "kind": "list",
            "matchKey": "cell_type",
            "k": 2,
            "expected": [
              {
                "key": "alveolar_macrophage",
                "fields": {}
              },
              {
                "key": "endothelial",
                "fields": {}
              }
            ]
          },
          {
            "name": "cf_cell_intrinsic_only",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "negligible_cell_intrinsic_only_effect",
              "remodeling_unresolved",
              "modest_remodeling_few_cell_types"
            ]
          },
          {
            "name": "cf_remodeling_is_contamination",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "apparent_remodeling_reflects_cancer_contamination"
            ]
          },
          {
            "name": "cf_perturbation_design_misread",
            "role": "hard_fail",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "multiple_tumor_suppressors_pooled",
              "no_additional_gene_difference_kras_only",
              "rb1_cdkn2a_double",
              "pcna_essential_gene_control",
              "difference_is_oncogene_not_suppressor"
            ]
          },
          {
            "name": "cf_age_independent_tme",
            "role": "hard_fail",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_independent"
            ]
          },
          {
            "name": "cf_compositional_shift",
            "role": "hard_fail",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_frequency_shift",
              "compositional_immune_infiltration",
              "composition_dominant_state_secondary",
              "both_state_and_composition"
            ]
          },
          {
            "name": "cf_epithelial_or_cancer_substitution",
            "role": "hard_fail",
            "field": "$.most_responsive_noncancer_cell_types[*].cell_type",
            "kind": "avoid",
            "expected": [
              "AT2_cell",
              "AT1_cell",
              "cancer_cell",
              "ciliated",
              "secretory",
              "club_cell",
              "basal_cell",
              "goblet_cell"
            ]
          },
          {
            "name": "cf_tme_breadth_broader_in_young",
            "role": "hard_fail",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_young"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gemini-3-5-flash-pi",
      "modelName": "gemini-3.5-flash",
      "harness": "pi",
      "task": "gse297023_q5_tme_age_dependent_remodeling",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 3,
      "passed": true,
      "score": null,
      "durationS": 1858,
      "cost": 1.6175257499999995,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "perturbation_is_pten",
            "role": "gate",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "pten"
            ]
          },
          {
            "name": "tme_non_cell_autonomous",
            "role": "gate",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "substantial_non_cell_autonomous_remodeling"
            ]
          },
          {
            "name": "tme_state_not_composition",
            "role": "gate",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "tme_age_dependent",
            "role": "gate",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_dependent"
            ]
          },
          {
            "name": "tme_response_broader_in_aged",
            "role": "gate",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_aged"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "most_responsive_noncancer_cell_types",
            "kind": "list",
            "matchKey": "cell_type",
            "k": 2,
            "expected": [
              {
                "key": "alveolar_macrophage",
                "fields": {}
              },
              {
                "key": "endothelial",
                "fields": {}
              }
            ]
          },
          {
            "name": "cf_cell_intrinsic_only",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "negligible_cell_intrinsic_only_effect",
              "remodeling_unresolved",
              "modest_remodeling_few_cell_types"
            ]
          },
          {
            "name": "cf_remodeling_is_contamination",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "apparent_remodeling_reflects_cancer_contamination"
            ]
          },
          {
            "name": "cf_perturbation_design_misread",
            "role": "hard_fail",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "multiple_tumor_suppressors_pooled",
              "no_additional_gene_difference_kras_only",
              "rb1_cdkn2a_double",
              "pcna_essential_gene_control",
              "difference_is_oncogene_not_suppressor"
            ]
          },
          {
            "name": "cf_age_independent_tme",
            "role": "hard_fail",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_independent"
            ]
          },
          {
            "name": "cf_compositional_shift",
            "role": "hard_fail",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_frequency_shift",
              "compositional_immune_infiltration",
              "composition_dominant_state_secondary",
              "both_state_and_composition"
            ]
          },
          {
            "name": "cf_epithelial_or_cancer_substitution",
            "role": "hard_fail",
            "field": "$.most_responsive_noncancer_cell_types[*].cell_type",
            "kind": "avoid",
            "expected": [
              "AT2_cell",
              "AT1_cell",
              "cancer_cell",
              "ciliated",
              "secretory",
              "club_cell",
              "basal_cell",
              "goblet_cell"
            ]
          },
          {
            "name": "cf_tme_breadth_broader_in_young",
            "role": "hard_fail",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_young"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-glm-5p2-pi",
      "modelName": "glm-5p2",
      "harness": "pi",
      "task": "gse297023_q5_tme_age_dependent_remodeling",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 8024,
      "cost": 6.30910878,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "perturbation_is_pten",
            "role": "gate",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "pten"
            ]
          },
          {
            "name": "tme_non_cell_autonomous",
            "role": "gate",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "substantial_non_cell_autonomous_remodeling"
            ]
          },
          {
            "name": "tme_state_not_composition",
            "role": "gate",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "tme_age_dependent",
            "role": "gate",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_dependent"
            ]
          },
          {
            "name": "tme_response_broader_in_aged",
            "role": "gate",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_aged"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "most_responsive_noncancer_cell_types",
            "kind": "list",
            "matchKey": "cell_type",
            "k": 2,
            "expected": [
              {
                "key": "alveolar_macrophage",
                "fields": {}
              },
              {
                "key": "endothelial",
                "fields": {}
              }
            ]
          },
          {
            "name": "cf_cell_intrinsic_only",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "negligible_cell_intrinsic_only_effect",
              "remodeling_unresolved",
              "modest_remodeling_few_cell_types"
            ]
          },
          {
            "name": "cf_remodeling_is_contamination",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "apparent_remodeling_reflects_cancer_contamination"
            ]
          },
          {
            "name": "cf_perturbation_design_misread",
            "role": "hard_fail",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "multiple_tumor_suppressors_pooled",
              "no_additional_gene_difference_kras_only",
              "rb1_cdkn2a_double",
              "pcna_essential_gene_control",
              "difference_is_oncogene_not_suppressor"
            ]
          },
          {
            "name": "cf_age_independent_tme",
            "role": "hard_fail",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_independent"
            ]
          },
          {
            "name": "cf_compositional_shift",
            "role": "hard_fail",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_frequency_shift",
              "compositional_immune_infiltration",
              "composition_dominant_state_secondary",
              "both_state_and_composition"
            ]
          },
          {
            "name": "cf_epithelial_or_cancer_substitution",
            "role": "hard_fail",
            "field": "$.most_responsive_noncancer_cell_types[*].cell_type",
            "kind": "avoid",
            "expected": [
              "AT2_cell",
              "AT1_cell",
              "cancer_cell",
              "ciliated",
              "secretory",
              "club_cell",
              "basal_cell",
              "goblet_cell"
            ]
          },
          {
            "name": "cf_tme_breadth_broader_in_young",
            "role": "hard_fail",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_young"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-glm-5p2-pi",
      "modelName": "glm-5p2",
      "harness": "pi",
      "task": "gse297023_q5_tme_age_dependent_remodeling",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 6330,
      "cost": 2.9559676,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "perturbation_is_pten",
            "role": "gate",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "pten"
            ]
          },
          {
            "name": "tme_non_cell_autonomous",
            "role": "gate",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "substantial_non_cell_autonomous_remodeling"
            ]
          },
          {
            "name": "tme_state_not_composition",
            "role": "gate",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "tme_age_dependent",
            "role": "gate",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_dependent"
            ]
          },
          {
            "name": "tme_response_broader_in_aged",
            "role": "gate",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_aged"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "most_responsive_noncancer_cell_types",
            "kind": "list",
            "matchKey": "cell_type",
            "k": 2,
            "expected": [
              {
                "key": "alveolar_macrophage",
                "fields": {}
              },
              {
                "key": "endothelial",
                "fields": {}
              }
            ]
          },
          {
            "name": "cf_cell_intrinsic_only",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "negligible_cell_intrinsic_only_effect",
              "remodeling_unresolved",
              "modest_remodeling_few_cell_types"
            ]
          },
          {
            "name": "cf_remodeling_is_contamination",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "apparent_remodeling_reflects_cancer_contamination"
            ]
          },
          {
            "name": "cf_perturbation_design_misread",
            "role": "hard_fail",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "multiple_tumor_suppressors_pooled",
              "no_additional_gene_difference_kras_only",
              "rb1_cdkn2a_double",
              "pcna_essential_gene_control",
              "difference_is_oncogene_not_suppressor"
            ]
          },
          {
            "name": "cf_age_independent_tme",
            "role": "hard_fail",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_independent"
            ]
          },
          {
            "name": "cf_compositional_shift",
            "role": "hard_fail",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_frequency_shift",
              "compositional_immune_infiltration",
              "composition_dominant_state_secondary",
              "both_state_and_composition"
            ]
          },
          {
            "name": "cf_epithelial_or_cancer_substitution",
            "role": "hard_fail",
            "field": "$.most_responsive_noncancer_cell_types[*].cell_type",
            "kind": "avoid",
            "expected": [
              "AT2_cell",
              "AT1_cell",
              "cancer_cell",
              "ciliated",
              "secretory",
              "club_cell",
              "basal_cell",
              "goblet_cell"
            ]
          },
          {
            "name": "cf_tme_breadth_broader_in_young",
            "role": "hard_fail",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_young"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-glm-5p2-pi",
      "modelName": "glm-5p2",
      "harness": "pi",
      "task": "gse297023_q5_tme_age_dependent_remodeling",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 10144,
      "cost": 6.804597679999998,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "perturbation_is_pten",
            "role": "gate",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "pten"
            ]
          },
          {
            "name": "tme_non_cell_autonomous",
            "role": "gate",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "substantial_non_cell_autonomous_remodeling"
            ]
          },
          {
            "name": "tme_state_not_composition",
            "role": "gate",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "tme_age_dependent",
            "role": "gate",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_dependent"
            ]
          },
          {
            "name": "tme_response_broader_in_aged",
            "role": "gate",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_aged"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "most_responsive_noncancer_cell_types",
            "kind": "list",
            "matchKey": "cell_type",
            "k": 2,
            "expected": [
              {
                "key": "alveolar_macrophage",
                "fields": {}
              },
              {
                "key": "endothelial",
                "fields": {}
              }
            ]
          },
          {
            "name": "cf_cell_intrinsic_only",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "negligible_cell_intrinsic_only_effect",
              "remodeling_unresolved",
              "modest_remodeling_few_cell_types"
            ]
          },
          {
            "name": "cf_remodeling_is_contamination",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "apparent_remodeling_reflects_cancer_contamination"
            ]
          },
          {
            "name": "cf_perturbation_design_misread",
            "role": "hard_fail",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "multiple_tumor_suppressors_pooled",
              "no_additional_gene_difference_kras_only",
              "rb1_cdkn2a_double",
              "pcna_essential_gene_control",
              "difference_is_oncogene_not_suppressor"
            ]
          },
          {
            "name": "cf_age_independent_tme",
            "role": "hard_fail",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_independent"
            ]
          },
          {
            "name": "cf_compositional_shift",
            "role": "hard_fail",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_frequency_shift",
              "compositional_immune_infiltration",
              "composition_dominant_state_secondary",
              "both_state_and_composition"
            ]
          },
          {
            "name": "cf_epithelial_or_cancer_substitution",
            "role": "hard_fail",
            "field": "$.most_responsive_noncancer_cell_types[*].cell_type",
            "kind": "avoid",
            "expected": [
              "AT2_cell",
              "AT1_cell",
              "cancer_cell",
              "ciliated",
              "secretory",
              "club_cell",
              "basal_cell",
              "goblet_cell"
            ]
          },
          {
            "name": "cf_tme_breadth_broader_in_young",
            "role": "hard_fail",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_young"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-4-codex",
      "modelName": "gpt-5.4",
      "harness": "openai-codex",
      "task": "gse297023_q5_tme_age_dependent_remodeling",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 1851,
      "cost": 2.7146689999999998,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "perturbation_is_pten",
            "role": "gate",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "pten"
            ]
          },
          {
            "name": "tme_non_cell_autonomous",
            "role": "gate",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "substantial_non_cell_autonomous_remodeling"
            ]
          },
          {
            "name": "tme_state_not_composition",
            "role": "gate",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "tme_age_dependent",
            "role": "gate",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_dependent"
            ]
          },
          {
            "name": "tme_response_broader_in_aged",
            "role": "gate",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_aged"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "most_responsive_noncancer_cell_types",
            "kind": "list",
            "matchKey": "cell_type",
            "k": 2,
            "expected": [
              {
                "key": "alveolar_macrophage",
                "fields": {}
              },
              {
                "key": "endothelial",
                "fields": {}
              }
            ]
          },
          {
            "name": "cf_cell_intrinsic_only",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "negligible_cell_intrinsic_only_effect",
              "remodeling_unresolved",
              "modest_remodeling_few_cell_types"
            ]
          },
          {
            "name": "cf_remodeling_is_contamination",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "apparent_remodeling_reflects_cancer_contamination"
            ]
          },
          {
            "name": "cf_perturbation_design_misread",
            "role": "hard_fail",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "multiple_tumor_suppressors_pooled",
              "no_additional_gene_difference_kras_only",
              "rb1_cdkn2a_double",
              "pcna_essential_gene_control",
              "difference_is_oncogene_not_suppressor"
            ]
          },
          {
            "name": "cf_age_independent_tme",
            "role": "hard_fail",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_independent"
            ]
          },
          {
            "name": "cf_compositional_shift",
            "role": "hard_fail",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_frequency_shift",
              "compositional_immune_infiltration",
              "composition_dominant_state_secondary",
              "both_state_and_composition"
            ]
          },
          {
            "name": "cf_epithelial_or_cancer_substitution",
            "role": "hard_fail",
            "field": "$.most_responsive_noncancer_cell_types[*].cell_type",
            "kind": "avoid",
            "expected": [
              "AT2_cell",
              "AT1_cell",
              "cancer_cell",
              "ciliated",
              "secretory",
              "club_cell",
              "basal_cell",
              "goblet_cell"
            ]
          },
          {
            "name": "cf_tme_breadth_broader_in_young",
            "role": "hard_fail",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_young"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-4-codex",
      "modelName": "gpt-5.4",
      "harness": "openai-codex",
      "task": "gse297023_q5_tme_age_dependent_remodeling",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 1474,
      "cost": 1.0236225,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "perturbation_is_pten",
            "role": "gate",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "pten"
            ]
          },
          {
            "name": "tme_non_cell_autonomous",
            "role": "gate",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "substantial_non_cell_autonomous_remodeling"
            ]
          },
          {
            "name": "tme_state_not_composition",
            "role": "gate",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "tme_age_dependent",
            "role": "gate",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_dependent"
            ]
          },
          {
            "name": "tme_response_broader_in_aged",
            "role": "gate",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_aged"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "most_responsive_noncancer_cell_types",
            "kind": "list",
            "matchKey": "cell_type",
            "k": 2,
            "expected": [
              {
                "key": "alveolar_macrophage",
                "fields": {}
              },
              {
                "key": "endothelial",
                "fields": {}
              }
            ]
          },
          {
            "name": "cf_cell_intrinsic_only",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "negligible_cell_intrinsic_only_effect",
              "remodeling_unresolved",
              "modest_remodeling_few_cell_types"
            ]
          },
          {
            "name": "cf_remodeling_is_contamination",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "apparent_remodeling_reflects_cancer_contamination"
            ]
          },
          {
            "name": "cf_perturbation_design_misread",
            "role": "hard_fail",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "multiple_tumor_suppressors_pooled",
              "no_additional_gene_difference_kras_only",
              "rb1_cdkn2a_double",
              "pcna_essential_gene_control",
              "difference_is_oncogene_not_suppressor"
            ]
          },
          {
            "name": "cf_age_independent_tme",
            "role": "hard_fail",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_independent"
            ]
          },
          {
            "name": "cf_compositional_shift",
            "role": "hard_fail",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_frequency_shift",
              "compositional_immune_infiltration",
              "composition_dominant_state_secondary",
              "both_state_and_composition"
            ]
          },
          {
            "name": "cf_epithelial_or_cancer_substitution",
            "role": "hard_fail",
            "field": "$.most_responsive_noncancer_cell_types[*].cell_type",
            "kind": "avoid",
            "expected": [
              "AT2_cell",
              "AT1_cell",
              "cancer_cell",
              "ciliated",
              "secretory",
              "club_cell",
              "basal_cell",
              "goblet_cell"
            ]
          },
          {
            "name": "cf_tme_breadth_broader_in_young",
            "role": "hard_fail",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_young"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-4-codex",
      "modelName": "gpt-5.4",
      "harness": "openai-codex",
      "task": "gse297023_q5_tme_age_dependent_remodeling",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 679,
      "cost": 2.2724655,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "perturbation_is_pten",
            "role": "gate",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "pten"
            ]
          },
          {
            "name": "tme_non_cell_autonomous",
            "role": "gate",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "substantial_non_cell_autonomous_remodeling"
            ]
          },
          {
            "name": "tme_state_not_composition",
            "role": "gate",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "tme_age_dependent",
            "role": "gate",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_dependent"
            ]
          },
          {
            "name": "tme_response_broader_in_aged",
            "role": "gate",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_aged"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "most_responsive_noncancer_cell_types",
            "kind": "list",
            "matchKey": "cell_type",
            "k": 2,
            "expected": [
              {
                "key": "alveolar_macrophage",
                "fields": {}
              },
              {
                "key": "endothelial",
                "fields": {}
              }
            ]
          },
          {
            "name": "cf_cell_intrinsic_only",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "negligible_cell_intrinsic_only_effect",
              "remodeling_unresolved",
              "modest_remodeling_few_cell_types"
            ]
          },
          {
            "name": "cf_remodeling_is_contamination",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "apparent_remodeling_reflects_cancer_contamination"
            ]
          },
          {
            "name": "cf_perturbation_design_misread",
            "role": "hard_fail",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "multiple_tumor_suppressors_pooled",
              "no_additional_gene_difference_kras_only",
              "rb1_cdkn2a_double",
              "pcna_essential_gene_control",
              "difference_is_oncogene_not_suppressor"
            ]
          },
          {
            "name": "cf_age_independent_tme",
            "role": "hard_fail",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_independent"
            ]
          },
          {
            "name": "cf_compositional_shift",
            "role": "hard_fail",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_frequency_shift",
              "compositional_immune_infiltration",
              "composition_dominant_state_secondary",
              "both_state_and_composition"
            ]
          },
          {
            "name": "cf_epithelial_or_cancer_substitution",
            "role": "hard_fail",
            "field": "$.most_responsive_noncancer_cell_types[*].cell_type",
            "kind": "avoid",
            "expected": [
              "AT2_cell",
              "AT1_cell",
              "cancer_cell",
              "ciliated",
              "secretory",
              "club_cell",
              "basal_cell",
              "goblet_cell"
            ]
          },
          {
            "name": "cf_tme_breadth_broader_in_young",
            "role": "hard_fail",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_young"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-4-pi",
      "modelName": "gpt-5.4",
      "harness": "pi",
      "task": "gse297023_q5_tme_age_dependent_remodeling",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 2025,
      "cost": 0.871645,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "perturbation_is_pten",
            "role": "gate",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "pten"
            ]
          },
          {
            "name": "tme_non_cell_autonomous",
            "role": "gate",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "substantial_non_cell_autonomous_remodeling"
            ]
          },
          {
            "name": "tme_state_not_composition",
            "role": "gate",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "tme_age_dependent",
            "role": "gate",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_dependent"
            ]
          },
          {
            "name": "tme_response_broader_in_aged",
            "role": "gate",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_aged"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "most_responsive_noncancer_cell_types",
            "kind": "list",
            "matchKey": "cell_type",
            "k": 2,
            "expected": [
              {
                "key": "alveolar_macrophage",
                "fields": {}
              },
              {
                "key": "endothelial",
                "fields": {}
              }
            ]
          },
          {
            "name": "cf_cell_intrinsic_only",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "negligible_cell_intrinsic_only_effect",
              "remodeling_unresolved",
              "modest_remodeling_few_cell_types"
            ]
          },
          {
            "name": "cf_remodeling_is_contamination",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "apparent_remodeling_reflects_cancer_contamination"
            ]
          },
          {
            "name": "cf_perturbation_design_misread",
            "role": "hard_fail",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "multiple_tumor_suppressors_pooled",
              "no_additional_gene_difference_kras_only",
              "rb1_cdkn2a_double",
              "pcna_essential_gene_control",
              "difference_is_oncogene_not_suppressor"
            ]
          },
          {
            "name": "cf_age_independent_tme",
            "role": "hard_fail",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_independent"
            ]
          },
          {
            "name": "cf_compositional_shift",
            "role": "hard_fail",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_frequency_shift",
              "compositional_immune_infiltration",
              "composition_dominant_state_secondary",
              "both_state_and_composition"
            ]
          },
          {
            "name": "cf_epithelial_or_cancer_substitution",
            "role": "hard_fail",
            "field": "$.most_responsive_noncancer_cell_types[*].cell_type",
            "kind": "avoid",
            "expected": [
              "AT2_cell",
              "AT1_cell",
              "cancer_cell",
              "ciliated",
              "secretory",
              "club_cell",
              "basal_cell",
              "goblet_cell"
            ]
          },
          {
            "name": "cf_tme_breadth_broader_in_young",
            "role": "hard_fail",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_young"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-4-pi",
      "modelName": "gpt-5.4",
      "harness": "pi",
      "task": "gse297023_q5_tme_age_dependent_remodeling",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 1774,
      "cost": 1.46762,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "perturbation_is_pten",
            "role": "gate",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "pten"
            ]
          },
          {
            "name": "tme_non_cell_autonomous",
            "role": "gate",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "substantial_non_cell_autonomous_remodeling"
            ]
          },
          {
            "name": "tme_state_not_composition",
            "role": "gate",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "tme_age_dependent",
            "role": "gate",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_dependent"
            ]
          },
          {
            "name": "tme_response_broader_in_aged",
            "role": "gate",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_aged"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "most_responsive_noncancer_cell_types",
            "kind": "list",
            "matchKey": "cell_type",
            "k": 2,
            "expected": [
              {
                "key": "alveolar_macrophage",
                "fields": {}
              },
              {
                "key": "endothelial",
                "fields": {}
              }
            ]
          },
          {
            "name": "cf_cell_intrinsic_only",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "negligible_cell_intrinsic_only_effect",
              "remodeling_unresolved",
              "modest_remodeling_few_cell_types"
            ]
          },
          {
            "name": "cf_remodeling_is_contamination",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "apparent_remodeling_reflects_cancer_contamination"
            ]
          },
          {
            "name": "cf_perturbation_design_misread",
            "role": "hard_fail",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "multiple_tumor_suppressors_pooled",
              "no_additional_gene_difference_kras_only",
              "rb1_cdkn2a_double",
              "pcna_essential_gene_control",
              "difference_is_oncogene_not_suppressor"
            ]
          },
          {
            "name": "cf_age_independent_tme",
            "role": "hard_fail",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_independent"
            ]
          },
          {
            "name": "cf_compositional_shift",
            "role": "hard_fail",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_frequency_shift",
              "compositional_immune_infiltration",
              "composition_dominant_state_secondary",
              "both_state_and_composition"
            ]
          },
          {
            "name": "cf_epithelial_or_cancer_substitution",
            "role": "hard_fail",
            "field": "$.most_responsive_noncancer_cell_types[*].cell_type",
            "kind": "avoid",
            "expected": [
              "AT2_cell",
              "AT1_cell",
              "cancer_cell",
              "ciliated",
              "secretory",
              "club_cell",
              "basal_cell",
              "goblet_cell"
            ]
          },
          {
            "name": "cf_tme_breadth_broader_in_young",
            "role": "hard_fail",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_young"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-4-pi",
      "modelName": "gpt-5.4",
      "harness": "pi",
      "task": "gse297023_q5_tme_age_dependent_remodeling",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 2761,
      "cost": 1.1446355000000001,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "perturbation_is_pten",
            "role": "gate",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "pten"
            ]
          },
          {
            "name": "tme_non_cell_autonomous",
            "role": "gate",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "substantial_non_cell_autonomous_remodeling"
            ]
          },
          {
            "name": "tme_state_not_composition",
            "role": "gate",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "tme_age_dependent",
            "role": "gate",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_dependent"
            ]
          },
          {
            "name": "tme_response_broader_in_aged",
            "role": "gate",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_aged"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "most_responsive_noncancer_cell_types",
            "kind": "list",
            "matchKey": "cell_type",
            "k": 2,
            "expected": [
              {
                "key": "alveolar_macrophage",
                "fields": {}
              },
              {
                "key": "endothelial",
                "fields": {}
              }
            ]
          },
          {
            "name": "cf_cell_intrinsic_only",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "negligible_cell_intrinsic_only_effect",
              "remodeling_unresolved",
              "modest_remodeling_few_cell_types"
            ]
          },
          {
            "name": "cf_remodeling_is_contamination",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "apparent_remodeling_reflects_cancer_contamination"
            ]
          },
          {
            "name": "cf_perturbation_design_misread",
            "role": "hard_fail",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "multiple_tumor_suppressors_pooled",
              "no_additional_gene_difference_kras_only",
              "rb1_cdkn2a_double",
              "pcna_essential_gene_control",
              "difference_is_oncogene_not_suppressor"
            ]
          },
          {
            "name": "cf_age_independent_tme",
            "role": "hard_fail",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_independent"
            ]
          },
          {
            "name": "cf_compositional_shift",
            "role": "hard_fail",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_frequency_shift",
              "compositional_immune_infiltration",
              "composition_dominant_state_secondary",
              "both_state_and_composition"
            ]
          },
          {
            "name": "cf_epithelial_or_cancer_substitution",
            "role": "hard_fail",
            "field": "$.most_responsive_noncancer_cell_types[*].cell_type",
            "kind": "avoid",
            "expected": [
              "AT2_cell",
              "AT1_cell",
              "cancer_cell",
              "ciliated",
              "secretory",
              "club_cell",
              "basal_cell",
              "goblet_cell"
            ]
          },
          {
            "name": "cf_tme_breadth_broader_in_young",
            "role": "hard_fail",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_young"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-5-codex",
      "modelName": "gpt-5.5",
      "harness": "openai-codex",
      "task": "gse297023_q5_tme_age_dependent_remodeling",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 3720,
      "cost": 3.3078462500000003,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "perturbation_is_pten",
            "role": "gate",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "pten"
            ]
          },
          {
            "name": "tme_non_cell_autonomous",
            "role": "gate",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "substantial_non_cell_autonomous_remodeling"
            ]
          },
          {
            "name": "tme_state_not_composition",
            "role": "gate",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "tme_age_dependent",
            "role": "gate",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_dependent"
            ]
          },
          {
            "name": "tme_response_broader_in_aged",
            "role": "gate",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_aged"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "most_responsive_noncancer_cell_types",
            "kind": "list",
            "matchKey": "cell_type",
            "k": 2,
            "expected": [
              {
                "key": "alveolar_macrophage",
                "fields": {}
              },
              {
                "key": "endothelial",
                "fields": {}
              }
            ]
          },
          {
            "name": "cf_cell_intrinsic_only",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "negligible_cell_intrinsic_only_effect",
              "remodeling_unresolved",
              "modest_remodeling_few_cell_types"
            ]
          },
          {
            "name": "cf_remodeling_is_contamination",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "apparent_remodeling_reflects_cancer_contamination"
            ]
          },
          {
            "name": "cf_perturbation_design_misread",
            "role": "hard_fail",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "multiple_tumor_suppressors_pooled",
              "no_additional_gene_difference_kras_only",
              "rb1_cdkn2a_double",
              "pcna_essential_gene_control",
              "difference_is_oncogene_not_suppressor"
            ]
          },
          {
            "name": "cf_age_independent_tme",
            "role": "hard_fail",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_independent"
            ]
          },
          {
            "name": "cf_compositional_shift",
            "role": "hard_fail",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_frequency_shift",
              "compositional_immune_infiltration",
              "composition_dominant_state_secondary",
              "both_state_and_composition"
            ]
          },
          {
            "name": "cf_epithelial_or_cancer_substitution",
            "role": "hard_fail",
            "field": "$.most_responsive_noncancer_cell_types[*].cell_type",
            "kind": "avoid",
            "expected": [
              "AT2_cell",
              "AT1_cell",
              "cancer_cell",
              "ciliated",
              "secretory",
              "club_cell",
              "basal_cell",
              "goblet_cell"
            ]
          },
          {
            "name": "cf_tme_breadth_broader_in_young",
            "role": "hard_fail",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_young"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-5-codex",
      "modelName": "gpt-5.5",
      "harness": "openai-codex",
      "task": "gse297023_q5_tme_age_dependent_remodeling",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 730,
      "cost": 2.3259309999999997,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "perturbation_is_pten",
            "role": "gate",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "pten"
            ]
          },
          {
            "name": "tme_non_cell_autonomous",
            "role": "gate",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "substantial_non_cell_autonomous_remodeling"
            ]
          },
          {
            "name": "tme_state_not_composition",
            "role": "gate",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "tme_age_dependent",
            "role": "gate",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_dependent"
            ]
          },
          {
            "name": "tme_response_broader_in_aged",
            "role": "gate",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_aged"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "most_responsive_noncancer_cell_types",
            "kind": "list",
            "matchKey": "cell_type",
            "k": 2,
            "expected": [
              {
                "key": "alveolar_macrophage",
                "fields": {}
              },
              {
                "key": "endothelial",
                "fields": {}
              }
            ]
          },
          {
            "name": "cf_cell_intrinsic_only",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "negligible_cell_intrinsic_only_effect",
              "remodeling_unresolved",
              "modest_remodeling_few_cell_types"
            ]
          },
          {
            "name": "cf_remodeling_is_contamination",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "apparent_remodeling_reflects_cancer_contamination"
            ]
          },
          {
            "name": "cf_perturbation_design_misread",
            "role": "hard_fail",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "multiple_tumor_suppressors_pooled",
              "no_additional_gene_difference_kras_only",
              "rb1_cdkn2a_double",
              "pcna_essential_gene_control",
              "difference_is_oncogene_not_suppressor"
            ]
          },
          {
            "name": "cf_age_independent_tme",
            "role": "hard_fail",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_independent"
            ]
          },
          {
            "name": "cf_compositional_shift",
            "role": "hard_fail",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_frequency_shift",
              "compositional_immune_infiltration",
              "composition_dominant_state_secondary",
              "both_state_and_composition"
            ]
          },
          {
            "name": "cf_epithelial_or_cancer_substitution",
            "role": "hard_fail",
            "field": "$.most_responsive_noncancer_cell_types[*].cell_type",
            "kind": "avoid",
            "expected": [
              "AT2_cell",
              "AT1_cell",
              "cancer_cell",
              "ciliated",
              "secretory",
              "club_cell",
              "basal_cell",
              "goblet_cell"
            ]
          },
          {
            "name": "cf_tme_breadth_broader_in_young",
            "role": "hard_fail",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_young"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-5-codex",
      "modelName": "gpt-5.5",
      "harness": "openai-codex",
      "task": "gse297023_q5_tme_age_dependent_remodeling",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1342,
      "cost": 2.056105,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "perturbation_is_pten",
            "role": "gate",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "pten"
            ]
          },
          {
            "name": "tme_non_cell_autonomous",
            "role": "gate",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "substantial_non_cell_autonomous_remodeling"
            ]
          },
          {
            "name": "tme_state_not_composition",
            "role": "gate",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "tme_age_dependent",
            "role": "gate",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_dependent"
            ]
          },
          {
            "name": "tme_response_broader_in_aged",
            "role": "gate",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_aged"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "most_responsive_noncancer_cell_types",
            "kind": "list",
            "matchKey": "cell_type",
            "k": 2,
            "expected": [
              {
                "key": "alveolar_macrophage",
                "fields": {}
              },
              {
                "key": "endothelial",
                "fields": {}
              }
            ]
          },
          {
            "name": "cf_cell_intrinsic_only",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "negligible_cell_intrinsic_only_effect",
              "remodeling_unresolved",
              "modest_remodeling_few_cell_types"
            ]
          },
          {
            "name": "cf_remodeling_is_contamination",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "apparent_remodeling_reflects_cancer_contamination"
            ]
          },
          {
            "name": "cf_perturbation_design_misread",
            "role": "hard_fail",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "multiple_tumor_suppressors_pooled",
              "no_additional_gene_difference_kras_only",
              "rb1_cdkn2a_double",
              "pcna_essential_gene_control",
              "difference_is_oncogene_not_suppressor"
            ]
          },
          {
            "name": "cf_age_independent_tme",
            "role": "hard_fail",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_independent"
            ]
          },
          {
            "name": "cf_compositional_shift",
            "role": "hard_fail",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_frequency_shift",
              "compositional_immune_infiltration",
              "composition_dominant_state_secondary",
              "both_state_and_composition"
            ]
          },
          {
            "name": "cf_epithelial_or_cancer_substitution",
            "role": "hard_fail",
            "field": "$.most_responsive_noncancer_cell_types[*].cell_type",
            "kind": "avoid",
            "expected": [
              "AT2_cell",
              "AT1_cell",
              "cancer_cell",
              "ciliated",
              "secretory",
              "club_cell",
              "basal_cell",
              "goblet_cell"
            ]
          },
          {
            "name": "cf_tme_breadth_broader_in_young",
            "role": "hard_fail",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_young"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-5-pi",
      "modelName": "gpt-5.5",
      "harness": "pi",
      "task": "gse297023_q5_tme_age_dependent_remodeling",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 5082,
      "cost": 2.410053,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "perturbation_is_pten",
            "role": "gate",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "pten"
            ]
          },
          {
            "name": "tme_non_cell_autonomous",
            "role": "gate",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "substantial_non_cell_autonomous_remodeling"
            ]
          },
          {
            "name": "tme_state_not_composition",
            "role": "gate",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "tme_age_dependent",
            "role": "gate",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_dependent"
            ]
          },
          {
            "name": "tme_response_broader_in_aged",
            "role": "gate",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_aged"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "most_responsive_noncancer_cell_types",
            "kind": "list",
            "matchKey": "cell_type",
            "k": 2,
            "expected": [
              {
                "key": "alveolar_macrophage",
                "fields": {}
              },
              {
                "key": "endothelial",
                "fields": {}
              }
            ]
          },
          {
            "name": "cf_cell_intrinsic_only",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "negligible_cell_intrinsic_only_effect",
              "remodeling_unresolved",
              "modest_remodeling_few_cell_types"
            ]
          },
          {
            "name": "cf_remodeling_is_contamination",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "apparent_remodeling_reflects_cancer_contamination"
            ]
          },
          {
            "name": "cf_perturbation_design_misread",
            "role": "hard_fail",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "multiple_tumor_suppressors_pooled",
              "no_additional_gene_difference_kras_only",
              "rb1_cdkn2a_double",
              "pcna_essential_gene_control",
              "difference_is_oncogene_not_suppressor"
            ]
          },
          {
            "name": "cf_age_independent_tme",
            "role": "hard_fail",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_independent"
            ]
          },
          {
            "name": "cf_compositional_shift",
            "role": "hard_fail",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_frequency_shift",
              "compositional_immune_infiltration",
              "composition_dominant_state_secondary",
              "both_state_and_composition"
            ]
          },
          {
            "name": "cf_epithelial_or_cancer_substitution",
            "role": "hard_fail",
            "field": "$.most_responsive_noncancer_cell_types[*].cell_type",
            "kind": "avoid",
            "expected": [
              "AT2_cell",
              "AT1_cell",
              "cancer_cell",
              "ciliated",
              "secretory",
              "club_cell",
              "basal_cell",
              "goblet_cell"
            ]
          },
          {
            "name": "cf_tme_breadth_broader_in_young",
            "role": "hard_fail",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_young"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-5-pi",
      "modelName": "gpt-5.5",
      "harness": "pi",
      "task": "gse297023_q5_tme_age_dependent_remodeling",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 968,
      "cost": 2.5592509999999997,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "perturbation_is_pten",
            "role": "gate",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "pten"
            ]
          },
          {
            "name": "tme_non_cell_autonomous",
            "role": "gate",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "substantial_non_cell_autonomous_remodeling"
            ]
          },
          {
            "name": "tme_state_not_composition",
            "role": "gate",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "tme_age_dependent",
            "role": "gate",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_dependent"
            ]
          },
          {
            "name": "tme_response_broader_in_aged",
            "role": "gate",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_aged"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "most_responsive_noncancer_cell_types",
            "kind": "list",
            "matchKey": "cell_type",
            "k": 2,
            "expected": [
              {
                "key": "alveolar_macrophage",
                "fields": {}
              },
              {
                "key": "endothelial",
                "fields": {}
              }
            ]
          },
          {
            "name": "cf_cell_intrinsic_only",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "negligible_cell_intrinsic_only_effect",
              "remodeling_unresolved",
              "modest_remodeling_few_cell_types"
            ]
          },
          {
            "name": "cf_remodeling_is_contamination",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "apparent_remodeling_reflects_cancer_contamination"
            ]
          },
          {
            "name": "cf_perturbation_design_misread",
            "role": "hard_fail",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "multiple_tumor_suppressors_pooled",
              "no_additional_gene_difference_kras_only",
              "rb1_cdkn2a_double",
              "pcna_essential_gene_control",
              "difference_is_oncogene_not_suppressor"
            ]
          },
          {
            "name": "cf_age_independent_tme",
            "role": "hard_fail",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_independent"
            ]
          },
          {
            "name": "cf_compositional_shift",
            "role": "hard_fail",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_frequency_shift",
              "compositional_immune_infiltration",
              "composition_dominant_state_secondary",
              "both_state_and_composition"
            ]
          },
          {
            "name": "cf_epithelial_or_cancer_substitution",
            "role": "hard_fail",
            "field": "$.most_responsive_noncancer_cell_types[*].cell_type",
            "kind": "avoid",
            "expected": [
              "AT2_cell",
              "AT1_cell",
              "cancer_cell",
              "ciliated",
              "secretory",
              "club_cell",
              "basal_cell",
              "goblet_cell"
            ]
          },
          {
            "name": "cf_tme_breadth_broader_in_young",
            "role": "hard_fail",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_young"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-5-pi",
      "modelName": "gpt-5.5",
      "harness": "pi",
      "task": "gse297023_q5_tme_age_dependent_remodeling",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 2582,
      "cost": 2.5359560000000005,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "perturbation_is_pten",
            "role": "gate",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "pten"
            ]
          },
          {
            "name": "tme_non_cell_autonomous",
            "role": "gate",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "substantial_non_cell_autonomous_remodeling"
            ]
          },
          {
            "name": "tme_state_not_composition",
            "role": "gate",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "tme_age_dependent",
            "role": "gate",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_dependent"
            ]
          },
          {
            "name": "tme_response_broader_in_aged",
            "role": "gate",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_aged"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "most_responsive_noncancer_cell_types",
            "kind": "list",
            "matchKey": "cell_type",
            "k": 2,
            "expected": [
              {
                "key": "alveolar_macrophage",
                "fields": {}
              },
              {
                "key": "endothelial",
                "fields": {}
              }
            ]
          },
          {
            "name": "cf_cell_intrinsic_only",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "negligible_cell_intrinsic_only_effect",
              "remodeling_unresolved",
              "modest_remodeling_few_cell_types"
            ]
          },
          {
            "name": "cf_remodeling_is_contamination",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "apparent_remodeling_reflects_cancer_contamination"
            ]
          },
          {
            "name": "cf_perturbation_design_misread",
            "role": "hard_fail",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "multiple_tumor_suppressors_pooled",
              "no_additional_gene_difference_kras_only",
              "rb1_cdkn2a_double",
              "pcna_essential_gene_control",
              "difference_is_oncogene_not_suppressor"
            ]
          },
          {
            "name": "cf_age_independent_tme",
            "role": "hard_fail",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_independent"
            ]
          },
          {
            "name": "cf_compositional_shift",
            "role": "hard_fail",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_frequency_shift",
              "compositional_immune_infiltration",
              "composition_dominant_state_secondary",
              "both_state_and_composition"
            ]
          },
          {
            "name": "cf_epithelial_or_cancer_substitution",
            "role": "hard_fail",
            "field": "$.most_responsive_noncancer_cell_types[*].cell_type",
            "kind": "avoid",
            "expected": [
              "AT2_cell",
              "AT1_cell",
              "cancer_cell",
              "ciliated",
              "secretory",
              "club_cell",
              "basal_cell",
              "goblet_cell"
            ]
          },
          {
            "name": "cf_tme_breadth_broader_in_young",
            "role": "hard_fail",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_young"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-20-beta-reasoning-pi",
      "modelName": "grok-4.20-beta-0309-reasoning",
      "harness": "pi",
      "task": "gse297023_q5_tme_age_dependent_remodeling",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 1023,
      "cost": 0.0599875,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "perturbation_is_pten",
            "role": "gate",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "pten"
            ]
          },
          {
            "name": "tme_non_cell_autonomous",
            "role": "gate",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "substantial_non_cell_autonomous_remodeling"
            ]
          },
          {
            "name": "tme_state_not_composition",
            "role": "gate",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "tme_age_dependent",
            "role": "gate",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_dependent"
            ]
          },
          {
            "name": "tme_response_broader_in_aged",
            "role": "gate",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_aged"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "most_responsive_noncancer_cell_types",
            "kind": "list",
            "matchKey": "cell_type",
            "k": 2,
            "expected": [
              {
                "key": "alveolar_macrophage",
                "fields": {}
              },
              {
                "key": "endothelial",
                "fields": {}
              }
            ]
          },
          {
            "name": "cf_cell_intrinsic_only",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "negligible_cell_intrinsic_only_effect",
              "remodeling_unresolved",
              "modest_remodeling_few_cell_types"
            ]
          },
          {
            "name": "cf_remodeling_is_contamination",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "apparent_remodeling_reflects_cancer_contamination"
            ]
          },
          {
            "name": "cf_perturbation_design_misread",
            "role": "hard_fail",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "multiple_tumor_suppressors_pooled",
              "no_additional_gene_difference_kras_only",
              "rb1_cdkn2a_double",
              "pcna_essential_gene_control",
              "difference_is_oncogene_not_suppressor"
            ]
          },
          {
            "name": "cf_age_independent_tme",
            "role": "hard_fail",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_independent"
            ]
          },
          {
            "name": "cf_compositional_shift",
            "role": "hard_fail",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_frequency_shift",
              "compositional_immune_infiltration",
              "composition_dominant_state_secondary",
              "both_state_and_composition"
            ]
          },
          {
            "name": "cf_epithelial_or_cancer_substitution",
            "role": "hard_fail",
            "field": "$.most_responsive_noncancer_cell_types[*].cell_type",
            "kind": "avoid",
            "expected": [
              "AT2_cell",
              "AT1_cell",
              "cancer_cell",
              "ciliated",
              "secretory",
              "club_cell",
              "basal_cell",
              "goblet_cell"
            ]
          },
          {
            "name": "cf_tme_breadth_broader_in_young",
            "role": "hard_fail",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_young"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-20-beta-reasoning-pi",
      "modelName": "grok-4.20-beta-0309-reasoning",
      "harness": "pi",
      "task": "gse297023_q5_tme_age_dependent_remodeling",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 93,
      "cost": 0.052604950000000004,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "perturbation_is_pten",
            "role": "gate",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "pten"
            ]
          },
          {
            "name": "tme_non_cell_autonomous",
            "role": "gate",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "substantial_non_cell_autonomous_remodeling"
            ]
          },
          {
            "name": "tme_state_not_composition",
            "role": "gate",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "tme_age_dependent",
            "role": "gate",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_dependent"
            ]
          },
          {
            "name": "tme_response_broader_in_aged",
            "role": "gate",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_aged"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "most_responsive_noncancer_cell_types",
            "kind": "list",
            "matchKey": "cell_type",
            "k": 2,
            "expected": [
              {
                "key": "alveolar_macrophage",
                "fields": {}
              },
              {
                "key": "endothelial",
                "fields": {}
              }
            ]
          },
          {
            "name": "cf_cell_intrinsic_only",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "negligible_cell_intrinsic_only_effect",
              "remodeling_unresolved",
              "modest_remodeling_few_cell_types"
            ]
          },
          {
            "name": "cf_remodeling_is_contamination",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "apparent_remodeling_reflects_cancer_contamination"
            ]
          },
          {
            "name": "cf_perturbation_design_misread",
            "role": "hard_fail",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "multiple_tumor_suppressors_pooled",
              "no_additional_gene_difference_kras_only",
              "rb1_cdkn2a_double",
              "pcna_essential_gene_control",
              "difference_is_oncogene_not_suppressor"
            ]
          },
          {
            "name": "cf_age_independent_tme",
            "role": "hard_fail",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_independent"
            ]
          },
          {
            "name": "cf_compositional_shift",
            "role": "hard_fail",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_frequency_shift",
              "compositional_immune_infiltration",
              "composition_dominant_state_secondary",
              "both_state_and_composition"
            ]
          },
          {
            "name": "cf_epithelial_or_cancer_substitution",
            "role": "hard_fail",
            "field": "$.most_responsive_noncancer_cell_types[*].cell_type",
            "kind": "avoid",
            "expected": [
              "AT2_cell",
              "AT1_cell",
              "cancer_cell",
              "ciliated",
              "secretory",
              "club_cell",
              "basal_cell",
              "goblet_cell"
            ]
          },
          {
            "name": "cf_tme_breadth_broader_in_young",
            "role": "hard_fail",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_young"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-20-beta-reasoning-pi",
      "modelName": "grok-4.20-beta-0309-reasoning",
      "harness": "pi",
      "task": "gse297023_q5_tme_age_dependent_remodeling",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 75,
      "cost": 0.10924420000000003,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "perturbation_is_pten",
            "role": "gate",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "pten"
            ]
          },
          {
            "name": "tme_non_cell_autonomous",
            "role": "gate",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "substantial_non_cell_autonomous_remodeling"
            ]
          },
          {
            "name": "tme_state_not_composition",
            "role": "gate",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "tme_age_dependent",
            "role": "gate",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_dependent"
            ]
          },
          {
            "name": "tme_response_broader_in_aged",
            "role": "gate",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_aged"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "most_responsive_noncancer_cell_types",
            "kind": "list",
            "matchKey": "cell_type",
            "k": 2,
            "expected": [
              {
                "key": "alveolar_macrophage",
                "fields": {}
              },
              {
                "key": "endothelial",
                "fields": {}
              }
            ]
          },
          {
            "name": "cf_cell_intrinsic_only",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "negligible_cell_intrinsic_only_effect",
              "remodeling_unresolved",
              "modest_remodeling_few_cell_types"
            ]
          },
          {
            "name": "cf_remodeling_is_contamination",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "apparent_remodeling_reflects_cancer_contamination"
            ]
          },
          {
            "name": "cf_perturbation_design_misread",
            "role": "hard_fail",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "multiple_tumor_suppressors_pooled",
              "no_additional_gene_difference_kras_only",
              "rb1_cdkn2a_double",
              "pcna_essential_gene_control",
              "difference_is_oncogene_not_suppressor"
            ]
          },
          {
            "name": "cf_age_independent_tme",
            "role": "hard_fail",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_independent"
            ]
          },
          {
            "name": "cf_compositional_shift",
            "role": "hard_fail",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_frequency_shift",
              "compositional_immune_infiltration",
              "composition_dominant_state_secondary",
              "both_state_and_composition"
            ]
          },
          {
            "name": "cf_epithelial_or_cancer_substitution",
            "role": "hard_fail",
            "field": "$.most_responsive_noncancer_cell_types[*].cell_type",
            "kind": "avoid",
            "expected": [
              "AT2_cell",
              "AT1_cell",
              "cancer_cell",
              "ciliated",
              "secretory",
              "club_cell",
              "basal_cell",
              "goblet_cell"
            ]
          },
          {
            "name": "cf_tme_breadth_broader_in_young",
            "role": "hard_fail",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_young"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-3-pi",
      "modelName": "grok-4.3",
      "harness": "pi",
      "task": "gse297023_q5_tme_age_dependent_remodeling",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 82,
      "cost": 0.00872025,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "perturbation_is_pten",
            "role": "gate",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "pten"
            ]
          },
          {
            "name": "tme_non_cell_autonomous",
            "role": "gate",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "substantial_non_cell_autonomous_remodeling"
            ]
          },
          {
            "name": "tme_state_not_composition",
            "role": "gate",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "tme_age_dependent",
            "role": "gate",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_dependent"
            ]
          },
          {
            "name": "tme_response_broader_in_aged",
            "role": "gate",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_aged"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "most_responsive_noncancer_cell_types",
            "kind": "list",
            "matchKey": "cell_type",
            "k": 2,
            "expected": [
              {
                "key": "alveolar_macrophage",
                "fields": {}
              },
              {
                "key": "endothelial",
                "fields": {}
              }
            ]
          },
          {
            "name": "cf_cell_intrinsic_only",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "negligible_cell_intrinsic_only_effect",
              "remodeling_unresolved",
              "modest_remodeling_few_cell_types"
            ]
          },
          {
            "name": "cf_remodeling_is_contamination",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "apparent_remodeling_reflects_cancer_contamination"
            ]
          },
          {
            "name": "cf_perturbation_design_misread",
            "role": "hard_fail",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "multiple_tumor_suppressors_pooled",
              "no_additional_gene_difference_kras_only",
              "rb1_cdkn2a_double",
              "pcna_essential_gene_control",
              "difference_is_oncogene_not_suppressor"
            ]
          },
          {
            "name": "cf_age_independent_tme",
            "role": "hard_fail",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_independent"
            ]
          },
          {
            "name": "cf_compositional_shift",
            "role": "hard_fail",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_frequency_shift",
              "compositional_immune_infiltration",
              "composition_dominant_state_secondary",
              "both_state_and_composition"
            ]
          },
          {
            "name": "cf_epithelial_or_cancer_substitution",
            "role": "hard_fail",
            "field": "$.most_responsive_noncancer_cell_types[*].cell_type",
            "kind": "avoid",
            "expected": [
              "AT2_cell",
              "AT1_cell",
              "cancer_cell",
              "ciliated",
              "secretory",
              "club_cell",
              "basal_cell",
              "goblet_cell"
            ]
          },
          {
            "name": "cf_tme_breadth_broader_in_young",
            "role": "hard_fail",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_young"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-3-pi",
      "modelName": "grok-4.3",
      "harness": "pi",
      "task": "gse297023_q5_tme_age_dependent_remodeling",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 113,
      "cost": 0.00957315,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "perturbation_is_pten",
            "role": "gate",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "pten"
            ]
          },
          {
            "name": "tme_non_cell_autonomous",
            "role": "gate",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "substantial_non_cell_autonomous_remodeling"
            ]
          },
          {
            "name": "tme_state_not_composition",
            "role": "gate",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "tme_age_dependent",
            "role": "gate",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_dependent"
            ]
          },
          {
            "name": "tme_response_broader_in_aged",
            "role": "gate",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_aged"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "most_responsive_noncancer_cell_types",
            "kind": "list",
            "matchKey": "cell_type",
            "k": 2,
            "expected": [
              {
                "key": "alveolar_macrophage",
                "fields": {}
              },
              {
                "key": "endothelial",
                "fields": {}
              }
            ]
          },
          {
            "name": "cf_cell_intrinsic_only",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "negligible_cell_intrinsic_only_effect",
              "remodeling_unresolved",
              "modest_remodeling_few_cell_types"
            ]
          },
          {
            "name": "cf_remodeling_is_contamination",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "apparent_remodeling_reflects_cancer_contamination"
            ]
          },
          {
            "name": "cf_perturbation_design_misread",
            "role": "hard_fail",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "multiple_tumor_suppressors_pooled",
              "no_additional_gene_difference_kras_only",
              "rb1_cdkn2a_double",
              "pcna_essential_gene_control",
              "difference_is_oncogene_not_suppressor"
            ]
          },
          {
            "name": "cf_age_independent_tme",
            "role": "hard_fail",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_independent"
            ]
          },
          {
            "name": "cf_compositional_shift",
            "role": "hard_fail",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_frequency_shift",
              "compositional_immune_infiltration",
              "composition_dominant_state_secondary",
              "both_state_and_composition"
            ]
          },
          {
            "name": "cf_epithelial_or_cancer_substitution",
            "role": "hard_fail",
            "field": "$.most_responsive_noncancer_cell_types[*].cell_type",
            "kind": "avoid",
            "expected": [
              "AT2_cell",
              "AT1_cell",
              "cancer_cell",
              "ciliated",
              "secretory",
              "club_cell",
              "basal_cell",
              "goblet_cell"
            ]
          },
          {
            "name": "cf_tme_breadth_broader_in_young",
            "role": "hard_fail",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_young"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-3-pi",
      "modelName": "grok-4.3",
      "harness": "pi",
      "task": "gse297023_q5_tme_age_dependent_remodeling",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 682,
      "cost": 0.0197314,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "perturbation_is_pten",
            "role": "gate",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "pten"
            ]
          },
          {
            "name": "tme_non_cell_autonomous",
            "role": "gate",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "substantial_non_cell_autonomous_remodeling"
            ]
          },
          {
            "name": "tme_state_not_composition",
            "role": "gate",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "tme_age_dependent",
            "role": "gate",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_dependent"
            ]
          },
          {
            "name": "tme_response_broader_in_aged",
            "role": "gate",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_aged"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "most_responsive_noncancer_cell_types",
            "kind": "list",
            "matchKey": "cell_type",
            "k": 2,
            "expected": [
              {
                "key": "alveolar_macrophage",
                "fields": {}
              },
              {
                "key": "endothelial",
                "fields": {}
              }
            ]
          },
          {
            "name": "cf_cell_intrinsic_only",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "negligible_cell_intrinsic_only_effect",
              "remodeling_unresolved",
              "modest_remodeling_few_cell_types"
            ]
          },
          {
            "name": "cf_remodeling_is_contamination",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "apparent_remodeling_reflects_cancer_contamination"
            ]
          },
          {
            "name": "cf_perturbation_design_misread",
            "role": "hard_fail",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "multiple_tumor_suppressors_pooled",
              "no_additional_gene_difference_kras_only",
              "rb1_cdkn2a_double",
              "pcna_essential_gene_control",
              "difference_is_oncogene_not_suppressor"
            ]
          },
          {
            "name": "cf_age_independent_tme",
            "role": "hard_fail",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_independent"
            ]
          },
          {
            "name": "cf_compositional_shift",
            "role": "hard_fail",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_frequency_shift",
              "compositional_immune_infiltration",
              "composition_dominant_state_secondary",
              "both_state_and_composition"
            ]
          },
          {
            "name": "cf_epithelial_or_cancer_substitution",
            "role": "hard_fail",
            "field": "$.most_responsive_noncancer_cell_types[*].cell_type",
            "kind": "avoid",
            "expected": [
              "AT2_cell",
              "AT1_cell",
              "cancer_cell",
              "ciliated",
              "secretory",
              "club_cell",
              "basal_cell",
              "goblet_cell"
            ]
          },
          {
            "name": "cf_tme_breadth_broader_in_young",
            "role": "hard_fail",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_young"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-kimi-k2p6-pi",
      "modelName": "kimi-k2p6",
      "harness": "pi",
      "task": "gse297023_q5_tme_age_dependent_remodeling",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 4549,
      "cost": 0.6901240599999999,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "perturbation_is_pten",
            "role": "gate",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "pten"
            ]
          },
          {
            "name": "tme_non_cell_autonomous",
            "role": "gate",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "substantial_non_cell_autonomous_remodeling"
            ]
          },
          {
            "name": "tme_state_not_composition",
            "role": "gate",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "tme_age_dependent",
            "role": "gate",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_dependent"
            ]
          },
          {
            "name": "tme_response_broader_in_aged",
            "role": "gate",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_aged"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "most_responsive_noncancer_cell_types",
            "kind": "list",
            "matchKey": "cell_type",
            "k": 2,
            "expected": [
              {
                "key": "alveolar_macrophage",
                "fields": {}
              },
              {
                "key": "endothelial",
                "fields": {}
              }
            ]
          },
          {
            "name": "cf_cell_intrinsic_only",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "negligible_cell_intrinsic_only_effect",
              "remodeling_unresolved",
              "modest_remodeling_few_cell_types"
            ]
          },
          {
            "name": "cf_remodeling_is_contamination",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "apparent_remodeling_reflects_cancer_contamination"
            ]
          },
          {
            "name": "cf_perturbation_design_misread",
            "role": "hard_fail",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "multiple_tumor_suppressors_pooled",
              "no_additional_gene_difference_kras_only",
              "rb1_cdkn2a_double",
              "pcna_essential_gene_control",
              "difference_is_oncogene_not_suppressor"
            ]
          },
          {
            "name": "cf_age_independent_tme",
            "role": "hard_fail",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_independent"
            ]
          },
          {
            "name": "cf_compositional_shift",
            "role": "hard_fail",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_frequency_shift",
              "compositional_immune_infiltration",
              "composition_dominant_state_secondary",
              "both_state_and_composition"
            ]
          },
          {
            "name": "cf_epithelial_or_cancer_substitution",
            "role": "hard_fail",
            "field": "$.most_responsive_noncancer_cell_types[*].cell_type",
            "kind": "avoid",
            "expected": [
              "AT2_cell",
              "AT1_cell",
              "cancer_cell",
              "ciliated",
              "secretory",
              "club_cell",
              "basal_cell",
              "goblet_cell"
            ]
          },
          {
            "name": "cf_tme_breadth_broader_in_young",
            "role": "hard_fail",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_young"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-kimi-k2p6-pi",
      "modelName": "kimi-k2p6",
      "harness": "pi",
      "task": "gse297023_q5_tme_age_dependent_remodeling",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 5780,
      "cost": 1.1862612499999998,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "perturbation_is_pten",
            "role": "gate",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "pten"
            ]
          },
          {
            "name": "tme_non_cell_autonomous",
            "role": "gate",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "substantial_non_cell_autonomous_remodeling"
            ]
          },
          {
            "name": "tme_state_not_composition",
            "role": "gate",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "tme_age_dependent",
            "role": "gate",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_dependent"
            ]
          },
          {
            "name": "tme_response_broader_in_aged",
            "role": "gate",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_aged"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "most_responsive_noncancer_cell_types",
            "kind": "list",
            "matchKey": "cell_type",
            "k": 2,
            "expected": [
              {
                "key": "alveolar_macrophage",
                "fields": {}
              },
              {
                "key": "endothelial",
                "fields": {}
              }
            ]
          },
          {
            "name": "cf_cell_intrinsic_only",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "negligible_cell_intrinsic_only_effect",
              "remodeling_unresolved",
              "modest_remodeling_few_cell_types"
            ]
          },
          {
            "name": "cf_remodeling_is_contamination",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "apparent_remodeling_reflects_cancer_contamination"
            ]
          },
          {
            "name": "cf_perturbation_design_misread",
            "role": "hard_fail",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "multiple_tumor_suppressors_pooled",
              "no_additional_gene_difference_kras_only",
              "rb1_cdkn2a_double",
              "pcna_essential_gene_control",
              "difference_is_oncogene_not_suppressor"
            ]
          },
          {
            "name": "cf_age_independent_tme",
            "role": "hard_fail",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_independent"
            ]
          },
          {
            "name": "cf_compositional_shift",
            "role": "hard_fail",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_frequency_shift",
              "compositional_immune_infiltration",
              "composition_dominant_state_secondary",
              "both_state_and_composition"
            ]
          },
          {
            "name": "cf_epithelial_or_cancer_substitution",
            "role": "hard_fail",
            "field": "$.most_responsive_noncancer_cell_types[*].cell_type",
            "kind": "avoid",
            "expected": [
              "AT2_cell",
              "AT1_cell",
              "cancer_cell",
              "ciliated",
              "secretory",
              "club_cell",
              "basal_cell",
              "goblet_cell"
            ]
          },
          {
            "name": "cf_tme_breadth_broader_in_young",
            "role": "hard_fail",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_young"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-6-claude-code",
      "modelName": "claude-opus-4-6",
      "harness": "claude-code",
      "task": "gse297023_q6_age_immune_infiltration",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 3076,
      "cost": 3.0199932499999997,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "composition_stable_with_age",
            "role": "gate",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "frequencies_largely_stable"
            ]
          },
          {
            "name": "age_signal_is_state_not_composition",
            "role": "gate",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "frequencies_uninformative_for_infiltration",
            "role": "gate",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_cannot_establish_infiltration"
            ]
          },
          {
            "name": "infiltration_increases_with_age",
            "role": "gate",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_with_age"
            ]
          },
          {
            "name": "infiltration_strength_calibrated",
            "role": "gate",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "significant_increase_strongest_markers_only",
              "consistent_increase_not_statistically_robust"
            ]
          },
          {
            "name": "genotype_marker_fraction_lower_in_pten",
            "role": "gate",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "lower_in_pten_deficient"
            ]
          },
          {
            "name": "cf_composition_shift_reading",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "major_compositional_shift_with_age",
              "increased_immune_fraction_with_age"
            ]
          },
          {
            "name": "cf_lineage_specific_composition_claim",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "myeloid_expansion_with_age",
              "lymphoid_expansion_with_age",
              "minor_shifts_in_some_cell_types"
            ]
          },
          {
            "name": "cf_immunosenescence_depletion_prior",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "immune_depletion_with_age"
            ]
          },
          {
            "name": "cf_age_modality_compositional",
            "role": "hard_fail",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_type_composition_shift",
              "both_equally",
              "composition_dominant_state_secondary"
            ]
          },
          {
            "name": "cf_overread_frequencies_as_infiltration",
            "role": "hard_fail",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_shows_no_infiltration_change",
              "single_cell_quantifies_infiltration",
              "frequencies_show_increased_infiltration",
              "frequencies_show_decreased_infiltration"
            ]
          },
          {
            "name": "cf_infiltration_decreases_immunosenescence",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "decreased_with_age"
            ]
          },
          {
            "name": "cf_infiltration_unchanged_scrna_overread",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "unchanged_with_age"
            ]
          },
          {
            "name": "cf_infiltration_indeterminate_unused_node",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "direction_indeterminate_from_data"
            ]
          },
          {
            "name": "cf_infiltration_mixed_direction",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "mixed_by_immune_cell_type"
            ]
          },
          {
            "name": "cf_infiltration_genotype_confusion",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_only_in_pten_deficient",
              "decreased_only_in_pten_deficient"
            ]
          },
          {
            "name": "cf_pten_infiltration_paper_text_recall",
            "role": "hard_fail",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "higher_in_pten_deficient",
              "higher_in_pten_deficient_only_in_young",
              "higher_in_pten_deficient_only_in_aged"
            ]
          },
          {
            "name": "cf_pseudoreplication_overclaim",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_increase_across_markers"
            ]
          },
          {
            "name": "cf_infiltration_strength_decrease_direction",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_decrease_across_markers",
              "significant_decrease_strongest_markers_only",
              "consistent_decrease_not_statistically_robust"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-6-claude-code",
      "modelName": "claude-opus-4-6",
      "harness": "claude-code",
      "task": "gse297023_q6_age_immune_infiltration",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 2,
      "passed": true,
      "score": null,
      "durationS": 2970,
      "cost": 2.6924957499999995,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "composition_stable_with_age",
            "role": "gate",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "frequencies_largely_stable"
            ]
          },
          {
            "name": "age_signal_is_state_not_composition",
            "role": "gate",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "frequencies_uninformative_for_infiltration",
            "role": "gate",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_cannot_establish_infiltration"
            ]
          },
          {
            "name": "infiltration_increases_with_age",
            "role": "gate",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_with_age"
            ]
          },
          {
            "name": "infiltration_strength_calibrated",
            "role": "gate",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "significant_increase_strongest_markers_only",
              "consistent_increase_not_statistically_robust"
            ]
          },
          {
            "name": "genotype_marker_fraction_lower_in_pten",
            "role": "gate",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "lower_in_pten_deficient"
            ]
          },
          {
            "name": "cf_composition_shift_reading",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "major_compositional_shift_with_age",
              "increased_immune_fraction_with_age"
            ]
          },
          {
            "name": "cf_lineage_specific_composition_claim",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "myeloid_expansion_with_age",
              "lymphoid_expansion_with_age",
              "minor_shifts_in_some_cell_types"
            ]
          },
          {
            "name": "cf_immunosenescence_depletion_prior",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "immune_depletion_with_age"
            ]
          },
          {
            "name": "cf_age_modality_compositional",
            "role": "hard_fail",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_type_composition_shift",
              "both_equally",
              "composition_dominant_state_secondary"
            ]
          },
          {
            "name": "cf_overread_frequencies_as_infiltration",
            "role": "hard_fail",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_shows_no_infiltration_change",
              "single_cell_quantifies_infiltration",
              "frequencies_show_increased_infiltration",
              "frequencies_show_decreased_infiltration"
            ]
          },
          {
            "name": "cf_infiltration_decreases_immunosenescence",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "decreased_with_age"
            ]
          },
          {
            "name": "cf_infiltration_unchanged_scrna_overread",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "unchanged_with_age"
            ]
          },
          {
            "name": "cf_infiltration_indeterminate_unused_node",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "direction_indeterminate_from_data"
            ]
          },
          {
            "name": "cf_infiltration_mixed_direction",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "mixed_by_immune_cell_type"
            ]
          },
          {
            "name": "cf_infiltration_genotype_confusion",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_only_in_pten_deficient",
              "decreased_only_in_pten_deficient"
            ]
          },
          {
            "name": "cf_pten_infiltration_paper_text_recall",
            "role": "hard_fail",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "higher_in_pten_deficient",
              "higher_in_pten_deficient_only_in_young",
              "higher_in_pten_deficient_only_in_aged"
            ]
          },
          {
            "name": "cf_pseudoreplication_overclaim",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_increase_across_markers"
            ]
          },
          {
            "name": "cf_infiltration_strength_decrease_direction",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_decrease_across_markers",
              "significant_decrease_strongest_markers_only",
              "consistent_decrease_not_statistically_robust"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-6-claude-code",
      "modelName": "claude-opus-4-6",
      "harness": "claude-code",
      "task": "gse297023_q6_age_immune_infiltration",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 3,
      "passed": true,
      "score": null,
      "durationS": 4745,
      "cost": 4.378474750000002,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "composition_stable_with_age",
            "role": "gate",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "frequencies_largely_stable"
            ]
          },
          {
            "name": "age_signal_is_state_not_composition",
            "role": "gate",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "frequencies_uninformative_for_infiltration",
            "role": "gate",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_cannot_establish_infiltration"
            ]
          },
          {
            "name": "infiltration_increases_with_age",
            "role": "gate",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_with_age"
            ]
          },
          {
            "name": "infiltration_strength_calibrated",
            "role": "gate",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "significant_increase_strongest_markers_only",
              "consistent_increase_not_statistically_robust"
            ]
          },
          {
            "name": "genotype_marker_fraction_lower_in_pten",
            "role": "gate",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "lower_in_pten_deficient"
            ]
          },
          {
            "name": "cf_composition_shift_reading",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "major_compositional_shift_with_age",
              "increased_immune_fraction_with_age"
            ]
          },
          {
            "name": "cf_lineage_specific_composition_claim",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "myeloid_expansion_with_age",
              "lymphoid_expansion_with_age",
              "minor_shifts_in_some_cell_types"
            ]
          },
          {
            "name": "cf_immunosenescence_depletion_prior",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "immune_depletion_with_age"
            ]
          },
          {
            "name": "cf_age_modality_compositional",
            "role": "hard_fail",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_type_composition_shift",
              "both_equally",
              "composition_dominant_state_secondary"
            ]
          },
          {
            "name": "cf_overread_frequencies_as_infiltration",
            "role": "hard_fail",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_shows_no_infiltration_change",
              "single_cell_quantifies_infiltration",
              "frequencies_show_increased_infiltration",
              "frequencies_show_decreased_infiltration"
            ]
          },
          {
            "name": "cf_infiltration_decreases_immunosenescence",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "decreased_with_age"
            ]
          },
          {
            "name": "cf_infiltration_unchanged_scrna_overread",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "unchanged_with_age"
            ]
          },
          {
            "name": "cf_infiltration_indeterminate_unused_node",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "direction_indeterminate_from_data"
            ]
          },
          {
            "name": "cf_infiltration_mixed_direction",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "mixed_by_immune_cell_type"
            ]
          },
          {
            "name": "cf_infiltration_genotype_confusion",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_only_in_pten_deficient",
              "decreased_only_in_pten_deficient"
            ]
          },
          {
            "name": "cf_pten_infiltration_paper_text_recall",
            "role": "hard_fail",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "higher_in_pten_deficient",
              "higher_in_pten_deficient_only_in_young",
              "higher_in_pten_deficient_only_in_aged"
            ]
          },
          {
            "name": "cf_pseudoreplication_overclaim",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_increase_across_markers"
            ]
          },
          {
            "name": "cf_infiltration_strength_decrease_direction",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_decrease_across_markers",
              "significant_decrease_strongest_markers_only",
              "consistent_decrease_not_statistically_robust"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-6-pi",
      "modelName": "claude-opus-4-6",
      "harness": "pi",
      "task": "gse297023_q6_age_immune_infiltration",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 1727,
      "cost": 1.3538117500000002,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "composition_stable_with_age",
            "role": "gate",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "frequencies_largely_stable"
            ]
          },
          {
            "name": "age_signal_is_state_not_composition",
            "role": "gate",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "frequencies_uninformative_for_infiltration",
            "role": "gate",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_cannot_establish_infiltration"
            ]
          },
          {
            "name": "infiltration_increases_with_age",
            "role": "gate",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_with_age"
            ]
          },
          {
            "name": "infiltration_strength_calibrated",
            "role": "gate",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "significant_increase_strongest_markers_only",
              "consistent_increase_not_statistically_robust"
            ]
          },
          {
            "name": "genotype_marker_fraction_lower_in_pten",
            "role": "gate",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "lower_in_pten_deficient"
            ]
          },
          {
            "name": "cf_composition_shift_reading",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "major_compositional_shift_with_age",
              "increased_immune_fraction_with_age"
            ]
          },
          {
            "name": "cf_lineage_specific_composition_claim",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "myeloid_expansion_with_age",
              "lymphoid_expansion_with_age",
              "minor_shifts_in_some_cell_types"
            ]
          },
          {
            "name": "cf_immunosenescence_depletion_prior",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "immune_depletion_with_age"
            ]
          },
          {
            "name": "cf_age_modality_compositional",
            "role": "hard_fail",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_type_composition_shift",
              "both_equally",
              "composition_dominant_state_secondary"
            ]
          },
          {
            "name": "cf_overread_frequencies_as_infiltration",
            "role": "hard_fail",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_shows_no_infiltration_change",
              "single_cell_quantifies_infiltration",
              "frequencies_show_increased_infiltration",
              "frequencies_show_decreased_infiltration"
            ]
          },
          {
            "name": "cf_infiltration_decreases_immunosenescence",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "decreased_with_age"
            ]
          },
          {
            "name": "cf_infiltration_unchanged_scrna_overread",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "unchanged_with_age"
            ]
          },
          {
            "name": "cf_infiltration_indeterminate_unused_node",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "direction_indeterminate_from_data"
            ]
          },
          {
            "name": "cf_infiltration_mixed_direction",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "mixed_by_immune_cell_type"
            ]
          },
          {
            "name": "cf_infiltration_genotype_confusion",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_only_in_pten_deficient",
              "decreased_only_in_pten_deficient"
            ]
          },
          {
            "name": "cf_pten_infiltration_paper_text_recall",
            "role": "hard_fail",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "higher_in_pten_deficient",
              "higher_in_pten_deficient_only_in_young",
              "higher_in_pten_deficient_only_in_aged"
            ]
          },
          {
            "name": "cf_pseudoreplication_overclaim",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_increase_across_markers"
            ]
          },
          {
            "name": "cf_infiltration_strength_decrease_direction",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_decrease_across_markers",
              "significant_decrease_strongest_markers_only",
              "consistent_decrease_not_statistically_robust"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-6-pi",
      "modelName": "claude-opus-4-6",
      "harness": "pi",
      "task": "gse297023_q6_age_immune_infiltration",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 1819,
      "cost": 1.47986375,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "composition_stable_with_age",
            "role": "gate",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "frequencies_largely_stable"
            ]
          },
          {
            "name": "age_signal_is_state_not_composition",
            "role": "gate",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "frequencies_uninformative_for_infiltration",
            "role": "gate",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_cannot_establish_infiltration"
            ]
          },
          {
            "name": "infiltration_increases_with_age",
            "role": "gate",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_with_age"
            ]
          },
          {
            "name": "infiltration_strength_calibrated",
            "role": "gate",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "significant_increase_strongest_markers_only",
              "consistent_increase_not_statistically_robust"
            ]
          },
          {
            "name": "genotype_marker_fraction_lower_in_pten",
            "role": "gate",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "lower_in_pten_deficient"
            ]
          },
          {
            "name": "cf_composition_shift_reading",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "major_compositional_shift_with_age",
              "increased_immune_fraction_with_age"
            ]
          },
          {
            "name": "cf_lineage_specific_composition_claim",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "myeloid_expansion_with_age",
              "lymphoid_expansion_with_age",
              "minor_shifts_in_some_cell_types"
            ]
          },
          {
            "name": "cf_immunosenescence_depletion_prior",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "immune_depletion_with_age"
            ]
          },
          {
            "name": "cf_age_modality_compositional",
            "role": "hard_fail",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_type_composition_shift",
              "both_equally",
              "composition_dominant_state_secondary"
            ]
          },
          {
            "name": "cf_overread_frequencies_as_infiltration",
            "role": "hard_fail",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_shows_no_infiltration_change",
              "single_cell_quantifies_infiltration",
              "frequencies_show_increased_infiltration",
              "frequencies_show_decreased_infiltration"
            ]
          },
          {
            "name": "cf_infiltration_decreases_immunosenescence",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "decreased_with_age"
            ]
          },
          {
            "name": "cf_infiltration_unchanged_scrna_overread",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "unchanged_with_age"
            ]
          },
          {
            "name": "cf_infiltration_indeterminate_unused_node",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "direction_indeterminate_from_data"
            ]
          },
          {
            "name": "cf_infiltration_mixed_direction",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "mixed_by_immune_cell_type"
            ]
          },
          {
            "name": "cf_infiltration_genotype_confusion",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_only_in_pten_deficient",
              "decreased_only_in_pten_deficient"
            ]
          },
          {
            "name": "cf_pten_infiltration_paper_text_recall",
            "role": "hard_fail",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "higher_in_pten_deficient",
              "higher_in_pten_deficient_only_in_young",
              "higher_in_pten_deficient_only_in_aged"
            ]
          },
          {
            "name": "cf_pseudoreplication_overclaim",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_increase_across_markers"
            ]
          },
          {
            "name": "cf_infiltration_strength_decrease_direction",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_decrease_across_markers",
              "significant_decrease_strongest_markers_only",
              "consistent_decrease_not_statistically_robust"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-6-pi",
      "modelName": "claude-opus-4-6",
      "harness": "pi",
      "task": "gse297023_q6_age_immune_infiltration",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1690,
      "cost": 1.5616789999999998,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "composition_stable_with_age",
            "role": "gate",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "frequencies_largely_stable"
            ]
          },
          {
            "name": "age_signal_is_state_not_composition",
            "role": "gate",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "frequencies_uninformative_for_infiltration",
            "role": "gate",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_cannot_establish_infiltration"
            ]
          },
          {
            "name": "infiltration_increases_with_age",
            "role": "gate",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_with_age"
            ]
          },
          {
            "name": "infiltration_strength_calibrated",
            "role": "gate",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "significant_increase_strongest_markers_only",
              "consistent_increase_not_statistically_robust"
            ]
          },
          {
            "name": "genotype_marker_fraction_lower_in_pten",
            "role": "gate",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "lower_in_pten_deficient"
            ]
          },
          {
            "name": "cf_composition_shift_reading",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "major_compositional_shift_with_age",
              "increased_immune_fraction_with_age"
            ]
          },
          {
            "name": "cf_lineage_specific_composition_claim",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "myeloid_expansion_with_age",
              "lymphoid_expansion_with_age",
              "minor_shifts_in_some_cell_types"
            ]
          },
          {
            "name": "cf_immunosenescence_depletion_prior",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "immune_depletion_with_age"
            ]
          },
          {
            "name": "cf_age_modality_compositional",
            "role": "hard_fail",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_type_composition_shift",
              "both_equally",
              "composition_dominant_state_secondary"
            ]
          },
          {
            "name": "cf_overread_frequencies_as_infiltration",
            "role": "hard_fail",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_shows_no_infiltration_change",
              "single_cell_quantifies_infiltration",
              "frequencies_show_increased_infiltration",
              "frequencies_show_decreased_infiltration"
            ]
          },
          {
            "name": "cf_infiltration_decreases_immunosenescence",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "decreased_with_age"
            ]
          },
          {
            "name": "cf_infiltration_unchanged_scrna_overread",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "unchanged_with_age"
            ]
          },
          {
            "name": "cf_infiltration_indeterminate_unused_node",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "direction_indeterminate_from_data"
            ]
          },
          {
            "name": "cf_infiltration_mixed_direction",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "mixed_by_immune_cell_type"
            ]
          },
          {
            "name": "cf_infiltration_genotype_confusion",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_only_in_pten_deficient",
              "decreased_only_in_pten_deficient"
            ]
          },
          {
            "name": "cf_pten_infiltration_paper_text_recall",
            "role": "hard_fail",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "higher_in_pten_deficient",
              "higher_in_pten_deficient_only_in_young",
              "higher_in_pten_deficient_only_in_aged"
            ]
          },
          {
            "name": "cf_pseudoreplication_overclaim",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_increase_across_markers"
            ]
          },
          {
            "name": "cf_infiltration_strength_decrease_direction",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_decrease_across_markers",
              "significant_decrease_strongest_markers_only",
              "consistent_decrease_not_statistically_robust"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-7-claude-code",
      "modelName": "claude-opus-4-7",
      "harness": "claude-code",
      "task": "gse297023_q6_age_immune_infiltration",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 757,
      "cost": 2.7094302499999996,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "composition_stable_with_age",
            "role": "gate",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "frequencies_largely_stable"
            ]
          },
          {
            "name": "age_signal_is_state_not_composition",
            "role": "gate",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "frequencies_uninformative_for_infiltration",
            "role": "gate",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_cannot_establish_infiltration"
            ]
          },
          {
            "name": "infiltration_increases_with_age",
            "role": "gate",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_with_age"
            ]
          },
          {
            "name": "infiltration_strength_calibrated",
            "role": "gate",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "significant_increase_strongest_markers_only",
              "consistent_increase_not_statistically_robust"
            ]
          },
          {
            "name": "genotype_marker_fraction_lower_in_pten",
            "role": "gate",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "lower_in_pten_deficient"
            ]
          },
          {
            "name": "cf_composition_shift_reading",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "major_compositional_shift_with_age",
              "increased_immune_fraction_with_age"
            ]
          },
          {
            "name": "cf_lineage_specific_composition_claim",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "myeloid_expansion_with_age",
              "lymphoid_expansion_with_age",
              "minor_shifts_in_some_cell_types"
            ]
          },
          {
            "name": "cf_immunosenescence_depletion_prior",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "immune_depletion_with_age"
            ]
          },
          {
            "name": "cf_age_modality_compositional",
            "role": "hard_fail",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_type_composition_shift",
              "both_equally",
              "composition_dominant_state_secondary"
            ]
          },
          {
            "name": "cf_overread_frequencies_as_infiltration",
            "role": "hard_fail",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_shows_no_infiltration_change",
              "single_cell_quantifies_infiltration",
              "frequencies_show_increased_infiltration",
              "frequencies_show_decreased_infiltration"
            ]
          },
          {
            "name": "cf_infiltration_decreases_immunosenescence",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "decreased_with_age"
            ]
          },
          {
            "name": "cf_infiltration_unchanged_scrna_overread",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "unchanged_with_age"
            ]
          },
          {
            "name": "cf_infiltration_indeterminate_unused_node",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "direction_indeterminate_from_data"
            ]
          },
          {
            "name": "cf_infiltration_mixed_direction",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "mixed_by_immune_cell_type"
            ]
          },
          {
            "name": "cf_infiltration_genotype_confusion",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_only_in_pten_deficient",
              "decreased_only_in_pten_deficient"
            ]
          },
          {
            "name": "cf_pten_infiltration_paper_text_recall",
            "role": "hard_fail",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "higher_in_pten_deficient",
              "higher_in_pten_deficient_only_in_young",
              "higher_in_pten_deficient_only_in_aged"
            ]
          },
          {
            "name": "cf_pseudoreplication_overclaim",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_increase_across_markers"
            ]
          },
          {
            "name": "cf_infiltration_strength_decrease_direction",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_decrease_across_markers",
              "significant_decrease_strongest_markers_only",
              "consistent_decrease_not_statistically_robust"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-7-claude-code",
      "modelName": "claude-opus-4-7",
      "harness": "claude-code",
      "task": "gse297023_q6_age_immune_infiltration",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 1532,
      "cost": 2.6843082500000004,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "composition_stable_with_age",
            "role": "gate",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "frequencies_largely_stable"
            ]
          },
          {
            "name": "age_signal_is_state_not_composition",
            "role": "gate",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "frequencies_uninformative_for_infiltration",
            "role": "gate",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_cannot_establish_infiltration"
            ]
          },
          {
            "name": "infiltration_increases_with_age",
            "role": "gate",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_with_age"
            ]
          },
          {
            "name": "infiltration_strength_calibrated",
            "role": "gate",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "significant_increase_strongest_markers_only",
              "consistent_increase_not_statistically_robust"
            ]
          },
          {
            "name": "genotype_marker_fraction_lower_in_pten",
            "role": "gate",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "lower_in_pten_deficient"
            ]
          },
          {
            "name": "cf_composition_shift_reading",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "major_compositional_shift_with_age",
              "increased_immune_fraction_with_age"
            ]
          },
          {
            "name": "cf_lineage_specific_composition_claim",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "myeloid_expansion_with_age",
              "lymphoid_expansion_with_age",
              "minor_shifts_in_some_cell_types"
            ]
          },
          {
            "name": "cf_immunosenescence_depletion_prior",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "immune_depletion_with_age"
            ]
          },
          {
            "name": "cf_age_modality_compositional",
            "role": "hard_fail",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_type_composition_shift",
              "both_equally",
              "composition_dominant_state_secondary"
            ]
          },
          {
            "name": "cf_overread_frequencies_as_infiltration",
            "role": "hard_fail",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_shows_no_infiltration_change",
              "single_cell_quantifies_infiltration",
              "frequencies_show_increased_infiltration",
              "frequencies_show_decreased_infiltration"
            ]
          },
          {
            "name": "cf_infiltration_decreases_immunosenescence",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "decreased_with_age"
            ]
          },
          {
            "name": "cf_infiltration_unchanged_scrna_overread",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "unchanged_with_age"
            ]
          },
          {
            "name": "cf_infiltration_indeterminate_unused_node",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "direction_indeterminate_from_data"
            ]
          },
          {
            "name": "cf_infiltration_mixed_direction",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "mixed_by_immune_cell_type"
            ]
          },
          {
            "name": "cf_infiltration_genotype_confusion",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_only_in_pten_deficient",
              "decreased_only_in_pten_deficient"
            ]
          },
          {
            "name": "cf_pten_infiltration_paper_text_recall",
            "role": "hard_fail",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "higher_in_pten_deficient",
              "higher_in_pten_deficient_only_in_young",
              "higher_in_pten_deficient_only_in_aged"
            ]
          },
          {
            "name": "cf_pseudoreplication_overclaim",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_increase_across_markers"
            ]
          },
          {
            "name": "cf_infiltration_strength_decrease_direction",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_decrease_across_markers",
              "significant_decrease_strongest_markers_only",
              "consistent_decrease_not_statistically_robust"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-7-claude-code",
      "modelName": "claude-opus-4-7",
      "harness": "claude-code",
      "task": "gse297023_q6_age_immune_infiltration",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1103,
      "cost": 3.0968204999999998,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "composition_stable_with_age",
            "role": "gate",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "frequencies_largely_stable"
            ]
          },
          {
            "name": "age_signal_is_state_not_composition",
            "role": "gate",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "frequencies_uninformative_for_infiltration",
            "role": "gate",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_cannot_establish_infiltration"
            ]
          },
          {
            "name": "infiltration_increases_with_age",
            "role": "gate",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_with_age"
            ]
          },
          {
            "name": "infiltration_strength_calibrated",
            "role": "gate",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "significant_increase_strongest_markers_only",
              "consistent_increase_not_statistically_robust"
            ]
          },
          {
            "name": "genotype_marker_fraction_lower_in_pten",
            "role": "gate",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "lower_in_pten_deficient"
            ]
          },
          {
            "name": "cf_composition_shift_reading",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "major_compositional_shift_with_age",
              "increased_immune_fraction_with_age"
            ]
          },
          {
            "name": "cf_lineage_specific_composition_claim",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "myeloid_expansion_with_age",
              "lymphoid_expansion_with_age",
              "minor_shifts_in_some_cell_types"
            ]
          },
          {
            "name": "cf_immunosenescence_depletion_prior",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "immune_depletion_with_age"
            ]
          },
          {
            "name": "cf_age_modality_compositional",
            "role": "hard_fail",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_type_composition_shift",
              "both_equally",
              "composition_dominant_state_secondary"
            ]
          },
          {
            "name": "cf_overread_frequencies_as_infiltration",
            "role": "hard_fail",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_shows_no_infiltration_change",
              "single_cell_quantifies_infiltration",
              "frequencies_show_increased_infiltration",
              "frequencies_show_decreased_infiltration"
            ]
          },
          {
            "name": "cf_infiltration_decreases_immunosenescence",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "decreased_with_age"
            ]
          },
          {
            "name": "cf_infiltration_unchanged_scrna_overread",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "unchanged_with_age"
            ]
          },
          {
            "name": "cf_infiltration_indeterminate_unused_node",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "direction_indeterminate_from_data"
            ]
          },
          {
            "name": "cf_infiltration_mixed_direction",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "mixed_by_immune_cell_type"
            ]
          },
          {
            "name": "cf_infiltration_genotype_confusion",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_only_in_pten_deficient",
              "decreased_only_in_pten_deficient"
            ]
          },
          {
            "name": "cf_pten_infiltration_paper_text_recall",
            "role": "hard_fail",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "higher_in_pten_deficient",
              "higher_in_pten_deficient_only_in_young",
              "higher_in_pten_deficient_only_in_aged"
            ]
          },
          {
            "name": "cf_pseudoreplication_overclaim",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_increase_across_markers"
            ]
          },
          {
            "name": "cf_infiltration_strength_decrease_direction",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_decrease_across_markers",
              "significant_decrease_strongest_markers_only",
              "consistent_decrease_not_statistically_robust"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-7-pi",
      "modelName": "claude-opus-4-7",
      "harness": "pi",
      "task": "gse297023_q6_age_immune_infiltration",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 475,
      "cost": 0.76545825,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "composition_stable_with_age",
            "role": "gate",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "frequencies_largely_stable"
            ]
          },
          {
            "name": "age_signal_is_state_not_composition",
            "role": "gate",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "frequencies_uninformative_for_infiltration",
            "role": "gate",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_cannot_establish_infiltration"
            ]
          },
          {
            "name": "infiltration_increases_with_age",
            "role": "gate",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_with_age"
            ]
          },
          {
            "name": "infiltration_strength_calibrated",
            "role": "gate",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "significant_increase_strongest_markers_only",
              "consistent_increase_not_statistically_robust"
            ]
          },
          {
            "name": "genotype_marker_fraction_lower_in_pten",
            "role": "gate",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "lower_in_pten_deficient"
            ]
          },
          {
            "name": "cf_composition_shift_reading",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "major_compositional_shift_with_age",
              "increased_immune_fraction_with_age"
            ]
          },
          {
            "name": "cf_lineage_specific_composition_claim",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "myeloid_expansion_with_age",
              "lymphoid_expansion_with_age",
              "minor_shifts_in_some_cell_types"
            ]
          },
          {
            "name": "cf_immunosenescence_depletion_prior",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "immune_depletion_with_age"
            ]
          },
          {
            "name": "cf_age_modality_compositional",
            "role": "hard_fail",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_type_composition_shift",
              "both_equally",
              "composition_dominant_state_secondary"
            ]
          },
          {
            "name": "cf_overread_frequencies_as_infiltration",
            "role": "hard_fail",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_shows_no_infiltration_change",
              "single_cell_quantifies_infiltration",
              "frequencies_show_increased_infiltration",
              "frequencies_show_decreased_infiltration"
            ]
          },
          {
            "name": "cf_infiltration_decreases_immunosenescence",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "decreased_with_age"
            ]
          },
          {
            "name": "cf_infiltration_unchanged_scrna_overread",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "unchanged_with_age"
            ]
          },
          {
            "name": "cf_infiltration_indeterminate_unused_node",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "direction_indeterminate_from_data"
            ]
          },
          {
            "name": "cf_infiltration_mixed_direction",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "mixed_by_immune_cell_type"
            ]
          },
          {
            "name": "cf_infiltration_genotype_confusion",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_only_in_pten_deficient",
              "decreased_only_in_pten_deficient"
            ]
          },
          {
            "name": "cf_pten_infiltration_paper_text_recall",
            "role": "hard_fail",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "higher_in_pten_deficient",
              "higher_in_pten_deficient_only_in_young",
              "higher_in_pten_deficient_only_in_aged"
            ]
          },
          {
            "name": "cf_pseudoreplication_overclaim",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_increase_across_markers"
            ]
          },
          {
            "name": "cf_infiltration_strength_decrease_direction",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_decrease_across_markers",
              "significant_decrease_strongest_markers_only",
              "consistent_decrease_not_statistically_robust"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-7-pi",
      "modelName": "claude-opus-4-7",
      "harness": "pi",
      "task": "gse297023_q6_age_immune_infiltration",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 825,
      "cost": 1.0419565,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "composition_stable_with_age",
            "role": "gate",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "frequencies_largely_stable"
            ]
          },
          {
            "name": "age_signal_is_state_not_composition",
            "role": "gate",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "frequencies_uninformative_for_infiltration",
            "role": "gate",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_cannot_establish_infiltration"
            ]
          },
          {
            "name": "infiltration_increases_with_age",
            "role": "gate",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_with_age"
            ]
          },
          {
            "name": "infiltration_strength_calibrated",
            "role": "gate",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "significant_increase_strongest_markers_only",
              "consistent_increase_not_statistically_robust"
            ]
          },
          {
            "name": "genotype_marker_fraction_lower_in_pten",
            "role": "gate",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "lower_in_pten_deficient"
            ]
          },
          {
            "name": "cf_composition_shift_reading",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "major_compositional_shift_with_age",
              "increased_immune_fraction_with_age"
            ]
          },
          {
            "name": "cf_lineage_specific_composition_claim",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "myeloid_expansion_with_age",
              "lymphoid_expansion_with_age",
              "minor_shifts_in_some_cell_types"
            ]
          },
          {
            "name": "cf_immunosenescence_depletion_prior",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "immune_depletion_with_age"
            ]
          },
          {
            "name": "cf_age_modality_compositional",
            "role": "hard_fail",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_type_composition_shift",
              "both_equally",
              "composition_dominant_state_secondary"
            ]
          },
          {
            "name": "cf_overread_frequencies_as_infiltration",
            "role": "hard_fail",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_shows_no_infiltration_change",
              "single_cell_quantifies_infiltration",
              "frequencies_show_increased_infiltration",
              "frequencies_show_decreased_infiltration"
            ]
          },
          {
            "name": "cf_infiltration_decreases_immunosenescence",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "decreased_with_age"
            ]
          },
          {
            "name": "cf_infiltration_unchanged_scrna_overread",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "unchanged_with_age"
            ]
          },
          {
            "name": "cf_infiltration_indeterminate_unused_node",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "direction_indeterminate_from_data"
            ]
          },
          {
            "name": "cf_infiltration_mixed_direction",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "mixed_by_immune_cell_type"
            ]
          },
          {
            "name": "cf_infiltration_genotype_confusion",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_only_in_pten_deficient",
              "decreased_only_in_pten_deficient"
            ]
          },
          {
            "name": "cf_pten_infiltration_paper_text_recall",
            "role": "hard_fail",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "higher_in_pten_deficient",
              "higher_in_pten_deficient_only_in_young",
              "higher_in_pten_deficient_only_in_aged"
            ]
          },
          {
            "name": "cf_pseudoreplication_overclaim",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_increase_across_markers"
            ]
          },
          {
            "name": "cf_infiltration_strength_decrease_direction",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_decrease_across_markers",
              "significant_decrease_strongest_markers_only",
              "consistent_decrease_not_statistically_robust"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-7-pi",
      "modelName": "claude-opus-4-7",
      "harness": "pi",
      "task": "gse297023_q6_age_immune_infiltration",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1062,
      "cost": 1.0411415000000002,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "composition_stable_with_age",
            "role": "gate",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "frequencies_largely_stable"
            ]
          },
          {
            "name": "age_signal_is_state_not_composition",
            "role": "gate",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "frequencies_uninformative_for_infiltration",
            "role": "gate",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_cannot_establish_infiltration"
            ]
          },
          {
            "name": "infiltration_increases_with_age",
            "role": "gate",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_with_age"
            ]
          },
          {
            "name": "infiltration_strength_calibrated",
            "role": "gate",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "significant_increase_strongest_markers_only",
              "consistent_increase_not_statistically_robust"
            ]
          },
          {
            "name": "genotype_marker_fraction_lower_in_pten",
            "role": "gate",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "lower_in_pten_deficient"
            ]
          },
          {
            "name": "cf_composition_shift_reading",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "major_compositional_shift_with_age",
              "increased_immune_fraction_with_age"
            ]
          },
          {
            "name": "cf_lineage_specific_composition_claim",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "myeloid_expansion_with_age",
              "lymphoid_expansion_with_age",
              "minor_shifts_in_some_cell_types"
            ]
          },
          {
            "name": "cf_immunosenescence_depletion_prior",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "immune_depletion_with_age"
            ]
          },
          {
            "name": "cf_age_modality_compositional",
            "role": "hard_fail",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_type_composition_shift",
              "both_equally",
              "composition_dominant_state_secondary"
            ]
          },
          {
            "name": "cf_overread_frequencies_as_infiltration",
            "role": "hard_fail",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_shows_no_infiltration_change",
              "single_cell_quantifies_infiltration",
              "frequencies_show_increased_infiltration",
              "frequencies_show_decreased_infiltration"
            ]
          },
          {
            "name": "cf_infiltration_decreases_immunosenescence",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "decreased_with_age"
            ]
          },
          {
            "name": "cf_infiltration_unchanged_scrna_overread",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "unchanged_with_age"
            ]
          },
          {
            "name": "cf_infiltration_indeterminate_unused_node",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "direction_indeterminate_from_data"
            ]
          },
          {
            "name": "cf_infiltration_mixed_direction",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "mixed_by_immune_cell_type"
            ]
          },
          {
            "name": "cf_infiltration_genotype_confusion",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_only_in_pten_deficient",
              "decreased_only_in_pten_deficient"
            ]
          },
          {
            "name": "cf_pten_infiltration_paper_text_recall",
            "role": "hard_fail",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "higher_in_pten_deficient",
              "higher_in_pten_deficient_only_in_young",
              "higher_in_pten_deficient_only_in_aged"
            ]
          },
          {
            "name": "cf_pseudoreplication_overclaim",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_increase_across_markers"
            ]
          },
          {
            "name": "cf_infiltration_strength_decrease_direction",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_decrease_across_markers",
              "significant_decrease_strongest_markers_only",
              "consistent_decrease_not_statistically_robust"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-8-claude-code",
      "modelName": "claude-opus-4-8",
      "harness": "claude-code",
      "task": "gse297023_q6_age_immune_infiltration",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 1,
      "passed": true,
      "score": null,
      "durationS": 1778,
      "cost": 10.375202500000007,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "composition_stable_with_age",
            "role": "gate",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "frequencies_largely_stable"
            ]
          },
          {
            "name": "age_signal_is_state_not_composition",
            "role": "gate",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "frequencies_uninformative_for_infiltration",
            "role": "gate",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_cannot_establish_infiltration"
            ]
          },
          {
            "name": "infiltration_increases_with_age",
            "role": "gate",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_with_age"
            ]
          },
          {
            "name": "infiltration_strength_calibrated",
            "role": "gate",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "significant_increase_strongest_markers_only",
              "consistent_increase_not_statistically_robust"
            ]
          },
          {
            "name": "genotype_marker_fraction_lower_in_pten",
            "role": "gate",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "lower_in_pten_deficient"
            ]
          },
          {
            "name": "cf_composition_shift_reading",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "major_compositional_shift_with_age",
              "increased_immune_fraction_with_age"
            ]
          },
          {
            "name": "cf_lineage_specific_composition_claim",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "myeloid_expansion_with_age",
              "lymphoid_expansion_with_age",
              "minor_shifts_in_some_cell_types"
            ]
          },
          {
            "name": "cf_immunosenescence_depletion_prior",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "immune_depletion_with_age"
            ]
          },
          {
            "name": "cf_age_modality_compositional",
            "role": "hard_fail",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_type_composition_shift",
              "both_equally",
              "composition_dominant_state_secondary"
            ]
          },
          {
            "name": "cf_overread_frequencies_as_infiltration",
            "role": "hard_fail",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_shows_no_infiltration_change",
              "single_cell_quantifies_infiltration",
              "frequencies_show_increased_infiltration",
              "frequencies_show_decreased_infiltration"
            ]
          },
          {
            "name": "cf_infiltration_decreases_immunosenescence",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "decreased_with_age"
            ]
          },
          {
            "name": "cf_infiltration_unchanged_scrna_overread",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "unchanged_with_age"
            ]
          },
          {
            "name": "cf_infiltration_indeterminate_unused_node",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "direction_indeterminate_from_data"
            ]
          },
          {
            "name": "cf_infiltration_mixed_direction",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "mixed_by_immune_cell_type"
            ]
          },
          {
            "name": "cf_infiltration_genotype_confusion",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_only_in_pten_deficient",
              "decreased_only_in_pten_deficient"
            ]
          },
          {
            "name": "cf_pten_infiltration_paper_text_recall",
            "role": "hard_fail",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "higher_in_pten_deficient",
              "higher_in_pten_deficient_only_in_young",
              "higher_in_pten_deficient_only_in_aged"
            ]
          },
          {
            "name": "cf_pseudoreplication_overclaim",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_increase_across_markers"
            ]
          },
          {
            "name": "cf_infiltration_strength_decrease_direction",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_decrease_across_markers",
              "significant_decrease_strongest_markers_only",
              "consistent_decrease_not_statistically_robust"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-8-claude-code",
      "modelName": "claude-opus-4-8",
      "harness": "claude-code",
      "task": "gse297023_q6_age_immune_infiltration",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 2,
      "passed": true,
      "score": null,
      "durationS": 4234,
      "cost": 12.080023500000003,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "composition_stable_with_age",
            "role": "gate",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "frequencies_largely_stable"
            ]
          },
          {
            "name": "age_signal_is_state_not_composition",
            "role": "gate",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "frequencies_uninformative_for_infiltration",
            "role": "gate",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_cannot_establish_infiltration"
            ]
          },
          {
            "name": "infiltration_increases_with_age",
            "role": "gate",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_with_age"
            ]
          },
          {
            "name": "infiltration_strength_calibrated",
            "role": "gate",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "significant_increase_strongest_markers_only",
              "consistent_increase_not_statistically_robust"
            ]
          },
          {
            "name": "genotype_marker_fraction_lower_in_pten",
            "role": "gate",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "lower_in_pten_deficient"
            ]
          },
          {
            "name": "cf_composition_shift_reading",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "major_compositional_shift_with_age",
              "increased_immune_fraction_with_age"
            ]
          },
          {
            "name": "cf_lineage_specific_composition_claim",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "myeloid_expansion_with_age",
              "lymphoid_expansion_with_age",
              "minor_shifts_in_some_cell_types"
            ]
          },
          {
            "name": "cf_immunosenescence_depletion_prior",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "immune_depletion_with_age"
            ]
          },
          {
            "name": "cf_age_modality_compositional",
            "role": "hard_fail",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_type_composition_shift",
              "both_equally",
              "composition_dominant_state_secondary"
            ]
          },
          {
            "name": "cf_overread_frequencies_as_infiltration",
            "role": "hard_fail",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_shows_no_infiltration_change",
              "single_cell_quantifies_infiltration",
              "frequencies_show_increased_infiltration",
              "frequencies_show_decreased_infiltration"
            ]
          },
          {
            "name": "cf_infiltration_decreases_immunosenescence",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "decreased_with_age"
            ]
          },
          {
            "name": "cf_infiltration_unchanged_scrna_overread",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "unchanged_with_age"
            ]
          },
          {
            "name": "cf_infiltration_indeterminate_unused_node",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "direction_indeterminate_from_data"
            ]
          },
          {
            "name": "cf_infiltration_mixed_direction",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "mixed_by_immune_cell_type"
            ]
          },
          {
            "name": "cf_infiltration_genotype_confusion",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_only_in_pten_deficient",
              "decreased_only_in_pten_deficient"
            ]
          },
          {
            "name": "cf_pten_infiltration_paper_text_recall",
            "role": "hard_fail",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "higher_in_pten_deficient",
              "higher_in_pten_deficient_only_in_young",
              "higher_in_pten_deficient_only_in_aged"
            ]
          },
          {
            "name": "cf_pseudoreplication_overclaim",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_increase_across_markers"
            ]
          },
          {
            "name": "cf_infiltration_strength_decrease_direction",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_decrease_across_markers",
              "significant_decrease_strongest_markers_only",
              "consistent_decrease_not_statistically_robust"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-8-claude-code",
      "modelName": "claude-opus-4-8",
      "harness": "claude-code",
      "task": "gse297023_q6_age_immune_infiltration",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 972,
      "cost": 4.0727667499999995,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "composition_stable_with_age",
            "role": "gate",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "frequencies_largely_stable"
            ]
          },
          {
            "name": "age_signal_is_state_not_composition",
            "role": "gate",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "frequencies_uninformative_for_infiltration",
            "role": "gate",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_cannot_establish_infiltration"
            ]
          },
          {
            "name": "infiltration_increases_with_age",
            "role": "gate",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_with_age"
            ]
          },
          {
            "name": "infiltration_strength_calibrated",
            "role": "gate",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "significant_increase_strongest_markers_only",
              "consistent_increase_not_statistically_robust"
            ]
          },
          {
            "name": "genotype_marker_fraction_lower_in_pten",
            "role": "gate",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "lower_in_pten_deficient"
            ]
          },
          {
            "name": "cf_composition_shift_reading",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "major_compositional_shift_with_age",
              "increased_immune_fraction_with_age"
            ]
          },
          {
            "name": "cf_lineage_specific_composition_claim",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "myeloid_expansion_with_age",
              "lymphoid_expansion_with_age",
              "minor_shifts_in_some_cell_types"
            ]
          },
          {
            "name": "cf_immunosenescence_depletion_prior",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "immune_depletion_with_age"
            ]
          },
          {
            "name": "cf_age_modality_compositional",
            "role": "hard_fail",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_type_composition_shift",
              "both_equally",
              "composition_dominant_state_secondary"
            ]
          },
          {
            "name": "cf_overread_frequencies_as_infiltration",
            "role": "hard_fail",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_shows_no_infiltration_change",
              "single_cell_quantifies_infiltration",
              "frequencies_show_increased_infiltration",
              "frequencies_show_decreased_infiltration"
            ]
          },
          {
            "name": "cf_infiltration_decreases_immunosenescence",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "decreased_with_age"
            ]
          },
          {
            "name": "cf_infiltration_unchanged_scrna_overread",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "unchanged_with_age"
            ]
          },
          {
            "name": "cf_infiltration_indeterminate_unused_node",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "direction_indeterminate_from_data"
            ]
          },
          {
            "name": "cf_infiltration_mixed_direction",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "mixed_by_immune_cell_type"
            ]
          },
          {
            "name": "cf_infiltration_genotype_confusion",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_only_in_pten_deficient",
              "decreased_only_in_pten_deficient"
            ]
          },
          {
            "name": "cf_pten_infiltration_paper_text_recall",
            "role": "hard_fail",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "higher_in_pten_deficient",
              "higher_in_pten_deficient_only_in_young",
              "higher_in_pten_deficient_only_in_aged"
            ]
          },
          {
            "name": "cf_pseudoreplication_overclaim",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_increase_across_markers"
            ]
          },
          {
            "name": "cf_infiltration_strength_decrease_direction",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_decrease_across_markers",
              "significant_decrease_strongest_markers_only",
              "consistent_decrease_not_statistically_robust"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-8-pi",
      "modelName": "claude-opus-4-8",
      "harness": "pi",
      "task": "gse297023_q6_age_immune_infiltration",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 758,
      "cost": 1.4424915,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "composition_stable_with_age",
            "role": "gate",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "frequencies_largely_stable"
            ]
          },
          {
            "name": "age_signal_is_state_not_composition",
            "role": "gate",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "frequencies_uninformative_for_infiltration",
            "role": "gate",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_cannot_establish_infiltration"
            ]
          },
          {
            "name": "infiltration_increases_with_age",
            "role": "gate",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_with_age"
            ]
          },
          {
            "name": "infiltration_strength_calibrated",
            "role": "gate",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "significant_increase_strongest_markers_only",
              "consistent_increase_not_statistically_robust"
            ]
          },
          {
            "name": "genotype_marker_fraction_lower_in_pten",
            "role": "gate",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "lower_in_pten_deficient"
            ]
          },
          {
            "name": "cf_composition_shift_reading",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "major_compositional_shift_with_age",
              "increased_immune_fraction_with_age"
            ]
          },
          {
            "name": "cf_lineage_specific_composition_claim",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "myeloid_expansion_with_age",
              "lymphoid_expansion_with_age",
              "minor_shifts_in_some_cell_types"
            ]
          },
          {
            "name": "cf_immunosenescence_depletion_prior",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "immune_depletion_with_age"
            ]
          },
          {
            "name": "cf_age_modality_compositional",
            "role": "hard_fail",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_type_composition_shift",
              "both_equally",
              "composition_dominant_state_secondary"
            ]
          },
          {
            "name": "cf_overread_frequencies_as_infiltration",
            "role": "hard_fail",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_shows_no_infiltration_change",
              "single_cell_quantifies_infiltration",
              "frequencies_show_increased_infiltration",
              "frequencies_show_decreased_infiltration"
            ]
          },
          {
            "name": "cf_infiltration_decreases_immunosenescence",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "decreased_with_age"
            ]
          },
          {
            "name": "cf_infiltration_unchanged_scrna_overread",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "unchanged_with_age"
            ]
          },
          {
            "name": "cf_infiltration_indeterminate_unused_node",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "direction_indeterminate_from_data"
            ]
          },
          {
            "name": "cf_infiltration_mixed_direction",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "mixed_by_immune_cell_type"
            ]
          },
          {
            "name": "cf_infiltration_genotype_confusion",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_only_in_pten_deficient",
              "decreased_only_in_pten_deficient"
            ]
          },
          {
            "name": "cf_pten_infiltration_paper_text_recall",
            "role": "hard_fail",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "higher_in_pten_deficient",
              "higher_in_pten_deficient_only_in_young",
              "higher_in_pten_deficient_only_in_aged"
            ]
          },
          {
            "name": "cf_pseudoreplication_overclaim",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_increase_across_markers"
            ]
          },
          {
            "name": "cf_infiltration_strength_decrease_direction",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_decrease_across_markers",
              "significant_decrease_strongest_markers_only",
              "consistent_decrease_not_statistically_robust"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-8-pi",
      "modelName": "claude-opus-4-8",
      "harness": "pi",
      "task": "gse297023_q6_age_immune_infiltration",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 2,
      "passed": true,
      "score": null,
      "durationS": 856,
      "cost": 1.2279200000000001,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "composition_stable_with_age",
            "role": "gate",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "frequencies_largely_stable"
            ]
          },
          {
            "name": "age_signal_is_state_not_composition",
            "role": "gate",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "frequencies_uninformative_for_infiltration",
            "role": "gate",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_cannot_establish_infiltration"
            ]
          },
          {
            "name": "infiltration_increases_with_age",
            "role": "gate",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_with_age"
            ]
          },
          {
            "name": "infiltration_strength_calibrated",
            "role": "gate",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "significant_increase_strongest_markers_only",
              "consistent_increase_not_statistically_robust"
            ]
          },
          {
            "name": "genotype_marker_fraction_lower_in_pten",
            "role": "gate",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "lower_in_pten_deficient"
            ]
          },
          {
            "name": "cf_composition_shift_reading",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "major_compositional_shift_with_age",
              "increased_immune_fraction_with_age"
            ]
          },
          {
            "name": "cf_lineage_specific_composition_claim",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "myeloid_expansion_with_age",
              "lymphoid_expansion_with_age",
              "minor_shifts_in_some_cell_types"
            ]
          },
          {
            "name": "cf_immunosenescence_depletion_prior",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "immune_depletion_with_age"
            ]
          },
          {
            "name": "cf_age_modality_compositional",
            "role": "hard_fail",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_type_composition_shift",
              "both_equally",
              "composition_dominant_state_secondary"
            ]
          },
          {
            "name": "cf_overread_frequencies_as_infiltration",
            "role": "hard_fail",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_shows_no_infiltration_change",
              "single_cell_quantifies_infiltration",
              "frequencies_show_increased_infiltration",
              "frequencies_show_decreased_infiltration"
            ]
          },
          {
            "name": "cf_infiltration_decreases_immunosenescence",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "decreased_with_age"
            ]
          },
          {
            "name": "cf_infiltration_unchanged_scrna_overread",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "unchanged_with_age"
            ]
          },
          {
            "name": "cf_infiltration_indeterminate_unused_node",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "direction_indeterminate_from_data"
            ]
          },
          {
            "name": "cf_infiltration_mixed_direction",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "mixed_by_immune_cell_type"
            ]
          },
          {
            "name": "cf_infiltration_genotype_confusion",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_only_in_pten_deficient",
              "decreased_only_in_pten_deficient"
            ]
          },
          {
            "name": "cf_pten_infiltration_paper_text_recall",
            "role": "hard_fail",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "higher_in_pten_deficient",
              "higher_in_pten_deficient_only_in_young",
              "higher_in_pten_deficient_only_in_aged"
            ]
          },
          {
            "name": "cf_pseudoreplication_overclaim",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_increase_across_markers"
            ]
          },
          {
            "name": "cf_infiltration_strength_decrease_direction",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_decrease_across_markers",
              "significant_decrease_strongest_markers_only",
              "consistent_decrease_not_statistically_robust"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-8-pi",
      "modelName": "claude-opus-4-8",
      "harness": "pi",
      "task": "gse297023_q6_age_immune_infiltration",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1384,
      "cost": 1.9581305,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "composition_stable_with_age",
            "role": "gate",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "frequencies_largely_stable"
            ]
          },
          {
            "name": "age_signal_is_state_not_composition",
            "role": "gate",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "frequencies_uninformative_for_infiltration",
            "role": "gate",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_cannot_establish_infiltration"
            ]
          },
          {
            "name": "infiltration_increases_with_age",
            "role": "gate",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_with_age"
            ]
          },
          {
            "name": "infiltration_strength_calibrated",
            "role": "gate",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "significant_increase_strongest_markers_only",
              "consistent_increase_not_statistically_robust"
            ]
          },
          {
            "name": "genotype_marker_fraction_lower_in_pten",
            "role": "gate",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "lower_in_pten_deficient"
            ]
          },
          {
            "name": "cf_composition_shift_reading",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "major_compositional_shift_with_age",
              "increased_immune_fraction_with_age"
            ]
          },
          {
            "name": "cf_lineage_specific_composition_claim",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "myeloid_expansion_with_age",
              "lymphoid_expansion_with_age",
              "minor_shifts_in_some_cell_types"
            ]
          },
          {
            "name": "cf_immunosenescence_depletion_prior",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "immune_depletion_with_age"
            ]
          },
          {
            "name": "cf_age_modality_compositional",
            "role": "hard_fail",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_type_composition_shift",
              "both_equally",
              "composition_dominant_state_secondary"
            ]
          },
          {
            "name": "cf_overread_frequencies_as_infiltration",
            "role": "hard_fail",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_shows_no_infiltration_change",
              "single_cell_quantifies_infiltration",
              "frequencies_show_increased_infiltration",
              "frequencies_show_decreased_infiltration"
            ]
          },
          {
            "name": "cf_infiltration_decreases_immunosenescence",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "decreased_with_age"
            ]
          },
          {
            "name": "cf_infiltration_unchanged_scrna_overread",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "unchanged_with_age"
            ]
          },
          {
            "name": "cf_infiltration_indeterminate_unused_node",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "direction_indeterminate_from_data"
            ]
          },
          {
            "name": "cf_infiltration_mixed_direction",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "mixed_by_immune_cell_type"
            ]
          },
          {
            "name": "cf_infiltration_genotype_confusion",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_only_in_pten_deficient",
              "decreased_only_in_pten_deficient"
            ]
          },
          {
            "name": "cf_pten_infiltration_paper_text_recall",
            "role": "hard_fail",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "higher_in_pten_deficient",
              "higher_in_pten_deficient_only_in_young",
              "higher_in_pten_deficient_only_in_aged"
            ]
          },
          {
            "name": "cf_pseudoreplication_overclaim",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_increase_across_markers"
            ]
          },
          {
            "name": "cf_infiltration_strength_decrease_direction",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_decrease_across_markers",
              "significant_decrease_strongest_markers_only",
              "consistent_decrease_not_statistically_robust"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-4-6-pi",
      "modelName": "claude-sonnet-4-6",
      "harness": "pi",
      "task": "gse297023_q6_age_immune_infiltration",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 1543,
      "cost": 1.46849685,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "composition_stable_with_age",
            "role": "gate",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "frequencies_largely_stable"
            ]
          },
          {
            "name": "age_signal_is_state_not_composition",
            "role": "gate",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "frequencies_uninformative_for_infiltration",
            "role": "gate",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_cannot_establish_infiltration"
            ]
          },
          {
            "name": "infiltration_increases_with_age",
            "role": "gate",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_with_age"
            ]
          },
          {
            "name": "infiltration_strength_calibrated",
            "role": "gate",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "significant_increase_strongest_markers_only",
              "consistent_increase_not_statistically_robust"
            ]
          },
          {
            "name": "genotype_marker_fraction_lower_in_pten",
            "role": "gate",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "lower_in_pten_deficient"
            ]
          },
          {
            "name": "cf_composition_shift_reading",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "major_compositional_shift_with_age",
              "increased_immune_fraction_with_age"
            ]
          },
          {
            "name": "cf_lineage_specific_composition_claim",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "myeloid_expansion_with_age",
              "lymphoid_expansion_with_age",
              "minor_shifts_in_some_cell_types"
            ]
          },
          {
            "name": "cf_immunosenescence_depletion_prior",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "immune_depletion_with_age"
            ]
          },
          {
            "name": "cf_age_modality_compositional",
            "role": "hard_fail",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_type_composition_shift",
              "both_equally",
              "composition_dominant_state_secondary"
            ]
          },
          {
            "name": "cf_overread_frequencies_as_infiltration",
            "role": "hard_fail",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_shows_no_infiltration_change",
              "single_cell_quantifies_infiltration",
              "frequencies_show_increased_infiltration",
              "frequencies_show_decreased_infiltration"
            ]
          },
          {
            "name": "cf_infiltration_decreases_immunosenescence",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "decreased_with_age"
            ]
          },
          {
            "name": "cf_infiltration_unchanged_scrna_overread",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "unchanged_with_age"
            ]
          },
          {
            "name": "cf_infiltration_indeterminate_unused_node",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "direction_indeterminate_from_data"
            ]
          },
          {
            "name": "cf_infiltration_mixed_direction",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "mixed_by_immune_cell_type"
            ]
          },
          {
            "name": "cf_infiltration_genotype_confusion",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_only_in_pten_deficient",
              "decreased_only_in_pten_deficient"
            ]
          },
          {
            "name": "cf_pten_infiltration_paper_text_recall",
            "role": "hard_fail",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "higher_in_pten_deficient",
              "higher_in_pten_deficient_only_in_young",
              "higher_in_pten_deficient_only_in_aged"
            ]
          },
          {
            "name": "cf_pseudoreplication_overclaim",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_increase_across_markers"
            ]
          },
          {
            "name": "cf_infiltration_strength_decrease_direction",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_decrease_across_markers",
              "significant_decrease_strongest_markers_only",
              "consistent_decrease_not_statistically_robust"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-4-6-pi",
      "modelName": "claude-sonnet-4-6",
      "harness": "pi",
      "task": "gse297023_q6_age_immune_infiltration",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 1970,
      "cost": 1.77086805,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "composition_stable_with_age",
            "role": "gate",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "frequencies_largely_stable"
            ]
          },
          {
            "name": "age_signal_is_state_not_composition",
            "role": "gate",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "frequencies_uninformative_for_infiltration",
            "role": "gate",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_cannot_establish_infiltration"
            ]
          },
          {
            "name": "infiltration_increases_with_age",
            "role": "gate",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_with_age"
            ]
          },
          {
            "name": "infiltration_strength_calibrated",
            "role": "gate",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "significant_increase_strongest_markers_only",
              "consistent_increase_not_statistically_robust"
            ]
          },
          {
            "name": "genotype_marker_fraction_lower_in_pten",
            "role": "gate",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "lower_in_pten_deficient"
            ]
          },
          {
            "name": "cf_composition_shift_reading",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "major_compositional_shift_with_age",
              "increased_immune_fraction_with_age"
            ]
          },
          {
            "name": "cf_lineage_specific_composition_claim",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "myeloid_expansion_with_age",
              "lymphoid_expansion_with_age",
              "minor_shifts_in_some_cell_types"
            ]
          },
          {
            "name": "cf_immunosenescence_depletion_prior",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "immune_depletion_with_age"
            ]
          },
          {
            "name": "cf_age_modality_compositional",
            "role": "hard_fail",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_type_composition_shift",
              "both_equally",
              "composition_dominant_state_secondary"
            ]
          },
          {
            "name": "cf_overread_frequencies_as_infiltration",
            "role": "hard_fail",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_shows_no_infiltration_change",
              "single_cell_quantifies_infiltration",
              "frequencies_show_increased_infiltration",
              "frequencies_show_decreased_infiltration"
            ]
          },
          {
            "name": "cf_infiltration_decreases_immunosenescence",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "decreased_with_age"
            ]
          },
          {
            "name": "cf_infiltration_unchanged_scrna_overread",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "unchanged_with_age"
            ]
          },
          {
            "name": "cf_infiltration_indeterminate_unused_node",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "direction_indeterminate_from_data"
            ]
          },
          {
            "name": "cf_infiltration_mixed_direction",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "mixed_by_immune_cell_type"
            ]
          },
          {
            "name": "cf_infiltration_genotype_confusion",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_only_in_pten_deficient",
              "decreased_only_in_pten_deficient"
            ]
          },
          {
            "name": "cf_pten_infiltration_paper_text_recall",
            "role": "hard_fail",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "higher_in_pten_deficient",
              "higher_in_pten_deficient_only_in_young",
              "higher_in_pten_deficient_only_in_aged"
            ]
          },
          {
            "name": "cf_pseudoreplication_overclaim",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_increase_across_markers"
            ]
          },
          {
            "name": "cf_infiltration_strength_decrease_direction",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_decrease_across_markers",
              "significant_decrease_strongest_markers_only",
              "consistent_decrease_not_statistically_robust"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-4-6-pi",
      "modelName": "claude-sonnet-4-6",
      "harness": "pi",
      "task": "gse297023_q6_age_immune_infiltration",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 2136,
      "cost": 1.6558500000000003,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "composition_stable_with_age",
            "role": "gate",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "frequencies_largely_stable"
            ]
          },
          {
            "name": "age_signal_is_state_not_composition",
            "role": "gate",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "frequencies_uninformative_for_infiltration",
            "role": "gate",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_cannot_establish_infiltration"
            ]
          },
          {
            "name": "infiltration_increases_with_age",
            "role": "gate",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_with_age"
            ]
          },
          {
            "name": "infiltration_strength_calibrated",
            "role": "gate",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "significant_increase_strongest_markers_only",
              "consistent_increase_not_statistically_robust"
            ]
          },
          {
            "name": "genotype_marker_fraction_lower_in_pten",
            "role": "gate",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "lower_in_pten_deficient"
            ]
          },
          {
            "name": "cf_composition_shift_reading",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "major_compositional_shift_with_age",
              "increased_immune_fraction_with_age"
            ]
          },
          {
            "name": "cf_lineage_specific_composition_claim",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "myeloid_expansion_with_age",
              "lymphoid_expansion_with_age",
              "minor_shifts_in_some_cell_types"
            ]
          },
          {
            "name": "cf_immunosenescence_depletion_prior",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "immune_depletion_with_age"
            ]
          },
          {
            "name": "cf_age_modality_compositional",
            "role": "hard_fail",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_type_composition_shift",
              "both_equally",
              "composition_dominant_state_secondary"
            ]
          },
          {
            "name": "cf_overread_frequencies_as_infiltration",
            "role": "hard_fail",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_shows_no_infiltration_change",
              "single_cell_quantifies_infiltration",
              "frequencies_show_increased_infiltration",
              "frequencies_show_decreased_infiltration"
            ]
          },
          {
            "name": "cf_infiltration_decreases_immunosenescence",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "decreased_with_age"
            ]
          },
          {
            "name": "cf_infiltration_unchanged_scrna_overread",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "unchanged_with_age"
            ]
          },
          {
            "name": "cf_infiltration_indeterminate_unused_node",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "direction_indeterminate_from_data"
            ]
          },
          {
            "name": "cf_infiltration_mixed_direction",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "mixed_by_immune_cell_type"
            ]
          },
          {
            "name": "cf_infiltration_genotype_confusion",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_only_in_pten_deficient",
              "decreased_only_in_pten_deficient"
            ]
          },
          {
            "name": "cf_pten_infiltration_paper_text_recall",
            "role": "hard_fail",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "higher_in_pten_deficient",
              "higher_in_pten_deficient_only_in_young",
              "higher_in_pten_deficient_only_in_aged"
            ]
          },
          {
            "name": "cf_pseudoreplication_overclaim",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_increase_across_markers"
            ]
          },
          {
            "name": "cf_infiltration_strength_decrease_direction",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_decrease_across_markers",
              "significant_decrease_strongest_markers_only",
              "consistent_decrease_not_statistically_robust"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gemini-3-1-pro-preview-pi",
      "modelName": "gemini-3.1-pro-preview",
      "harness": "pi",
      "task": "gse297023_q6_age_immune_infiltration",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 1,
      "passed": true,
      "score": null,
      "durationS": 1567,
      "cost": 0.6100503999999999,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "composition_stable_with_age",
            "role": "gate",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "frequencies_largely_stable"
            ]
          },
          {
            "name": "age_signal_is_state_not_composition",
            "role": "gate",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "frequencies_uninformative_for_infiltration",
            "role": "gate",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_cannot_establish_infiltration"
            ]
          },
          {
            "name": "infiltration_increases_with_age",
            "role": "gate",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_with_age"
            ]
          },
          {
            "name": "infiltration_strength_calibrated",
            "role": "gate",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "significant_increase_strongest_markers_only",
              "consistent_increase_not_statistically_robust"
            ]
          },
          {
            "name": "genotype_marker_fraction_lower_in_pten",
            "role": "gate",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "lower_in_pten_deficient"
            ]
          },
          {
            "name": "cf_composition_shift_reading",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "major_compositional_shift_with_age",
              "increased_immune_fraction_with_age"
            ]
          },
          {
            "name": "cf_lineage_specific_composition_claim",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "myeloid_expansion_with_age",
              "lymphoid_expansion_with_age",
              "minor_shifts_in_some_cell_types"
            ]
          },
          {
            "name": "cf_immunosenescence_depletion_prior",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "immune_depletion_with_age"
            ]
          },
          {
            "name": "cf_age_modality_compositional",
            "role": "hard_fail",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_type_composition_shift",
              "both_equally",
              "composition_dominant_state_secondary"
            ]
          },
          {
            "name": "cf_overread_frequencies_as_infiltration",
            "role": "hard_fail",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_shows_no_infiltration_change",
              "single_cell_quantifies_infiltration",
              "frequencies_show_increased_infiltration",
              "frequencies_show_decreased_infiltration"
            ]
          },
          {
            "name": "cf_infiltration_decreases_immunosenescence",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "decreased_with_age"
            ]
          },
          {
            "name": "cf_infiltration_unchanged_scrna_overread",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "unchanged_with_age"
            ]
          },
          {
            "name": "cf_infiltration_indeterminate_unused_node",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "direction_indeterminate_from_data"
            ]
          },
          {
            "name": "cf_infiltration_mixed_direction",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "mixed_by_immune_cell_type"
            ]
          },
          {
            "name": "cf_infiltration_genotype_confusion",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_only_in_pten_deficient",
              "decreased_only_in_pten_deficient"
            ]
          },
          {
            "name": "cf_pten_infiltration_paper_text_recall",
            "role": "hard_fail",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "higher_in_pten_deficient",
              "higher_in_pten_deficient_only_in_young",
              "higher_in_pten_deficient_only_in_aged"
            ]
          },
          {
            "name": "cf_pseudoreplication_overclaim",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_increase_across_markers"
            ]
          },
          {
            "name": "cf_infiltration_strength_decrease_direction",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_decrease_across_markers",
              "significant_decrease_strongest_markers_only",
              "consistent_decrease_not_statistically_robust"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gemini-3-1-pro-preview-pi",
      "modelName": "gemini-3.1-pro-preview",
      "harness": "pi",
      "task": "gse297023_q6_age_immune_infiltration",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 890,
      "cost": 0.4119442,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "composition_stable_with_age",
            "role": "gate",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "frequencies_largely_stable"
            ]
          },
          {
            "name": "age_signal_is_state_not_composition",
            "role": "gate",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "frequencies_uninformative_for_infiltration",
            "role": "gate",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_cannot_establish_infiltration"
            ]
          },
          {
            "name": "infiltration_increases_with_age",
            "role": "gate",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_with_age"
            ]
          },
          {
            "name": "infiltration_strength_calibrated",
            "role": "gate",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "significant_increase_strongest_markers_only",
              "consistent_increase_not_statistically_robust"
            ]
          },
          {
            "name": "genotype_marker_fraction_lower_in_pten",
            "role": "gate",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "lower_in_pten_deficient"
            ]
          },
          {
            "name": "cf_composition_shift_reading",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "major_compositional_shift_with_age",
              "increased_immune_fraction_with_age"
            ]
          },
          {
            "name": "cf_lineage_specific_composition_claim",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "myeloid_expansion_with_age",
              "lymphoid_expansion_with_age",
              "minor_shifts_in_some_cell_types"
            ]
          },
          {
            "name": "cf_immunosenescence_depletion_prior",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "immune_depletion_with_age"
            ]
          },
          {
            "name": "cf_age_modality_compositional",
            "role": "hard_fail",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_type_composition_shift",
              "both_equally",
              "composition_dominant_state_secondary"
            ]
          },
          {
            "name": "cf_overread_frequencies_as_infiltration",
            "role": "hard_fail",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_shows_no_infiltration_change",
              "single_cell_quantifies_infiltration",
              "frequencies_show_increased_infiltration",
              "frequencies_show_decreased_infiltration"
            ]
          },
          {
            "name": "cf_infiltration_decreases_immunosenescence",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "decreased_with_age"
            ]
          },
          {
            "name": "cf_infiltration_unchanged_scrna_overread",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "unchanged_with_age"
            ]
          },
          {
            "name": "cf_infiltration_indeterminate_unused_node",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "direction_indeterminate_from_data"
            ]
          },
          {
            "name": "cf_infiltration_mixed_direction",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "mixed_by_immune_cell_type"
            ]
          },
          {
            "name": "cf_infiltration_genotype_confusion",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_only_in_pten_deficient",
              "decreased_only_in_pten_deficient"
            ]
          },
          {
            "name": "cf_pten_infiltration_paper_text_recall",
            "role": "hard_fail",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "higher_in_pten_deficient",
              "higher_in_pten_deficient_only_in_young",
              "higher_in_pten_deficient_only_in_aged"
            ]
          },
          {
            "name": "cf_pseudoreplication_overclaim",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_increase_across_markers"
            ]
          },
          {
            "name": "cf_infiltration_strength_decrease_direction",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_decrease_across_markers",
              "significant_decrease_strongest_markers_only",
              "consistent_decrease_not_statistically_robust"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gemini-3-1-pro-preview-pi",
      "modelName": "gemini-3.1-pro-preview",
      "harness": "pi",
      "task": "gse297023_q6_age_immune_infiltration",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 3,
      "passed": true,
      "score": null,
      "durationS": 851,
      "cost": 0.4696738,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "composition_stable_with_age",
            "role": "gate",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "frequencies_largely_stable"
            ]
          },
          {
            "name": "age_signal_is_state_not_composition",
            "role": "gate",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "frequencies_uninformative_for_infiltration",
            "role": "gate",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_cannot_establish_infiltration"
            ]
          },
          {
            "name": "infiltration_increases_with_age",
            "role": "gate",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_with_age"
            ]
          },
          {
            "name": "infiltration_strength_calibrated",
            "role": "gate",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "significant_increase_strongest_markers_only",
              "consistent_increase_not_statistically_robust"
            ]
          },
          {
            "name": "genotype_marker_fraction_lower_in_pten",
            "role": "gate",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "lower_in_pten_deficient"
            ]
          },
          {
            "name": "cf_composition_shift_reading",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "major_compositional_shift_with_age",
              "increased_immune_fraction_with_age"
            ]
          },
          {
            "name": "cf_lineage_specific_composition_claim",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "myeloid_expansion_with_age",
              "lymphoid_expansion_with_age",
              "minor_shifts_in_some_cell_types"
            ]
          },
          {
            "name": "cf_immunosenescence_depletion_prior",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "immune_depletion_with_age"
            ]
          },
          {
            "name": "cf_age_modality_compositional",
            "role": "hard_fail",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_type_composition_shift",
              "both_equally",
              "composition_dominant_state_secondary"
            ]
          },
          {
            "name": "cf_overread_frequencies_as_infiltration",
            "role": "hard_fail",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_shows_no_infiltration_change",
              "single_cell_quantifies_infiltration",
              "frequencies_show_increased_infiltration",
              "frequencies_show_decreased_infiltration"
            ]
          },
          {
            "name": "cf_infiltration_decreases_immunosenescence",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "decreased_with_age"
            ]
          },
          {
            "name": "cf_infiltration_unchanged_scrna_overread",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "unchanged_with_age"
            ]
          },
          {
            "name": "cf_infiltration_indeterminate_unused_node",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "direction_indeterminate_from_data"
            ]
          },
          {
            "name": "cf_infiltration_mixed_direction",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "mixed_by_immune_cell_type"
            ]
          },
          {
            "name": "cf_infiltration_genotype_confusion",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_only_in_pten_deficient",
              "decreased_only_in_pten_deficient"
            ]
          },
          {
            "name": "cf_pten_infiltration_paper_text_recall",
            "role": "hard_fail",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "higher_in_pten_deficient",
              "higher_in_pten_deficient_only_in_young",
              "higher_in_pten_deficient_only_in_aged"
            ]
          },
          {
            "name": "cf_pseudoreplication_overclaim",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_increase_across_markers"
            ]
          },
          {
            "name": "cf_infiltration_strength_decrease_direction",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_decrease_across_markers",
              "significant_decrease_strongest_markers_only",
              "consistent_decrease_not_statistically_robust"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gemini-3-5-flash-pi",
      "modelName": "gemini-3.5-flash",
      "harness": "pi",
      "task": "gse297023_q6_age_immune_infiltration",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 305,
      "cost": 0.7782828000000002,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "composition_stable_with_age",
            "role": "gate",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "frequencies_largely_stable"
            ]
          },
          {
            "name": "age_signal_is_state_not_composition",
            "role": "gate",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "frequencies_uninformative_for_infiltration",
            "role": "gate",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_cannot_establish_infiltration"
            ]
          },
          {
            "name": "infiltration_increases_with_age",
            "role": "gate",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_with_age"
            ]
          },
          {
            "name": "infiltration_strength_calibrated",
            "role": "gate",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "significant_increase_strongest_markers_only",
              "consistent_increase_not_statistically_robust"
            ]
          },
          {
            "name": "genotype_marker_fraction_lower_in_pten",
            "role": "gate",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "lower_in_pten_deficient"
            ]
          },
          {
            "name": "cf_composition_shift_reading",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "major_compositional_shift_with_age",
              "increased_immune_fraction_with_age"
            ]
          },
          {
            "name": "cf_lineage_specific_composition_claim",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "myeloid_expansion_with_age",
              "lymphoid_expansion_with_age",
              "minor_shifts_in_some_cell_types"
            ]
          },
          {
            "name": "cf_immunosenescence_depletion_prior",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "immune_depletion_with_age"
            ]
          },
          {
            "name": "cf_age_modality_compositional",
            "role": "hard_fail",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_type_composition_shift",
              "both_equally",
              "composition_dominant_state_secondary"
            ]
          },
          {
            "name": "cf_overread_frequencies_as_infiltration",
            "role": "hard_fail",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_shows_no_infiltration_change",
              "single_cell_quantifies_infiltration",
              "frequencies_show_increased_infiltration",
              "frequencies_show_decreased_infiltration"
            ]
          },
          {
            "name": "cf_infiltration_decreases_immunosenescence",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "decreased_with_age"
            ]
          },
          {
            "name": "cf_infiltration_unchanged_scrna_overread",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "unchanged_with_age"
            ]
          },
          {
            "name": "cf_infiltration_indeterminate_unused_node",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "direction_indeterminate_from_data"
            ]
          },
          {
            "name": "cf_infiltration_mixed_direction",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "mixed_by_immune_cell_type"
            ]
          },
          {
            "name": "cf_infiltration_genotype_confusion",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_only_in_pten_deficient",
              "decreased_only_in_pten_deficient"
            ]
          },
          {
            "name": "cf_pten_infiltration_paper_text_recall",
            "role": "hard_fail",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "higher_in_pten_deficient",
              "higher_in_pten_deficient_only_in_young",
              "higher_in_pten_deficient_only_in_aged"
            ]
          },
          {
            "name": "cf_pseudoreplication_overclaim",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_increase_across_markers"
            ]
          },
          {
            "name": "cf_infiltration_strength_decrease_direction",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_decrease_across_markers",
              "significant_decrease_strongest_markers_only",
              "consistent_decrease_not_statistically_robust"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gemini-3-5-flash-pi",
      "modelName": "gemini-3.5-flash",
      "harness": "pi",
      "task": "gse297023_q6_age_immune_infiltration",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 326,
      "cost": 0.96833145,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "composition_stable_with_age",
            "role": "gate",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "frequencies_largely_stable"
            ]
          },
          {
            "name": "age_signal_is_state_not_composition",
            "role": "gate",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "frequencies_uninformative_for_infiltration",
            "role": "gate",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_cannot_establish_infiltration"
            ]
          },
          {
            "name": "infiltration_increases_with_age",
            "role": "gate",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_with_age"
            ]
          },
          {
            "name": "infiltration_strength_calibrated",
            "role": "gate",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "significant_increase_strongest_markers_only",
              "consistent_increase_not_statistically_robust"
            ]
          },
          {
            "name": "genotype_marker_fraction_lower_in_pten",
            "role": "gate",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "lower_in_pten_deficient"
            ]
          },
          {
            "name": "cf_composition_shift_reading",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "major_compositional_shift_with_age",
              "increased_immune_fraction_with_age"
            ]
          },
          {
            "name": "cf_lineage_specific_composition_claim",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "myeloid_expansion_with_age",
              "lymphoid_expansion_with_age",
              "minor_shifts_in_some_cell_types"
            ]
          },
          {
            "name": "cf_immunosenescence_depletion_prior",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "immune_depletion_with_age"
            ]
          },
          {
            "name": "cf_age_modality_compositional",
            "role": "hard_fail",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_type_composition_shift",
              "both_equally",
              "composition_dominant_state_secondary"
            ]
          },
          {
            "name": "cf_overread_frequencies_as_infiltration",
            "role": "hard_fail",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_shows_no_infiltration_change",
              "single_cell_quantifies_infiltration",
              "frequencies_show_increased_infiltration",
              "frequencies_show_decreased_infiltration"
            ]
          },
          {
            "name": "cf_infiltration_decreases_immunosenescence",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "decreased_with_age"
            ]
          },
          {
            "name": "cf_infiltration_unchanged_scrna_overread",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "unchanged_with_age"
            ]
          },
          {
            "name": "cf_infiltration_indeterminate_unused_node",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "direction_indeterminate_from_data"
            ]
          },
          {
            "name": "cf_infiltration_mixed_direction",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "mixed_by_immune_cell_type"
            ]
          },
          {
            "name": "cf_infiltration_genotype_confusion",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_only_in_pten_deficient",
              "decreased_only_in_pten_deficient"
            ]
          },
          {
            "name": "cf_pten_infiltration_paper_text_recall",
            "role": "hard_fail",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "higher_in_pten_deficient",
              "higher_in_pten_deficient_only_in_young",
              "higher_in_pten_deficient_only_in_aged"
            ]
          },
          {
            "name": "cf_pseudoreplication_overclaim",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_increase_across_markers"
            ]
          },
          {
            "name": "cf_infiltration_strength_decrease_direction",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_decrease_across_markers",
              "significant_decrease_strongest_markers_only",
              "consistent_decrease_not_statistically_robust"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gemini-3-5-flash-pi",
      "modelName": "gemini-3.5-flash",
      "harness": "pi",
      "task": "gse297023_q6_age_immune_infiltration",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 401,
      "cost": 1.0159938,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "composition_stable_with_age",
            "role": "gate",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "frequencies_largely_stable"
            ]
          },
          {
            "name": "age_signal_is_state_not_composition",
            "role": "gate",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "frequencies_uninformative_for_infiltration",
            "role": "gate",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_cannot_establish_infiltration"
            ]
          },
          {
            "name": "infiltration_increases_with_age",
            "role": "gate",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_with_age"
            ]
          },
          {
            "name": "infiltration_strength_calibrated",
            "role": "gate",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "significant_increase_strongest_markers_only",
              "consistent_increase_not_statistically_robust"
            ]
          },
          {
            "name": "genotype_marker_fraction_lower_in_pten",
            "role": "gate",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "lower_in_pten_deficient"
            ]
          },
          {
            "name": "cf_composition_shift_reading",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "major_compositional_shift_with_age",
              "increased_immune_fraction_with_age"
            ]
          },
          {
            "name": "cf_lineage_specific_composition_claim",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "myeloid_expansion_with_age",
              "lymphoid_expansion_with_age",
              "minor_shifts_in_some_cell_types"
            ]
          },
          {
            "name": "cf_immunosenescence_depletion_prior",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "immune_depletion_with_age"
            ]
          },
          {
            "name": "cf_age_modality_compositional",
            "role": "hard_fail",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_type_composition_shift",
              "both_equally",
              "composition_dominant_state_secondary"
            ]
          },
          {
            "name": "cf_overread_frequencies_as_infiltration",
            "role": "hard_fail",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_shows_no_infiltration_change",
              "single_cell_quantifies_infiltration",
              "frequencies_show_increased_infiltration",
              "frequencies_show_decreased_infiltration"
            ]
          },
          {
            "name": "cf_infiltration_decreases_immunosenescence",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "decreased_with_age"
            ]
          },
          {
            "name": "cf_infiltration_unchanged_scrna_overread",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "unchanged_with_age"
            ]
          },
          {
            "name": "cf_infiltration_indeterminate_unused_node",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "direction_indeterminate_from_data"
            ]
          },
          {
            "name": "cf_infiltration_mixed_direction",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "mixed_by_immune_cell_type"
            ]
          },
          {
            "name": "cf_infiltration_genotype_confusion",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_only_in_pten_deficient",
              "decreased_only_in_pten_deficient"
            ]
          },
          {
            "name": "cf_pten_infiltration_paper_text_recall",
            "role": "hard_fail",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "higher_in_pten_deficient",
              "higher_in_pten_deficient_only_in_young",
              "higher_in_pten_deficient_only_in_aged"
            ]
          },
          {
            "name": "cf_pseudoreplication_overclaim",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_increase_across_markers"
            ]
          },
          {
            "name": "cf_infiltration_strength_decrease_direction",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_decrease_across_markers",
              "significant_decrease_strongest_markers_only",
              "consistent_decrease_not_statistically_robust"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-glm-5p2-pi",
      "modelName": "glm-5p2",
      "harness": "pi",
      "task": "gse297023_q6_age_immune_infiltration",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 1,
      "passed": true,
      "score": null,
      "durationS": 4946,
      "cost": 4.504291879999999,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "composition_stable_with_age",
            "role": "gate",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "frequencies_largely_stable"
            ]
          },
          {
            "name": "age_signal_is_state_not_composition",
            "role": "gate",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "frequencies_uninformative_for_infiltration",
            "role": "gate",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_cannot_establish_infiltration"
            ]
          },
          {
            "name": "infiltration_increases_with_age",
            "role": "gate",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_with_age"
            ]
          },
          {
            "name": "infiltration_strength_calibrated",
            "role": "gate",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "significant_increase_strongest_markers_only",
              "consistent_increase_not_statistically_robust"
            ]
          },
          {
            "name": "genotype_marker_fraction_lower_in_pten",
            "role": "gate",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "lower_in_pten_deficient"
            ]
          },
          {
            "name": "cf_composition_shift_reading",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "major_compositional_shift_with_age",
              "increased_immune_fraction_with_age"
            ]
          },
          {
            "name": "cf_lineage_specific_composition_claim",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "myeloid_expansion_with_age",
              "lymphoid_expansion_with_age",
              "minor_shifts_in_some_cell_types"
            ]
          },
          {
            "name": "cf_immunosenescence_depletion_prior",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "immune_depletion_with_age"
            ]
          },
          {
            "name": "cf_age_modality_compositional",
            "role": "hard_fail",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_type_composition_shift",
              "both_equally",
              "composition_dominant_state_secondary"
            ]
          },
          {
            "name": "cf_overread_frequencies_as_infiltration",
            "role": "hard_fail",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_shows_no_infiltration_change",
              "single_cell_quantifies_infiltration",
              "frequencies_show_increased_infiltration",
              "frequencies_show_decreased_infiltration"
            ]
          },
          {
            "name": "cf_infiltration_decreases_immunosenescence",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "decreased_with_age"
            ]
          },
          {
            "name": "cf_infiltration_unchanged_scrna_overread",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "unchanged_with_age"
            ]
          },
          {
            "name": "cf_infiltration_indeterminate_unused_node",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "direction_indeterminate_from_data"
            ]
          },
          {
            "name": "cf_infiltration_mixed_direction",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "mixed_by_immune_cell_type"
            ]
          },
          {
            "name": "cf_infiltration_genotype_confusion",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_only_in_pten_deficient",
              "decreased_only_in_pten_deficient"
            ]
          },
          {
            "name": "cf_pten_infiltration_paper_text_recall",
            "role": "hard_fail",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "higher_in_pten_deficient",
              "higher_in_pten_deficient_only_in_young",
              "higher_in_pten_deficient_only_in_aged"
            ]
          },
          {
            "name": "cf_pseudoreplication_overclaim",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_increase_across_markers"
            ]
          },
          {
            "name": "cf_infiltration_strength_decrease_direction",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_decrease_across_markers",
              "significant_decrease_strongest_markers_only",
              "consistent_decrease_not_statistically_robust"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-glm-5p2-pi",
      "modelName": "glm-5p2",
      "harness": "pi",
      "task": "gse297023_q6_age_immune_infiltration",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 4430,
      "cost": 3.284096660000001,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "composition_stable_with_age",
            "role": "gate",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "frequencies_largely_stable"
            ]
          },
          {
            "name": "age_signal_is_state_not_composition",
            "role": "gate",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "frequencies_uninformative_for_infiltration",
            "role": "gate",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_cannot_establish_infiltration"
            ]
          },
          {
            "name": "infiltration_increases_with_age",
            "role": "gate",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_with_age"
            ]
          },
          {
            "name": "infiltration_strength_calibrated",
            "role": "gate",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "significant_increase_strongest_markers_only",
              "consistent_increase_not_statistically_robust"
            ]
          },
          {
            "name": "genotype_marker_fraction_lower_in_pten",
            "role": "gate",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "lower_in_pten_deficient"
            ]
          },
          {
            "name": "cf_composition_shift_reading",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "major_compositional_shift_with_age",
              "increased_immune_fraction_with_age"
            ]
          },
          {
            "name": "cf_lineage_specific_composition_claim",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "myeloid_expansion_with_age",
              "lymphoid_expansion_with_age",
              "minor_shifts_in_some_cell_types"
            ]
          },
          {
            "name": "cf_immunosenescence_depletion_prior",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "immune_depletion_with_age"
            ]
          },
          {
            "name": "cf_age_modality_compositional",
            "role": "hard_fail",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_type_composition_shift",
              "both_equally",
              "composition_dominant_state_secondary"
            ]
          },
          {
            "name": "cf_overread_frequencies_as_infiltration",
            "role": "hard_fail",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_shows_no_infiltration_change",
              "single_cell_quantifies_infiltration",
              "frequencies_show_increased_infiltration",
              "frequencies_show_decreased_infiltration"
            ]
          },
          {
            "name": "cf_infiltration_decreases_immunosenescence",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "decreased_with_age"
            ]
          },
          {
            "name": "cf_infiltration_unchanged_scrna_overread",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "unchanged_with_age"
            ]
          },
          {
            "name": "cf_infiltration_indeterminate_unused_node",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "direction_indeterminate_from_data"
            ]
          },
          {
            "name": "cf_infiltration_mixed_direction",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "mixed_by_immune_cell_type"
            ]
          },
          {
            "name": "cf_infiltration_genotype_confusion",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_only_in_pten_deficient",
              "decreased_only_in_pten_deficient"
            ]
          },
          {
            "name": "cf_pten_infiltration_paper_text_recall",
            "role": "hard_fail",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "higher_in_pten_deficient",
              "higher_in_pten_deficient_only_in_young",
              "higher_in_pten_deficient_only_in_aged"
            ]
          },
          {
            "name": "cf_pseudoreplication_overclaim",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_increase_across_markers"
            ]
          },
          {
            "name": "cf_infiltration_strength_decrease_direction",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_decrease_across_markers",
              "significant_decrease_strongest_markers_only",
              "consistent_decrease_not_statistically_robust"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-glm-5p2-pi",
      "modelName": "glm-5p2",
      "harness": "pi",
      "task": "gse297023_q6_age_immune_infiltration",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 6249,
      "cost": 2.90536906,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "composition_stable_with_age",
            "role": "gate",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "frequencies_largely_stable"
            ]
          },
          {
            "name": "age_signal_is_state_not_composition",
            "role": "gate",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "frequencies_uninformative_for_infiltration",
            "role": "gate",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_cannot_establish_infiltration"
            ]
          },
          {
            "name": "infiltration_increases_with_age",
            "role": "gate",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_with_age"
            ]
          },
          {
            "name": "infiltration_strength_calibrated",
            "role": "gate",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "significant_increase_strongest_markers_only",
              "consistent_increase_not_statistically_robust"
            ]
          },
          {
            "name": "genotype_marker_fraction_lower_in_pten",
            "role": "gate",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "lower_in_pten_deficient"
            ]
          },
          {
            "name": "cf_composition_shift_reading",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "major_compositional_shift_with_age",
              "increased_immune_fraction_with_age"
            ]
          },
          {
            "name": "cf_lineage_specific_composition_claim",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "myeloid_expansion_with_age",
              "lymphoid_expansion_with_age",
              "minor_shifts_in_some_cell_types"
            ]
          },
          {
            "name": "cf_immunosenescence_depletion_prior",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "immune_depletion_with_age"
            ]
          },
          {
            "name": "cf_age_modality_compositional",
            "role": "hard_fail",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_type_composition_shift",
              "both_equally",
              "composition_dominant_state_secondary"
            ]
          },
          {
            "name": "cf_overread_frequencies_as_infiltration",
            "role": "hard_fail",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_shows_no_infiltration_change",
              "single_cell_quantifies_infiltration",
              "frequencies_show_increased_infiltration",
              "frequencies_show_decreased_infiltration"
            ]
          },
          {
            "name": "cf_infiltration_decreases_immunosenescence",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "decreased_with_age"
            ]
          },
          {
            "name": "cf_infiltration_unchanged_scrna_overread",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "unchanged_with_age"
            ]
          },
          {
            "name": "cf_infiltration_indeterminate_unused_node",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "direction_indeterminate_from_data"
            ]
          },
          {
            "name": "cf_infiltration_mixed_direction",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "mixed_by_immune_cell_type"
            ]
          },
          {
            "name": "cf_infiltration_genotype_confusion",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_only_in_pten_deficient",
              "decreased_only_in_pten_deficient"
            ]
          },
          {
            "name": "cf_pten_infiltration_paper_text_recall",
            "role": "hard_fail",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "higher_in_pten_deficient",
              "higher_in_pten_deficient_only_in_young",
              "higher_in_pten_deficient_only_in_aged"
            ]
          },
          {
            "name": "cf_pseudoreplication_overclaim",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_increase_across_markers"
            ]
          },
          {
            "name": "cf_infiltration_strength_decrease_direction",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_decrease_across_markers",
              "significant_decrease_strongest_markers_only",
              "consistent_decrease_not_statistically_robust"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-4-codex",
      "modelName": "gpt-5.4",
      "harness": "openai-codex",
      "task": "gse297023_q6_age_immune_infiltration",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 826,
      "cost": 2.6648985,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "composition_stable_with_age",
            "role": "gate",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "frequencies_largely_stable"
            ]
          },
          {
            "name": "age_signal_is_state_not_composition",
            "role": "gate",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "frequencies_uninformative_for_infiltration",
            "role": "gate",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_cannot_establish_infiltration"
            ]
          },
          {
            "name": "infiltration_increases_with_age",
            "role": "gate",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_with_age"
            ]
          },
          {
            "name": "infiltration_strength_calibrated",
            "role": "gate",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "significant_increase_strongest_markers_only",
              "consistent_increase_not_statistically_robust"
            ]
          },
          {
            "name": "genotype_marker_fraction_lower_in_pten",
            "role": "gate",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "lower_in_pten_deficient"
            ]
          },
          {
            "name": "cf_composition_shift_reading",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "major_compositional_shift_with_age",
              "increased_immune_fraction_with_age"
            ]
          },
          {
            "name": "cf_lineage_specific_composition_claim",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "myeloid_expansion_with_age",
              "lymphoid_expansion_with_age",
              "minor_shifts_in_some_cell_types"
            ]
          },
          {
            "name": "cf_immunosenescence_depletion_prior",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "immune_depletion_with_age"
            ]
          },
          {
            "name": "cf_age_modality_compositional",
            "role": "hard_fail",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_type_composition_shift",
              "both_equally",
              "composition_dominant_state_secondary"
            ]
          },
          {
            "name": "cf_overread_frequencies_as_infiltration",
            "role": "hard_fail",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_shows_no_infiltration_change",
              "single_cell_quantifies_infiltration",
              "frequencies_show_increased_infiltration",
              "frequencies_show_decreased_infiltration"
            ]
          },
          {
            "name": "cf_infiltration_decreases_immunosenescence",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "decreased_with_age"
            ]
          },
          {
            "name": "cf_infiltration_unchanged_scrna_overread",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "unchanged_with_age"
            ]
          },
          {
            "name": "cf_infiltration_indeterminate_unused_node",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "direction_indeterminate_from_data"
            ]
          },
          {
            "name": "cf_infiltration_mixed_direction",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "mixed_by_immune_cell_type"
            ]
          },
          {
            "name": "cf_infiltration_genotype_confusion",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_only_in_pten_deficient",
              "decreased_only_in_pten_deficient"
            ]
          },
          {
            "name": "cf_pten_infiltration_paper_text_recall",
            "role": "hard_fail",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "higher_in_pten_deficient",
              "higher_in_pten_deficient_only_in_young",
              "higher_in_pten_deficient_only_in_aged"
            ]
          },
          {
            "name": "cf_pseudoreplication_overclaim",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_increase_across_markers"
            ]
          },
          {
            "name": "cf_infiltration_strength_decrease_direction",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_decrease_across_markers",
              "significant_decrease_strongest_markers_only",
              "consistent_decrease_not_statistically_robust"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-4-codex",
      "modelName": "gpt-5.4",
      "harness": "openai-codex",
      "task": "gse297023_q6_age_immune_infiltration",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 370,
      "cost": 1.1783525,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "composition_stable_with_age",
            "role": "gate",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "frequencies_largely_stable"
            ]
          },
          {
            "name": "age_signal_is_state_not_composition",
            "role": "gate",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "frequencies_uninformative_for_infiltration",
            "role": "gate",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_cannot_establish_infiltration"
            ]
          },
          {
            "name": "infiltration_increases_with_age",
            "role": "gate",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_with_age"
            ]
          },
          {
            "name": "infiltration_strength_calibrated",
            "role": "gate",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "significant_increase_strongest_markers_only",
              "consistent_increase_not_statistically_robust"
            ]
          },
          {
            "name": "genotype_marker_fraction_lower_in_pten",
            "role": "gate",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "lower_in_pten_deficient"
            ]
          },
          {
            "name": "cf_composition_shift_reading",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "major_compositional_shift_with_age",
              "increased_immune_fraction_with_age"
            ]
          },
          {
            "name": "cf_lineage_specific_composition_claim",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "myeloid_expansion_with_age",
              "lymphoid_expansion_with_age",
              "minor_shifts_in_some_cell_types"
            ]
          },
          {
            "name": "cf_immunosenescence_depletion_prior",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "immune_depletion_with_age"
            ]
          },
          {
            "name": "cf_age_modality_compositional",
            "role": "hard_fail",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_type_composition_shift",
              "both_equally",
              "composition_dominant_state_secondary"
            ]
          },
          {
            "name": "cf_overread_frequencies_as_infiltration",
            "role": "hard_fail",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_shows_no_infiltration_change",
              "single_cell_quantifies_infiltration",
              "frequencies_show_increased_infiltration",
              "frequencies_show_decreased_infiltration"
            ]
          },
          {
            "name": "cf_infiltration_decreases_immunosenescence",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "decreased_with_age"
            ]
          },
          {
            "name": "cf_infiltration_unchanged_scrna_overread",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "unchanged_with_age"
            ]
          },
          {
            "name": "cf_infiltration_indeterminate_unused_node",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "direction_indeterminate_from_data"
            ]
          },
          {
            "name": "cf_infiltration_mixed_direction",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "mixed_by_immune_cell_type"
            ]
          },
          {
            "name": "cf_infiltration_genotype_confusion",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_only_in_pten_deficient",
              "decreased_only_in_pten_deficient"
            ]
          },
          {
            "name": "cf_pten_infiltration_paper_text_recall",
            "role": "hard_fail",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "higher_in_pten_deficient",
              "higher_in_pten_deficient_only_in_young",
              "higher_in_pten_deficient_only_in_aged"
            ]
          },
          {
            "name": "cf_pseudoreplication_overclaim",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_increase_across_markers"
            ]
          },
          {
            "name": "cf_infiltration_strength_decrease_direction",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_decrease_across_markers",
              "significant_decrease_strongest_markers_only",
              "consistent_decrease_not_statistically_robust"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-4-codex",
      "modelName": "gpt-5.4",
      "harness": "openai-codex",
      "task": "gse297023_q6_age_immune_infiltration",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 584,
      "cost": 1.2517845,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "composition_stable_with_age",
            "role": "gate",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "frequencies_largely_stable"
            ]
          },
          {
            "name": "age_signal_is_state_not_composition",
            "role": "gate",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "frequencies_uninformative_for_infiltration",
            "role": "gate",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_cannot_establish_infiltration"
            ]
          },
          {
            "name": "infiltration_increases_with_age",
            "role": "gate",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_with_age"
            ]
          },
          {
            "name": "infiltration_strength_calibrated",
            "role": "gate",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "significant_increase_strongest_markers_only",
              "consistent_increase_not_statistically_robust"
            ]
          },
          {
            "name": "genotype_marker_fraction_lower_in_pten",
            "role": "gate",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "lower_in_pten_deficient"
            ]
          },
          {
            "name": "cf_composition_shift_reading",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "major_compositional_shift_with_age",
              "increased_immune_fraction_with_age"
            ]
          },
          {
            "name": "cf_lineage_specific_composition_claim",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "myeloid_expansion_with_age",
              "lymphoid_expansion_with_age",
              "minor_shifts_in_some_cell_types"
            ]
          },
          {
            "name": "cf_immunosenescence_depletion_prior",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "immune_depletion_with_age"
            ]
          },
          {
            "name": "cf_age_modality_compositional",
            "role": "hard_fail",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_type_composition_shift",
              "both_equally",
              "composition_dominant_state_secondary"
            ]
          },
          {
            "name": "cf_overread_frequencies_as_infiltration",
            "role": "hard_fail",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_shows_no_infiltration_change",
              "single_cell_quantifies_infiltration",
              "frequencies_show_increased_infiltration",
              "frequencies_show_decreased_infiltration"
            ]
          },
          {
            "name": "cf_infiltration_decreases_immunosenescence",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "decreased_with_age"
            ]
          },
          {
            "name": "cf_infiltration_unchanged_scrna_overread",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "unchanged_with_age"
            ]
          },
          {
            "name": "cf_infiltration_indeterminate_unused_node",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "direction_indeterminate_from_data"
            ]
          },
          {
            "name": "cf_infiltration_mixed_direction",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "mixed_by_immune_cell_type"
            ]
          },
          {
            "name": "cf_infiltration_genotype_confusion",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_only_in_pten_deficient",
              "decreased_only_in_pten_deficient"
            ]
          },
          {
            "name": "cf_pten_infiltration_paper_text_recall",
            "role": "hard_fail",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "higher_in_pten_deficient",
              "higher_in_pten_deficient_only_in_young",
              "higher_in_pten_deficient_only_in_aged"
            ]
          },
          {
            "name": "cf_pseudoreplication_overclaim",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_increase_across_markers"
            ]
          },
          {
            "name": "cf_infiltration_strength_decrease_direction",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_decrease_across_markers",
              "significant_decrease_strongest_markers_only",
              "consistent_decrease_not_statistically_robust"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-4-pi",
      "modelName": "gpt-5.4",
      "harness": "pi",
      "task": "gse297023_q6_age_immune_infiltration",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 1,
      "passed": true,
      "score": null,
      "durationS": 1210,
      "cost": 1.0869060000000004,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "composition_stable_with_age",
            "role": "gate",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "frequencies_largely_stable"
            ]
          },
          {
            "name": "age_signal_is_state_not_composition",
            "role": "gate",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "frequencies_uninformative_for_infiltration",
            "role": "gate",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_cannot_establish_infiltration"
            ]
          },
          {
            "name": "infiltration_increases_with_age",
            "role": "gate",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_with_age"
            ]
          },
          {
            "name": "infiltration_strength_calibrated",
            "role": "gate",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "significant_increase_strongest_markers_only",
              "consistent_increase_not_statistically_robust"
            ]
          },
          {
            "name": "genotype_marker_fraction_lower_in_pten",
            "role": "gate",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "lower_in_pten_deficient"
            ]
          },
          {
            "name": "cf_composition_shift_reading",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "major_compositional_shift_with_age",
              "increased_immune_fraction_with_age"
            ]
          },
          {
            "name": "cf_lineage_specific_composition_claim",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "myeloid_expansion_with_age",
              "lymphoid_expansion_with_age",
              "minor_shifts_in_some_cell_types"
            ]
          },
          {
            "name": "cf_immunosenescence_depletion_prior",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "immune_depletion_with_age"
            ]
          },
          {
            "name": "cf_age_modality_compositional",
            "role": "hard_fail",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_type_composition_shift",
              "both_equally",
              "composition_dominant_state_secondary"
            ]
          },
          {
            "name": "cf_overread_frequencies_as_infiltration",
            "role": "hard_fail",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_shows_no_infiltration_change",
              "single_cell_quantifies_infiltration",
              "frequencies_show_increased_infiltration",
              "frequencies_show_decreased_infiltration"
            ]
          },
          {
            "name": "cf_infiltration_decreases_immunosenescence",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "decreased_with_age"
            ]
          },
          {
            "name": "cf_infiltration_unchanged_scrna_overread",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "unchanged_with_age"
            ]
          },
          {
            "name": "cf_infiltration_indeterminate_unused_node",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "direction_indeterminate_from_data"
            ]
          },
          {
            "name": "cf_infiltration_mixed_direction",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "mixed_by_immune_cell_type"
            ]
          },
          {
            "name": "cf_infiltration_genotype_confusion",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_only_in_pten_deficient",
              "decreased_only_in_pten_deficient"
            ]
          },
          {
            "name": "cf_pten_infiltration_paper_text_recall",
            "role": "hard_fail",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "higher_in_pten_deficient",
              "higher_in_pten_deficient_only_in_young",
              "higher_in_pten_deficient_only_in_aged"
            ]
          },
          {
            "name": "cf_pseudoreplication_overclaim",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_increase_across_markers"
            ]
          },
          {
            "name": "cf_infiltration_strength_decrease_direction",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_decrease_across_markers",
              "significant_decrease_strongest_markers_only",
              "consistent_decrease_not_statistically_robust"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-4-pi",
      "modelName": "gpt-5.4",
      "harness": "pi",
      "task": "gse297023_q6_age_immune_infiltration",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 1827,
      "cost": 1.2082125,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "composition_stable_with_age",
            "role": "gate",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "frequencies_largely_stable"
            ]
          },
          {
            "name": "age_signal_is_state_not_composition",
            "role": "gate",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "frequencies_uninformative_for_infiltration",
            "role": "gate",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_cannot_establish_infiltration"
            ]
          },
          {
            "name": "infiltration_increases_with_age",
            "role": "gate",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_with_age"
            ]
          },
          {
            "name": "infiltration_strength_calibrated",
            "role": "gate",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "significant_increase_strongest_markers_only",
              "consistent_increase_not_statistically_robust"
            ]
          },
          {
            "name": "genotype_marker_fraction_lower_in_pten",
            "role": "gate",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "lower_in_pten_deficient"
            ]
          },
          {
            "name": "cf_composition_shift_reading",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "major_compositional_shift_with_age",
              "increased_immune_fraction_with_age"
            ]
          },
          {
            "name": "cf_lineage_specific_composition_claim",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "myeloid_expansion_with_age",
              "lymphoid_expansion_with_age",
              "minor_shifts_in_some_cell_types"
            ]
          },
          {
            "name": "cf_immunosenescence_depletion_prior",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "immune_depletion_with_age"
            ]
          },
          {
            "name": "cf_age_modality_compositional",
            "role": "hard_fail",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_type_composition_shift",
              "both_equally",
              "composition_dominant_state_secondary"
            ]
          },
          {
            "name": "cf_overread_frequencies_as_infiltration",
            "role": "hard_fail",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_shows_no_infiltration_change",
              "single_cell_quantifies_infiltration",
              "frequencies_show_increased_infiltration",
              "frequencies_show_decreased_infiltration"
            ]
          },
          {
            "name": "cf_infiltration_decreases_immunosenescence",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "decreased_with_age"
            ]
          },
          {
            "name": "cf_infiltration_unchanged_scrna_overread",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "unchanged_with_age"
            ]
          },
          {
            "name": "cf_infiltration_indeterminate_unused_node",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "direction_indeterminate_from_data"
            ]
          },
          {
            "name": "cf_infiltration_mixed_direction",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "mixed_by_immune_cell_type"
            ]
          },
          {
            "name": "cf_infiltration_genotype_confusion",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_only_in_pten_deficient",
              "decreased_only_in_pten_deficient"
            ]
          },
          {
            "name": "cf_pten_infiltration_paper_text_recall",
            "role": "hard_fail",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "higher_in_pten_deficient",
              "higher_in_pten_deficient_only_in_young",
              "higher_in_pten_deficient_only_in_aged"
            ]
          },
          {
            "name": "cf_pseudoreplication_overclaim",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_increase_across_markers"
            ]
          },
          {
            "name": "cf_infiltration_strength_decrease_direction",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_decrease_across_markers",
              "significant_decrease_strongest_markers_only",
              "consistent_decrease_not_statistically_robust"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-4-pi",
      "modelName": "gpt-5.4",
      "harness": "pi",
      "task": "gse297023_q6_age_immune_infiltration",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1567,
      "cost": 1.2178094999999998,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "composition_stable_with_age",
            "role": "gate",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "frequencies_largely_stable"
            ]
          },
          {
            "name": "age_signal_is_state_not_composition",
            "role": "gate",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "frequencies_uninformative_for_infiltration",
            "role": "gate",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_cannot_establish_infiltration"
            ]
          },
          {
            "name": "infiltration_increases_with_age",
            "role": "gate",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_with_age"
            ]
          },
          {
            "name": "infiltration_strength_calibrated",
            "role": "gate",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "significant_increase_strongest_markers_only",
              "consistent_increase_not_statistically_robust"
            ]
          },
          {
            "name": "genotype_marker_fraction_lower_in_pten",
            "role": "gate",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "lower_in_pten_deficient"
            ]
          },
          {
            "name": "cf_composition_shift_reading",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "major_compositional_shift_with_age",
              "increased_immune_fraction_with_age"
            ]
          },
          {
            "name": "cf_lineage_specific_composition_claim",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "myeloid_expansion_with_age",
              "lymphoid_expansion_with_age",
              "minor_shifts_in_some_cell_types"
            ]
          },
          {
            "name": "cf_immunosenescence_depletion_prior",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "immune_depletion_with_age"
            ]
          },
          {
            "name": "cf_age_modality_compositional",
            "role": "hard_fail",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_type_composition_shift",
              "both_equally",
              "composition_dominant_state_secondary"
            ]
          },
          {
            "name": "cf_overread_frequencies_as_infiltration",
            "role": "hard_fail",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_shows_no_infiltration_change",
              "single_cell_quantifies_infiltration",
              "frequencies_show_increased_infiltration",
              "frequencies_show_decreased_infiltration"
            ]
          },
          {
            "name": "cf_infiltration_decreases_immunosenescence",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "decreased_with_age"
            ]
          },
          {
            "name": "cf_infiltration_unchanged_scrna_overread",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "unchanged_with_age"
            ]
          },
          {
            "name": "cf_infiltration_indeterminate_unused_node",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "direction_indeterminate_from_data"
            ]
          },
          {
            "name": "cf_infiltration_mixed_direction",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "mixed_by_immune_cell_type"
            ]
          },
          {
            "name": "cf_infiltration_genotype_confusion",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_only_in_pten_deficient",
              "decreased_only_in_pten_deficient"
            ]
          },
          {
            "name": "cf_pten_infiltration_paper_text_recall",
            "role": "hard_fail",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "higher_in_pten_deficient",
              "higher_in_pten_deficient_only_in_young",
              "higher_in_pten_deficient_only_in_aged"
            ]
          },
          {
            "name": "cf_pseudoreplication_overclaim",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_increase_across_markers"
            ]
          },
          {
            "name": "cf_infiltration_strength_decrease_direction",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_decrease_across_markers",
              "significant_decrease_strongest_markers_only",
              "consistent_decrease_not_statistically_robust"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-5-codex",
      "modelName": "gpt-5.5",
      "harness": "openai-codex",
      "task": "gse297023_q6_age_immune_infiltration",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 722,
      "cost": 2.420468,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "composition_stable_with_age",
            "role": "gate",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "frequencies_largely_stable"
            ]
          },
          {
            "name": "age_signal_is_state_not_composition",
            "role": "gate",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "frequencies_uninformative_for_infiltration",
            "role": "gate",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_cannot_establish_infiltration"
            ]
          },
          {
            "name": "infiltration_increases_with_age",
            "role": "gate",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_with_age"
            ]
          },
          {
            "name": "infiltration_strength_calibrated",
            "role": "gate",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "significant_increase_strongest_markers_only",
              "consistent_increase_not_statistically_robust"
            ]
          },
          {
            "name": "genotype_marker_fraction_lower_in_pten",
            "role": "gate",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "lower_in_pten_deficient"
            ]
          },
          {
            "name": "cf_composition_shift_reading",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "major_compositional_shift_with_age",
              "increased_immune_fraction_with_age"
            ]
          },
          {
            "name": "cf_lineage_specific_composition_claim",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "myeloid_expansion_with_age",
              "lymphoid_expansion_with_age",
              "minor_shifts_in_some_cell_types"
            ]
          },
          {
            "name": "cf_immunosenescence_depletion_prior",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "immune_depletion_with_age"
            ]
          },
          {
            "name": "cf_age_modality_compositional",
            "role": "hard_fail",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_type_composition_shift",
              "both_equally",
              "composition_dominant_state_secondary"
            ]
          },
          {
            "name": "cf_overread_frequencies_as_infiltration",
            "role": "hard_fail",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_shows_no_infiltration_change",
              "single_cell_quantifies_infiltration",
              "frequencies_show_increased_infiltration",
              "frequencies_show_decreased_infiltration"
            ]
          },
          {
            "name": "cf_infiltration_decreases_immunosenescence",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "decreased_with_age"
            ]
          },
          {
            "name": "cf_infiltration_unchanged_scrna_overread",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "unchanged_with_age"
            ]
          },
          {
            "name": "cf_infiltration_indeterminate_unused_node",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "direction_indeterminate_from_data"
            ]
          },
          {
            "name": "cf_infiltration_mixed_direction",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "mixed_by_immune_cell_type"
            ]
          },
          {
            "name": "cf_infiltration_genotype_confusion",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_only_in_pten_deficient",
              "decreased_only_in_pten_deficient"
            ]
          },
          {
            "name": "cf_pten_infiltration_paper_text_recall",
            "role": "hard_fail",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "higher_in_pten_deficient",
              "higher_in_pten_deficient_only_in_young",
              "higher_in_pten_deficient_only_in_aged"
            ]
          },
          {
            "name": "cf_pseudoreplication_overclaim",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_increase_across_markers"
            ]
          },
          {
            "name": "cf_infiltration_strength_decrease_direction",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_decrease_across_markers",
              "significant_decrease_strongest_markers_only",
              "consistent_decrease_not_statistically_robust"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-5-codex",
      "modelName": "gpt-5.5",
      "harness": "openai-codex",
      "task": "gse297023_q6_age_immune_infiltration",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 363,
      "cost": 1.0402040000000001,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "composition_stable_with_age",
            "role": "gate",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "frequencies_largely_stable"
            ]
          },
          {
            "name": "age_signal_is_state_not_composition",
            "role": "gate",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "frequencies_uninformative_for_infiltration",
            "role": "gate",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_cannot_establish_infiltration"
            ]
          },
          {
            "name": "infiltration_increases_with_age",
            "role": "gate",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_with_age"
            ]
          },
          {
            "name": "infiltration_strength_calibrated",
            "role": "gate",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "significant_increase_strongest_markers_only",
              "consistent_increase_not_statistically_robust"
            ]
          },
          {
            "name": "genotype_marker_fraction_lower_in_pten",
            "role": "gate",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "lower_in_pten_deficient"
            ]
          },
          {
            "name": "cf_composition_shift_reading",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "major_compositional_shift_with_age",
              "increased_immune_fraction_with_age"
            ]
          },
          {
            "name": "cf_lineage_specific_composition_claim",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "myeloid_expansion_with_age",
              "lymphoid_expansion_with_age",
              "minor_shifts_in_some_cell_types"
            ]
          },
          {
            "name": "cf_immunosenescence_depletion_prior",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "immune_depletion_with_age"
            ]
          },
          {
            "name": "cf_age_modality_compositional",
            "role": "hard_fail",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_type_composition_shift",
              "both_equally",
              "composition_dominant_state_secondary"
            ]
          },
          {
            "name": "cf_overread_frequencies_as_infiltration",
            "role": "hard_fail",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_shows_no_infiltration_change",
              "single_cell_quantifies_infiltration",
              "frequencies_show_increased_infiltration",
              "frequencies_show_decreased_infiltration"
            ]
          },
          {
            "name": "cf_infiltration_decreases_immunosenescence",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "decreased_with_age"
            ]
          },
          {
            "name": "cf_infiltration_unchanged_scrna_overread",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "unchanged_with_age"
            ]
          },
          {
            "name": "cf_infiltration_indeterminate_unused_node",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "direction_indeterminate_from_data"
            ]
          },
          {
            "name": "cf_infiltration_mixed_direction",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "mixed_by_immune_cell_type"
            ]
          },
          {
            "name": "cf_infiltration_genotype_confusion",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_only_in_pten_deficient",
              "decreased_only_in_pten_deficient"
            ]
          },
          {
            "name": "cf_pten_infiltration_paper_text_recall",
            "role": "hard_fail",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "higher_in_pten_deficient",
              "higher_in_pten_deficient_only_in_young",
              "higher_in_pten_deficient_only_in_aged"
            ]
          },
          {
            "name": "cf_pseudoreplication_overclaim",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_increase_across_markers"
            ]
          },
          {
            "name": "cf_infiltration_strength_decrease_direction",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_decrease_across_markers",
              "significant_decrease_strongest_markers_only",
              "consistent_decrease_not_statistically_robust"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-5-codex",
      "modelName": "gpt-5.5",
      "harness": "openai-codex",
      "task": "gse297023_q6_age_immune_infiltration",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 526,
      "cost": 1.4977390000000002,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "composition_stable_with_age",
            "role": "gate",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "frequencies_largely_stable"
            ]
          },
          {
            "name": "age_signal_is_state_not_composition",
            "role": "gate",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "frequencies_uninformative_for_infiltration",
            "role": "gate",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_cannot_establish_infiltration"
            ]
          },
          {
            "name": "infiltration_increases_with_age",
            "role": "gate",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_with_age"
            ]
          },
          {
            "name": "infiltration_strength_calibrated",
            "role": "gate",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "significant_increase_strongest_markers_only",
              "consistent_increase_not_statistically_robust"
            ]
          },
          {
            "name": "genotype_marker_fraction_lower_in_pten",
            "role": "gate",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "lower_in_pten_deficient"
            ]
          },
          {
            "name": "cf_composition_shift_reading",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "major_compositional_shift_with_age",
              "increased_immune_fraction_with_age"
            ]
          },
          {
            "name": "cf_lineage_specific_composition_claim",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "myeloid_expansion_with_age",
              "lymphoid_expansion_with_age",
              "minor_shifts_in_some_cell_types"
            ]
          },
          {
            "name": "cf_immunosenescence_depletion_prior",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "immune_depletion_with_age"
            ]
          },
          {
            "name": "cf_age_modality_compositional",
            "role": "hard_fail",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_type_composition_shift",
              "both_equally",
              "composition_dominant_state_secondary"
            ]
          },
          {
            "name": "cf_overread_frequencies_as_infiltration",
            "role": "hard_fail",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_shows_no_infiltration_change",
              "single_cell_quantifies_infiltration",
              "frequencies_show_increased_infiltration",
              "frequencies_show_decreased_infiltration"
            ]
          },
          {
            "name": "cf_infiltration_decreases_immunosenescence",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "decreased_with_age"
            ]
          },
          {
            "name": "cf_infiltration_unchanged_scrna_overread",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "unchanged_with_age"
            ]
          },
          {
            "name": "cf_infiltration_indeterminate_unused_node",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "direction_indeterminate_from_data"
            ]
          },
          {
            "name": "cf_infiltration_mixed_direction",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "mixed_by_immune_cell_type"
            ]
          },
          {
            "name": "cf_infiltration_genotype_confusion",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_only_in_pten_deficient",
              "decreased_only_in_pten_deficient"
            ]
          },
          {
            "name": "cf_pten_infiltration_paper_text_recall",
            "role": "hard_fail",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "higher_in_pten_deficient",
              "higher_in_pten_deficient_only_in_young",
              "higher_in_pten_deficient_only_in_aged"
            ]
          },
          {
            "name": "cf_pseudoreplication_overclaim",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_increase_across_markers"
            ]
          },
          {
            "name": "cf_infiltration_strength_decrease_direction",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_decrease_across_markers",
              "significant_decrease_strongest_markers_only",
              "consistent_decrease_not_statistically_robust"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-5-pi",
      "modelName": "gpt-5.5",
      "harness": "pi",
      "task": "gse297023_q6_age_immune_infiltration",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 647,
      "cost": 1.7866410000000001,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "composition_stable_with_age",
            "role": "gate",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "frequencies_largely_stable"
            ]
          },
          {
            "name": "age_signal_is_state_not_composition",
            "role": "gate",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "frequencies_uninformative_for_infiltration",
            "role": "gate",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_cannot_establish_infiltration"
            ]
          },
          {
            "name": "infiltration_increases_with_age",
            "role": "gate",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_with_age"
            ]
          },
          {
            "name": "infiltration_strength_calibrated",
            "role": "gate",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "significant_increase_strongest_markers_only",
              "consistent_increase_not_statistically_robust"
            ]
          },
          {
            "name": "genotype_marker_fraction_lower_in_pten",
            "role": "gate",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "lower_in_pten_deficient"
            ]
          },
          {
            "name": "cf_composition_shift_reading",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "major_compositional_shift_with_age",
              "increased_immune_fraction_with_age"
            ]
          },
          {
            "name": "cf_lineage_specific_composition_claim",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "myeloid_expansion_with_age",
              "lymphoid_expansion_with_age",
              "minor_shifts_in_some_cell_types"
            ]
          },
          {
            "name": "cf_immunosenescence_depletion_prior",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "immune_depletion_with_age"
            ]
          },
          {
            "name": "cf_age_modality_compositional",
            "role": "hard_fail",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_type_composition_shift",
              "both_equally",
              "composition_dominant_state_secondary"
            ]
          },
          {
            "name": "cf_overread_frequencies_as_infiltration",
            "role": "hard_fail",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_shows_no_infiltration_change",
              "single_cell_quantifies_infiltration",
              "frequencies_show_increased_infiltration",
              "frequencies_show_decreased_infiltration"
            ]
          },
          {
            "name": "cf_infiltration_decreases_immunosenescence",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "decreased_with_age"
            ]
          },
          {
            "name": "cf_infiltration_unchanged_scrna_overread",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "unchanged_with_age"
            ]
          },
          {
            "name": "cf_infiltration_indeterminate_unused_node",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "direction_indeterminate_from_data"
            ]
          },
          {
            "name": "cf_infiltration_mixed_direction",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "mixed_by_immune_cell_type"
            ]
          },
          {
            "name": "cf_infiltration_genotype_confusion",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_only_in_pten_deficient",
              "decreased_only_in_pten_deficient"
            ]
          },
          {
            "name": "cf_pten_infiltration_paper_text_recall",
            "role": "hard_fail",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "higher_in_pten_deficient",
              "higher_in_pten_deficient_only_in_young",
              "higher_in_pten_deficient_only_in_aged"
            ]
          },
          {
            "name": "cf_pseudoreplication_overclaim",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_increase_across_markers"
            ]
          },
          {
            "name": "cf_infiltration_strength_decrease_direction",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_decrease_across_markers",
              "significant_decrease_strongest_markers_only",
              "consistent_decrease_not_statistically_robust"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-5-pi",
      "modelName": "gpt-5.5",
      "harness": "pi",
      "task": "gse297023_q6_age_immune_infiltration",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 2,
      "passed": true,
      "score": null,
      "durationS": 754,
      "cost": 1.4776410000000002,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "composition_stable_with_age",
            "role": "gate",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "frequencies_largely_stable"
            ]
          },
          {
            "name": "age_signal_is_state_not_composition",
            "role": "gate",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "frequencies_uninformative_for_infiltration",
            "role": "gate",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_cannot_establish_infiltration"
            ]
          },
          {
            "name": "infiltration_increases_with_age",
            "role": "gate",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_with_age"
            ]
          },
          {
            "name": "infiltration_strength_calibrated",
            "role": "gate",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "significant_increase_strongest_markers_only",
              "consistent_increase_not_statistically_robust"
            ]
          },
          {
            "name": "genotype_marker_fraction_lower_in_pten",
            "role": "gate",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "lower_in_pten_deficient"
            ]
          },
          {
            "name": "cf_composition_shift_reading",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "major_compositional_shift_with_age",
              "increased_immune_fraction_with_age"
            ]
          },
          {
            "name": "cf_lineage_specific_composition_claim",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "myeloid_expansion_with_age",
              "lymphoid_expansion_with_age",
              "minor_shifts_in_some_cell_types"
            ]
          },
          {
            "name": "cf_immunosenescence_depletion_prior",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "immune_depletion_with_age"
            ]
          },
          {
            "name": "cf_age_modality_compositional",
            "role": "hard_fail",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_type_composition_shift",
              "both_equally",
              "composition_dominant_state_secondary"
            ]
          },
          {
            "name": "cf_overread_frequencies_as_infiltration",
            "role": "hard_fail",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_shows_no_infiltration_change",
              "single_cell_quantifies_infiltration",
              "frequencies_show_increased_infiltration",
              "frequencies_show_decreased_infiltration"
            ]
          },
          {
            "name": "cf_infiltration_decreases_immunosenescence",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "decreased_with_age"
            ]
          },
          {
            "name": "cf_infiltration_unchanged_scrna_overread",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "unchanged_with_age"
            ]
          },
          {
            "name": "cf_infiltration_indeterminate_unused_node",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "direction_indeterminate_from_data"
            ]
          },
          {
            "name": "cf_infiltration_mixed_direction",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "mixed_by_immune_cell_type"
            ]
          },
          {
            "name": "cf_infiltration_genotype_confusion",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_only_in_pten_deficient",
              "decreased_only_in_pten_deficient"
            ]
          },
          {
            "name": "cf_pten_infiltration_paper_text_recall",
            "role": "hard_fail",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "higher_in_pten_deficient",
              "higher_in_pten_deficient_only_in_young",
              "higher_in_pten_deficient_only_in_aged"
            ]
          },
          {
            "name": "cf_pseudoreplication_overclaim",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_increase_across_markers"
            ]
          },
          {
            "name": "cf_infiltration_strength_decrease_direction",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_decrease_across_markers",
              "significant_decrease_strongest_markers_only",
              "consistent_decrease_not_statistically_robust"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-5-pi",
      "modelName": "gpt-5.5",
      "harness": "pi",
      "task": "gse297023_q6_age_immune_infiltration",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1069,
      "cost": 1.4370359999999998,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "composition_stable_with_age",
            "role": "gate",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "frequencies_largely_stable"
            ]
          },
          {
            "name": "age_signal_is_state_not_composition",
            "role": "gate",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "frequencies_uninformative_for_infiltration",
            "role": "gate",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_cannot_establish_infiltration"
            ]
          },
          {
            "name": "infiltration_increases_with_age",
            "role": "gate",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_with_age"
            ]
          },
          {
            "name": "infiltration_strength_calibrated",
            "role": "gate",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "significant_increase_strongest_markers_only",
              "consistent_increase_not_statistically_robust"
            ]
          },
          {
            "name": "genotype_marker_fraction_lower_in_pten",
            "role": "gate",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "lower_in_pten_deficient"
            ]
          },
          {
            "name": "cf_composition_shift_reading",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "major_compositional_shift_with_age",
              "increased_immune_fraction_with_age"
            ]
          },
          {
            "name": "cf_lineage_specific_composition_claim",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "myeloid_expansion_with_age",
              "lymphoid_expansion_with_age",
              "minor_shifts_in_some_cell_types"
            ]
          },
          {
            "name": "cf_immunosenescence_depletion_prior",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "immune_depletion_with_age"
            ]
          },
          {
            "name": "cf_age_modality_compositional",
            "role": "hard_fail",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_type_composition_shift",
              "both_equally",
              "composition_dominant_state_secondary"
            ]
          },
          {
            "name": "cf_overread_frequencies_as_infiltration",
            "role": "hard_fail",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_shows_no_infiltration_change",
              "single_cell_quantifies_infiltration",
              "frequencies_show_increased_infiltration",
              "frequencies_show_decreased_infiltration"
            ]
          },
          {
            "name": "cf_infiltration_decreases_immunosenescence",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "decreased_with_age"
            ]
          },
          {
            "name": "cf_infiltration_unchanged_scrna_overread",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "unchanged_with_age"
            ]
          },
          {
            "name": "cf_infiltration_indeterminate_unused_node",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "direction_indeterminate_from_data"
            ]
          },
          {
            "name": "cf_infiltration_mixed_direction",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "mixed_by_immune_cell_type"
            ]
          },
          {
            "name": "cf_infiltration_genotype_confusion",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_only_in_pten_deficient",
              "decreased_only_in_pten_deficient"
            ]
          },
          {
            "name": "cf_pten_infiltration_paper_text_recall",
            "role": "hard_fail",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "higher_in_pten_deficient",
              "higher_in_pten_deficient_only_in_young",
              "higher_in_pten_deficient_only_in_aged"
            ]
          },
          {
            "name": "cf_pseudoreplication_overclaim",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_increase_across_markers"
            ]
          },
          {
            "name": "cf_infiltration_strength_decrease_direction",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_decrease_across_markers",
              "significant_decrease_strongest_markers_only",
              "consistent_decrease_not_statistically_robust"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-20-beta-reasoning-pi",
      "modelName": "grok-4.20-beta-0309-reasoning",
      "harness": "pi",
      "task": "gse297023_q6_age_immune_infiltration",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 102,
      "cost": 0.11751565,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "composition_stable_with_age",
            "role": "gate",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "frequencies_largely_stable"
            ]
          },
          {
            "name": "age_signal_is_state_not_composition",
            "role": "gate",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "frequencies_uninformative_for_infiltration",
            "role": "gate",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_cannot_establish_infiltration"
            ]
          },
          {
            "name": "infiltration_increases_with_age",
            "role": "gate",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_with_age"
            ]
          },
          {
            "name": "infiltration_strength_calibrated",
            "role": "gate",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "significant_increase_strongest_markers_only",
              "consistent_increase_not_statistically_robust"
            ]
          },
          {
            "name": "genotype_marker_fraction_lower_in_pten",
            "role": "gate",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "lower_in_pten_deficient"
            ]
          },
          {
            "name": "cf_composition_shift_reading",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "major_compositional_shift_with_age",
              "increased_immune_fraction_with_age"
            ]
          },
          {
            "name": "cf_lineage_specific_composition_claim",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "myeloid_expansion_with_age",
              "lymphoid_expansion_with_age",
              "minor_shifts_in_some_cell_types"
            ]
          },
          {
            "name": "cf_immunosenescence_depletion_prior",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "immune_depletion_with_age"
            ]
          },
          {
            "name": "cf_age_modality_compositional",
            "role": "hard_fail",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_type_composition_shift",
              "both_equally",
              "composition_dominant_state_secondary"
            ]
          },
          {
            "name": "cf_overread_frequencies_as_infiltration",
            "role": "hard_fail",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_shows_no_infiltration_change",
              "single_cell_quantifies_infiltration",
              "frequencies_show_increased_infiltration",
              "frequencies_show_decreased_infiltration"
            ]
          },
          {
            "name": "cf_infiltration_decreases_immunosenescence",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "decreased_with_age"
            ]
          },
          {
            "name": "cf_infiltration_unchanged_scrna_overread",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "unchanged_with_age"
            ]
          },
          {
            "name": "cf_infiltration_indeterminate_unused_node",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "direction_indeterminate_from_data"
            ]
          },
          {
            "name": "cf_infiltration_mixed_direction",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "mixed_by_immune_cell_type"
            ]
          },
          {
            "name": "cf_infiltration_genotype_confusion",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_only_in_pten_deficient",
              "decreased_only_in_pten_deficient"
            ]
          },
          {
            "name": "cf_pten_infiltration_paper_text_recall",
            "role": "hard_fail",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "higher_in_pten_deficient",
              "higher_in_pten_deficient_only_in_young",
              "higher_in_pten_deficient_only_in_aged"
            ]
          },
          {
            "name": "cf_pseudoreplication_overclaim",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_increase_across_markers"
            ]
          },
          {
            "name": "cf_infiltration_strength_decrease_direction",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_decrease_across_markers",
              "significant_decrease_strongest_markers_only",
              "consistent_decrease_not_statistically_robust"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-20-beta-reasoning-pi",
      "modelName": "grok-4.20-beta-0309-reasoning",
      "harness": "pi",
      "task": "gse297023_q6_age_immune_infiltration",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 760,
      "cost": 0.08448100000000001,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "composition_stable_with_age",
            "role": "gate",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "frequencies_largely_stable"
            ]
          },
          {
            "name": "age_signal_is_state_not_composition",
            "role": "gate",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "frequencies_uninformative_for_infiltration",
            "role": "gate",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_cannot_establish_infiltration"
            ]
          },
          {
            "name": "infiltration_increases_with_age",
            "role": "gate",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_with_age"
            ]
          },
          {
            "name": "infiltration_strength_calibrated",
            "role": "gate",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "significant_increase_strongest_markers_only",
              "consistent_increase_not_statistically_robust"
            ]
          },
          {
            "name": "genotype_marker_fraction_lower_in_pten",
            "role": "gate",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "lower_in_pten_deficient"
            ]
          },
          {
            "name": "cf_composition_shift_reading",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "major_compositional_shift_with_age",
              "increased_immune_fraction_with_age"
            ]
          },
          {
            "name": "cf_lineage_specific_composition_claim",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "myeloid_expansion_with_age",
              "lymphoid_expansion_with_age",
              "minor_shifts_in_some_cell_types"
            ]
          },
          {
            "name": "cf_immunosenescence_depletion_prior",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "immune_depletion_with_age"
            ]
          },
          {
            "name": "cf_age_modality_compositional",
            "role": "hard_fail",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_type_composition_shift",
              "both_equally",
              "composition_dominant_state_secondary"
            ]
          },
          {
            "name": "cf_overread_frequencies_as_infiltration",
            "role": "hard_fail",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_shows_no_infiltration_change",
              "single_cell_quantifies_infiltration",
              "frequencies_show_increased_infiltration",
              "frequencies_show_decreased_infiltration"
            ]
          },
          {
            "name": "cf_infiltration_decreases_immunosenescence",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "decreased_with_age"
            ]
          },
          {
            "name": "cf_infiltration_unchanged_scrna_overread",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "unchanged_with_age"
            ]
          },
          {
            "name": "cf_infiltration_indeterminate_unused_node",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "direction_indeterminate_from_data"
            ]
          },
          {
            "name": "cf_infiltration_mixed_direction",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "mixed_by_immune_cell_type"
            ]
          },
          {
            "name": "cf_infiltration_genotype_confusion",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_only_in_pten_deficient",
              "decreased_only_in_pten_deficient"
            ]
          },
          {
            "name": "cf_pten_infiltration_paper_text_recall",
            "role": "hard_fail",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "higher_in_pten_deficient",
              "higher_in_pten_deficient_only_in_young",
              "higher_in_pten_deficient_only_in_aged"
            ]
          },
          {
            "name": "cf_pseudoreplication_overclaim",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_increase_across_markers"
            ]
          },
          {
            "name": "cf_infiltration_strength_decrease_direction",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_decrease_across_markers",
              "significant_decrease_strongest_markers_only",
              "consistent_decrease_not_statistically_robust"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-20-beta-reasoning-pi",
      "modelName": "grok-4.20-beta-0309-reasoning",
      "harness": "pi",
      "task": "gse297023_q6_age_immune_infiltration",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 199,
      "cost": 0.12170615,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "composition_stable_with_age",
            "role": "gate",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "frequencies_largely_stable"
            ]
          },
          {
            "name": "age_signal_is_state_not_composition",
            "role": "gate",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "frequencies_uninformative_for_infiltration",
            "role": "gate",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_cannot_establish_infiltration"
            ]
          },
          {
            "name": "infiltration_increases_with_age",
            "role": "gate",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_with_age"
            ]
          },
          {
            "name": "infiltration_strength_calibrated",
            "role": "gate",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "significant_increase_strongest_markers_only",
              "consistent_increase_not_statistically_robust"
            ]
          },
          {
            "name": "genotype_marker_fraction_lower_in_pten",
            "role": "gate",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "lower_in_pten_deficient"
            ]
          },
          {
            "name": "cf_composition_shift_reading",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "major_compositional_shift_with_age",
              "increased_immune_fraction_with_age"
            ]
          },
          {
            "name": "cf_lineage_specific_composition_claim",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "myeloid_expansion_with_age",
              "lymphoid_expansion_with_age",
              "minor_shifts_in_some_cell_types"
            ]
          },
          {
            "name": "cf_immunosenescence_depletion_prior",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "immune_depletion_with_age"
            ]
          },
          {
            "name": "cf_age_modality_compositional",
            "role": "hard_fail",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_type_composition_shift",
              "both_equally",
              "composition_dominant_state_secondary"
            ]
          },
          {
            "name": "cf_overread_frequencies_as_infiltration",
            "role": "hard_fail",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_shows_no_infiltration_change",
              "single_cell_quantifies_infiltration",
              "frequencies_show_increased_infiltration",
              "frequencies_show_decreased_infiltration"
            ]
          },
          {
            "name": "cf_infiltration_decreases_immunosenescence",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "decreased_with_age"
            ]
          },
          {
            "name": "cf_infiltration_unchanged_scrna_overread",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "unchanged_with_age"
            ]
          },
          {
            "name": "cf_infiltration_indeterminate_unused_node",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "direction_indeterminate_from_data"
            ]
          },
          {
            "name": "cf_infiltration_mixed_direction",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "mixed_by_immune_cell_type"
            ]
          },
          {
            "name": "cf_infiltration_genotype_confusion",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_only_in_pten_deficient",
              "decreased_only_in_pten_deficient"
            ]
          },
          {
            "name": "cf_pten_infiltration_paper_text_recall",
            "role": "hard_fail",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "higher_in_pten_deficient",
              "higher_in_pten_deficient_only_in_young",
              "higher_in_pten_deficient_only_in_aged"
            ]
          },
          {
            "name": "cf_pseudoreplication_overclaim",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_increase_across_markers"
            ]
          },
          {
            "name": "cf_infiltration_strength_decrease_direction",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_decrease_across_markers",
              "significant_decrease_strongest_markers_only",
              "consistent_decrease_not_statistically_robust"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-3-pi",
      "modelName": "grok-4.3",
      "harness": "pi",
      "task": "gse297023_q6_age_immune_infiltration",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 120,
      "cost": 0.016539750000000002,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "composition_stable_with_age",
            "role": "gate",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "frequencies_largely_stable"
            ]
          },
          {
            "name": "age_signal_is_state_not_composition",
            "role": "gate",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "frequencies_uninformative_for_infiltration",
            "role": "gate",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_cannot_establish_infiltration"
            ]
          },
          {
            "name": "infiltration_increases_with_age",
            "role": "gate",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_with_age"
            ]
          },
          {
            "name": "infiltration_strength_calibrated",
            "role": "gate",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "significant_increase_strongest_markers_only",
              "consistent_increase_not_statistically_robust"
            ]
          },
          {
            "name": "genotype_marker_fraction_lower_in_pten",
            "role": "gate",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "lower_in_pten_deficient"
            ]
          },
          {
            "name": "cf_composition_shift_reading",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "major_compositional_shift_with_age",
              "increased_immune_fraction_with_age"
            ]
          },
          {
            "name": "cf_lineage_specific_composition_claim",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "myeloid_expansion_with_age",
              "lymphoid_expansion_with_age",
              "minor_shifts_in_some_cell_types"
            ]
          },
          {
            "name": "cf_immunosenescence_depletion_prior",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "immune_depletion_with_age"
            ]
          },
          {
            "name": "cf_age_modality_compositional",
            "role": "hard_fail",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_type_composition_shift",
              "both_equally",
              "composition_dominant_state_secondary"
            ]
          },
          {
            "name": "cf_overread_frequencies_as_infiltration",
            "role": "hard_fail",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_shows_no_infiltration_change",
              "single_cell_quantifies_infiltration",
              "frequencies_show_increased_infiltration",
              "frequencies_show_decreased_infiltration"
            ]
          },
          {
            "name": "cf_infiltration_decreases_immunosenescence",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "decreased_with_age"
            ]
          },
          {
            "name": "cf_infiltration_unchanged_scrna_overread",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "unchanged_with_age"
            ]
          },
          {
            "name": "cf_infiltration_indeterminate_unused_node",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "direction_indeterminate_from_data"
            ]
          },
          {
            "name": "cf_infiltration_mixed_direction",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "mixed_by_immune_cell_type"
            ]
          },
          {
            "name": "cf_infiltration_genotype_confusion",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_only_in_pten_deficient",
              "decreased_only_in_pten_deficient"
            ]
          },
          {
            "name": "cf_pten_infiltration_paper_text_recall",
            "role": "hard_fail",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "higher_in_pten_deficient",
              "higher_in_pten_deficient_only_in_young",
              "higher_in_pten_deficient_only_in_aged"
            ]
          },
          {
            "name": "cf_pseudoreplication_overclaim",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_increase_across_markers"
            ]
          },
          {
            "name": "cf_infiltration_strength_decrease_direction",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_decrease_across_markers",
              "significant_decrease_strongest_markers_only",
              "consistent_decrease_not_statistically_robust"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-3-pi",
      "modelName": "grok-4.3",
      "harness": "pi",
      "task": "gse297023_q6_age_immune_infiltration",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 96,
      "cost": 0.023845050000000003,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "composition_stable_with_age",
            "role": "gate",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "frequencies_largely_stable"
            ]
          },
          {
            "name": "age_signal_is_state_not_composition",
            "role": "gate",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "frequencies_uninformative_for_infiltration",
            "role": "gate",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_cannot_establish_infiltration"
            ]
          },
          {
            "name": "infiltration_increases_with_age",
            "role": "gate",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_with_age"
            ]
          },
          {
            "name": "infiltration_strength_calibrated",
            "role": "gate",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "significant_increase_strongest_markers_only",
              "consistent_increase_not_statistically_robust"
            ]
          },
          {
            "name": "genotype_marker_fraction_lower_in_pten",
            "role": "gate",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "lower_in_pten_deficient"
            ]
          },
          {
            "name": "cf_composition_shift_reading",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "major_compositional_shift_with_age",
              "increased_immune_fraction_with_age"
            ]
          },
          {
            "name": "cf_lineage_specific_composition_claim",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "myeloid_expansion_with_age",
              "lymphoid_expansion_with_age",
              "minor_shifts_in_some_cell_types"
            ]
          },
          {
            "name": "cf_immunosenescence_depletion_prior",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "immune_depletion_with_age"
            ]
          },
          {
            "name": "cf_age_modality_compositional",
            "role": "hard_fail",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_type_composition_shift",
              "both_equally",
              "composition_dominant_state_secondary"
            ]
          },
          {
            "name": "cf_overread_frequencies_as_infiltration",
            "role": "hard_fail",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_shows_no_infiltration_change",
              "single_cell_quantifies_infiltration",
              "frequencies_show_increased_infiltration",
              "frequencies_show_decreased_infiltration"
            ]
          },
          {
            "name": "cf_infiltration_decreases_immunosenescence",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "decreased_with_age"
            ]
          },
          {
            "name": "cf_infiltration_unchanged_scrna_overread",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "unchanged_with_age"
            ]
          },
          {
            "name": "cf_infiltration_indeterminate_unused_node",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "direction_indeterminate_from_data"
            ]
          },
          {
            "name": "cf_infiltration_mixed_direction",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "mixed_by_immune_cell_type"
            ]
          },
          {
            "name": "cf_infiltration_genotype_confusion",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_only_in_pten_deficient",
              "decreased_only_in_pten_deficient"
            ]
          },
          {
            "name": "cf_pten_infiltration_paper_text_recall",
            "role": "hard_fail",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "higher_in_pten_deficient",
              "higher_in_pten_deficient_only_in_young",
              "higher_in_pten_deficient_only_in_aged"
            ]
          },
          {
            "name": "cf_pseudoreplication_overclaim",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_increase_across_markers"
            ]
          },
          {
            "name": "cf_infiltration_strength_decrease_direction",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_decrease_across_markers",
              "significant_decrease_strongest_markers_only",
              "consistent_decrease_not_statistically_robust"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-3-pi",
      "modelName": "grok-4.3",
      "harness": "pi",
      "task": "gse297023_q6_age_immune_infiltration",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 83,
      "cost": 0.017433,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "composition_stable_with_age",
            "role": "gate",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "frequencies_largely_stable"
            ]
          },
          {
            "name": "age_signal_is_state_not_composition",
            "role": "gate",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "frequencies_uninformative_for_infiltration",
            "role": "gate",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_cannot_establish_infiltration"
            ]
          },
          {
            "name": "infiltration_increases_with_age",
            "role": "gate",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_with_age"
            ]
          },
          {
            "name": "infiltration_strength_calibrated",
            "role": "gate",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "significant_increase_strongest_markers_only",
              "consistent_increase_not_statistically_robust"
            ]
          },
          {
            "name": "genotype_marker_fraction_lower_in_pten",
            "role": "gate",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "lower_in_pten_deficient"
            ]
          },
          {
            "name": "cf_composition_shift_reading",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "major_compositional_shift_with_age",
              "increased_immune_fraction_with_age"
            ]
          },
          {
            "name": "cf_lineage_specific_composition_claim",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "myeloid_expansion_with_age",
              "lymphoid_expansion_with_age",
              "minor_shifts_in_some_cell_types"
            ]
          },
          {
            "name": "cf_immunosenescence_depletion_prior",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "immune_depletion_with_age"
            ]
          },
          {
            "name": "cf_age_modality_compositional",
            "role": "hard_fail",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_type_composition_shift",
              "both_equally",
              "composition_dominant_state_secondary"
            ]
          },
          {
            "name": "cf_overread_frequencies_as_infiltration",
            "role": "hard_fail",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_shows_no_infiltration_change",
              "single_cell_quantifies_infiltration",
              "frequencies_show_increased_infiltration",
              "frequencies_show_decreased_infiltration"
            ]
          },
          {
            "name": "cf_infiltration_decreases_immunosenescence",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "decreased_with_age"
            ]
          },
          {
            "name": "cf_infiltration_unchanged_scrna_overread",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "unchanged_with_age"
            ]
          },
          {
            "name": "cf_infiltration_indeterminate_unused_node",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "direction_indeterminate_from_data"
            ]
          },
          {
            "name": "cf_infiltration_mixed_direction",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "mixed_by_immune_cell_type"
            ]
          },
          {
            "name": "cf_infiltration_genotype_confusion",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_only_in_pten_deficient",
              "decreased_only_in_pten_deficient"
            ]
          },
          {
            "name": "cf_pten_infiltration_paper_text_recall",
            "role": "hard_fail",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "higher_in_pten_deficient",
              "higher_in_pten_deficient_only_in_young",
              "higher_in_pten_deficient_only_in_aged"
            ]
          },
          {
            "name": "cf_pseudoreplication_overclaim",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_increase_across_markers"
            ]
          },
          {
            "name": "cf_infiltration_strength_decrease_direction",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_decrease_across_markers",
              "significant_decrease_strongest_markers_only",
              "consistent_decrease_not_statistically_robust"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-kimi-k2p6-pi",
      "modelName": "kimi-k2p6",
      "harness": "pi",
      "task": "gse297023_q6_age_immune_infiltration",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 1463,
      "cost": 0.7140193599999999,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "composition_stable_with_age",
            "role": "gate",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "frequencies_largely_stable"
            ]
          },
          {
            "name": "age_signal_is_state_not_composition",
            "role": "gate",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "frequencies_uninformative_for_infiltration",
            "role": "gate",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_cannot_establish_infiltration"
            ]
          },
          {
            "name": "infiltration_increases_with_age",
            "role": "gate",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_with_age"
            ]
          },
          {
            "name": "infiltration_strength_calibrated",
            "role": "gate",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "significant_increase_strongest_markers_only",
              "consistent_increase_not_statistically_robust"
            ]
          },
          {
            "name": "genotype_marker_fraction_lower_in_pten",
            "role": "gate",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "lower_in_pten_deficient"
            ]
          },
          {
            "name": "cf_composition_shift_reading",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "major_compositional_shift_with_age",
              "increased_immune_fraction_with_age"
            ]
          },
          {
            "name": "cf_lineage_specific_composition_claim",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "myeloid_expansion_with_age",
              "lymphoid_expansion_with_age",
              "minor_shifts_in_some_cell_types"
            ]
          },
          {
            "name": "cf_immunosenescence_depletion_prior",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "immune_depletion_with_age"
            ]
          },
          {
            "name": "cf_age_modality_compositional",
            "role": "hard_fail",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_type_composition_shift",
              "both_equally",
              "composition_dominant_state_secondary"
            ]
          },
          {
            "name": "cf_overread_frequencies_as_infiltration",
            "role": "hard_fail",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_shows_no_infiltration_change",
              "single_cell_quantifies_infiltration",
              "frequencies_show_increased_infiltration",
              "frequencies_show_decreased_infiltration"
            ]
          },
          {
            "name": "cf_infiltration_decreases_immunosenescence",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "decreased_with_age"
            ]
          },
          {
            "name": "cf_infiltration_unchanged_scrna_overread",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "unchanged_with_age"
            ]
          },
          {
            "name": "cf_infiltration_indeterminate_unused_node",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "direction_indeterminate_from_data"
            ]
          },
          {
            "name": "cf_infiltration_mixed_direction",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "mixed_by_immune_cell_type"
            ]
          },
          {
            "name": "cf_infiltration_genotype_confusion",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_only_in_pten_deficient",
              "decreased_only_in_pten_deficient"
            ]
          },
          {
            "name": "cf_pten_infiltration_paper_text_recall",
            "role": "hard_fail",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "higher_in_pten_deficient",
              "higher_in_pten_deficient_only_in_young",
              "higher_in_pten_deficient_only_in_aged"
            ]
          },
          {
            "name": "cf_pseudoreplication_overclaim",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_increase_across_markers"
            ]
          },
          {
            "name": "cf_infiltration_strength_decrease_direction",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_decrease_across_markers",
              "significant_decrease_strongest_markers_only",
              "consistent_decrease_not_statistically_robust"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-kimi-k2p6-pi",
      "modelName": "kimi-k2p6",
      "harness": "pi",
      "task": "gse297023_q6_age_immune_infiltration",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 4558,
      "cost": 0.7568265099999999,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "composition_stable_with_age",
            "role": "gate",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "frequencies_largely_stable"
            ]
          },
          {
            "name": "age_signal_is_state_not_composition",
            "role": "gate",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "frequencies_uninformative_for_infiltration",
            "role": "gate",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_cannot_establish_infiltration"
            ]
          },
          {
            "name": "infiltration_increases_with_age",
            "role": "gate",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_with_age"
            ]
          },
          {
            "name": "infiltration_strength_calibrated",
            "role": "gate",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "significant_increase_strongest_markers_only",
              "consistent_increase_not_statistically_robust"
            ]
          },
          {
            "name": "genotype_marker_fraction_lower_in_pten",
            "role": "gate",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "lower_in_pten_deficient"
            ]
          },
          {
            "name": "cf_composition_shift_reading",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "major_compositional_shift_with_age",
              "increased_immune_fraction_with_age"
            ]
          },
          {
            "name": "cf_lineage_specific_composition_claim",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "myeloid_expansion_with_age",
              "lymphoid_expansion_with_age",
              "minor_shifts_in_some_cell_types"
            ]
          },
          {
            "name": "cf_immunosenescence_depletion_prior",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "immune_depletion_with_age"
            ]
          },
          {
            "name": "cf_age_modality_compositional",
            "role": "hard_fail",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_type_composition_shift",
              "both_equally",
              "composition_dominant_state_secondary"
            ]
          },
          {
            "name": "cf_overread_frequencies_as_infiltration",
            "role": "hard_fail",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_shows_no_infiltration_change",
              "single_cell_quantifies_infiltration",
              "frequencies_show_increased_infiltration",
              "frequencies_show_decreased_infiltration"
            ]
          },
          {
            "name": "cf_infiltration_decreases_immunosenescence",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "decreased_with_age"
            ]
          },
          {
            "name": "cf_infiltration_unchanged_scrna_overread",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "unchanged_with_age"
            ]
          },
          {
            "name": "cf_infiltration_indeterminate_unused_node",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "direction_indeterminate_from_data"
            ]
          },
          {
            "name": "cf_infiltration_mixed_direction",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "mixed_by_immune_cell_type"
            ]
          },
          {
            "name": "cf_infiltration_genotype_confusion",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_only_in_pten_deficient",
              "decreased_only_in_pten_deficient"
            ]
          },
          {
            "name": "cf_pten_infiltration_paper_text_recall",
            "role": "hard_fail",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "higher_in_pten_deficient",
              "higher_in_pten_deficient_only_in_young",
              "higher_in_pten_deficient_only_in_aged"
            ]
          },
          {
            "name": "cf_pseudoreplication_overclaim",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_increase_across_markers"
            ]
          },
          {
            "name": "cf_infiltration_strength_decrease_direction",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_decrease_across_markers",
              "significant_decrease_strongest_markers_only",
              "consistent_decrease_not_statistically_robust"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-kimi-k2p6-pi",
      "modelName": "kimi-k2p6",
      "harness": "pi",
      "task": "gse297023_q6_age_immune_infiltration",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1329,
      "cost": 0.37676261,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "composition_stable_with_age",
            "role": "gate",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "frequencies_largely_stable"
            ]
          },
          {
            "name": "age_signal_is_state_not_composition",
            "role": "gate",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "frequencies_uninformative_for_infiltration",
            "role": "gate",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_cannot_establish_infiltration"
            ]
          },
          {
            "name": "infiltration_increases_with_age",
            "role": "gate",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_with_age"
            ]
          },
          {
            "name": "infiltration_strength_calibrated",
            "role": "gate",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "significant_increase_strongest_markers_only",
              "consistent_increase_not_statistically_robust"
            ]
          },
          {
            "name": "genotype_marker_fraction_lower_in_pten",
            "role": "gate",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "lower_in_pten_deficient"
            ]
          },
          {
            "name": "cf_composition_shift_reading",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "major_compositional_shift_with_age",
              "increased_immune_fraction_with_age"
            ]
          },
          {
            "name": "cf_lineage_specific_composition_claim",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "myeloid_expansion_with_age",
              "lymphoid_expansion_with_age",
              "minor_shifts_in_some_cell_types"
            ]
          },
          {
            "name": "cf_immunosenescence_depletion_prior",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "immune_depletion_with_age"
            ]
          },
          {
            "name": "cf_age_modality_compositional",
            "role": "hard_fail",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_type_composition_shift",
              "both_equally",
              "composition_dominant_state_secondary"
            ]
          },
          {
            "name": "cf_overread_frequencies_as_infiltration",
            "role": "hard_fail",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_shows_no_infiltration_change",
              "single_cell_quantifies_infiltration",
              "frequencies_show_increased_infiltration",
              "frequencies_show_decreased_infiltration"
            ]
          },
          {
            "name": "cf_infiltration_decreases_immunosenescence",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "decreased_with_age"
            ]
          },
          {
            "name": "cf_infiltration_unchanged_scrna_overread",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "unchanged_with_age"
            ]
          },
          {
            "name": "cf_infiltration_indeterminate_unused_node",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "direction_indeterminate_from_data"
            ]
          },
          {
            "name": "cf_infiltration_mixed_direction",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "mixed_by_immune_cell_type"
            ]
          },
          {
            "name": "cf_infiltration_genotype_confusion",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_only_in_pten_deficient",
              "decreased_only_in_pten_deficient"
            ]
          },
          {
            "name": "cf_pten_infiltration_paper_text_recall",
            "role": "hard_fail",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "higher_in_pten_deficient",
              "higher_in_pten_deficient_only_in_young",
              "higher_in_pten_deficient_only_in_aged"
            ]
          },
          {
            "name": "cf_pseudoreplication_overclaim",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_increase_across_markers"
            ]
          },
          {
            "name": "cf_infiltration_strength_decrease_direction",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_decrease_across_markers",
              "significant_decrease_strongest_markers_only",
              "consistent_decrease_not_statistically_robust"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-6-claude-code",
      "modelName": "claude-opus-4-6",
      "harness": "claude-code",
      "task": "q6_myeloid_subset_restricted_de",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 1275,
      "cost": 1.7519377500000002,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "myeloid_subset_with_strongest_change_gate",
            "role": "gate",
            "field": "$.myeloid_subset_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "subset_gamma"
            ]
          },
          {
            "name": "gene_with_strongest_change_gate",
            "role": "gate",
            "field": "$.gene_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "AXL",
              "PROS1"
            ]
          },
          {
            "name": "direction_of_change_gate",
            "role": "gate",
            "field": "$.direction_of_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "down_in_covid"
            ]
          },
          {
            "name": "biological_function_interpretation_gate",
            "role": "gate",
            "field": "$.biological_function_interpretation",
            "kind": "value",
            "op": "in",
            "expected": [
              "efferocytosis",
              "phagocytosis"
            ]
          },
          {
            "name": "myeloid_compartment_abundance_gate",
            "role": "gate",
            "field": "$.myeloid_compartment_abundance",
            "kind": "value",
            "op": "in",
            "expected": [
              "expands_in_covid"
            ]
          },
          {
            "name": "myeloid_expansion_localization_gate",
            "role": "gate",
            "field": "$.myeloid_expansion_localization",
            "kind": "value",
            "op": "in",
            "expected": [
              "recruited_driven_resident_dysfunction"
            ]
          },
          {
            "name": "dominant_signature_gate",
            "role": "gate",
            "field": "$.dominant_signature",
            "kind": "value",
            "op": "in",
            "expected": [
              "technical_or_agonal_artifact"
            ]
          },
          {
            "name": "dominant_signature_direction_gate",
            "role": "gate",
            "field": "$.dominant_signature_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_in_covid"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-6-claude-code",
      "modelName": "claude-opus-4-6",
      "harness": "claude-code",
      "task": "q6_myeloid_subset_restricted_de",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 2114,
      "cost": 4.0105465,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "myeloid_subset_with_strongest_change_gate",
            "role": "gate",
            "field": "$.myeloid_subset_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "subset_gamma"
            ]
          },
          {
            "name": "gene_with_strongest_change_gate",
            "role": "gate",
            "field": "$.gene_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "AXL",
              "PROS1"
            ]
          },
          {
            "name": "direction_of_change_gate",
            "role": "gate",
            "field": "$.direction_of_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "down_in_covid"
            ]
          },
          {
            "name": "biological_function_interpretation_gate",
            "role": "gate",
            "field": "$.biological_function_interpretation",
            "kind": "value",
            "op": "in",
            "expected": [
              "efferocytosis",
              "phagocytosis"
            ]
          },
          {
            "name": "myeloid_compartment_abundance_gate",
            "role": "gate",
            "field": "$.myeloid_compartment_abundance",
            "kind": "value",
            "op": "in",
            "expected": [
              "expands_in_covid"
            ]
          },
          {
            "name": "myeloid_expansion_localization_gate",
            "role": "gate",
            "field": "$.myeloid_expansion_localization",
            "kind": "value",
            "op": "in",
            "expected": [
              "recruited_driven_resident_dysfunction"
            ]
          },
          {
            "name": "dominant_signature_gate",
            "role": "gate",
            "field": "$.dominant_signature",
            "kind": "value",
            "op": "in",
            "expected": [
              "technical_or_agonal_artifact"
            ]
          },
          {
            "name": "dominant_signature_direction_gate",
            "role": "gate",
            "field": "$.dominant_signature_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_in_covid"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-6-claude-code",
      "modelName": "claude-opus-4-6",
      "harness": "claude-code",
      "task": "q6_myeloid_subset_restricted_de",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1268,
      "cost": 1.4594339999999997,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "myeloid_subset_with_strongest_change_gate",
            "role": "gate",
            "field": "$.myeloid_subset_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "subset_gamma"
            ]
          },
          {
            "name": "gene_with_strongest_change_gate",
            "role": "gate",
            "field": "$.gene_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "AXL",
              "PROS1"
            ]
          },
          {
            "name": "direction_of_change_gate",
            "role": "gate",
            "field": "$.direction_of_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "down_in_covid"
            ]
          },
          {
            "name": "biological_function_interpretation_gate",
            "role": "gate",
            "field": "$.biological_function_interpretation",
            "kind": "value",
            "op": "in",
            "expected": [
              "efferocytosis",
              "phagocytosis"
            ]
          },
          {
            "name": "myeloid_compartment_abundance_gate",
            "role": "gate",
            "field": "$.myeloid_compartment_abundance",
            "kind": "value",
            "op": "in",
            "expected": [
              "expands_in_covid"
            ]
          },
          {
            "name": "myeloid_expansion_localization_gate",
            "role": "gate",
            "field": "$.myeloid_expansion_localization",
            "kind": "value",
            "op": "in",
            "expected": [
              "recruited_driven_resident_dysfunction"
            ]
          },
          {
            "name": "dominant_signature_gate",
            "role": "gate",
            "field": "$.dominant_signature",
            "kind": "value",
            "op": "in",
            "expected": [
              "technical_or_agonal_artifact"
            ]
          },
          {
            "name": "dominant_signature_direction_gate",
            "role": "gate",
            "field": "$.dominant_signature_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_in_covid"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-6-pi",
      "modelName": "claude-opus-4-6",
      "harness": "pi",
      "task": "q6_myeloid_subset_restricted_de",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 1453,
      "cost": 1.9215345,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "myeloid_subset_with_strongest_change_gate",
            "role": "gate",
            "field": "$.myeloid_subset_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "subset_gamma"
            ]
          },
          {
            "name": "gene_with_strongest_change_gate",
            "role": "gate",
            "field": "$.gene_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "AXL",
              "PROS1"
            ]
          },
          {
            "name": "direction_of_change_gate",
            "role": "gate",
            "field": "$.direction_of_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "down_in_covid"
            ]
          },
          {
            "name": "biological_function_interpretation_gate",
            "role": "gate",
            "field": "$.biological_function_interpretation",
            "kind": "value",
            "op": "in",
            "expected": [
              "efferocytosis",
              "phagocytosis"
            ]
          },
          {
            "name": "myeloid_compartment_abundance_gate",
            "role": "gate",
            "field": "$.myeloid_compartment_abundance",
            "kind": "value",
            "op": "in",
            "expected": [
              "expands_in_covid"
            ]
          },
          {
            "name": "myeloid_expansion_localization_gate",
            "role": "gate",
            "field": "$.myeloid_expansion_localization",
            "kind": "value",
            "op": "in",
            "expected": [
              "recruited_driven_resident_dysfunction"
            ]
          },
          {
            "name": "dominant_signature_gate",
            "role": "gate",
            "field": "$.dominant_signature",
            "kind": "value",
            "op": "in",
            "expected": [
              "technical_or_agonal_artifact"
            ]
          },
          {
            "name": "dominant_signature_direction_gate",
            "role": "gate",
            "field": "$.dominant_signature_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_in_covid"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-6-pi",
      "modelName": "claude-opus-4-6",
      "harness": "pi",
      "task": "q6_myeloid_subset_restricted_de",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 1974,
      "cost": 1.4819150000000003,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "myeloid_subset_with_strongest_change_gate",
            "role": "gate",
            "field": "$.myeloid_subset_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "subset_gamma"
            ]
          },
          {
            "name": "gene_with_strongest_change_gate",
            "role": "gate",
            "field": "$.gene_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "AXL",
              "PROS1"
            ]
          },
          {
            "name": "direction_of_change_gate",
            "role": "gate",
            "field": "$.direction_of_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "down_in_covid"
            ]
          },
          {
            "name": "biological_function_interpretation_gate",
            "role": "gate",
            "field": "$.biological_function_interpretation",
            "kind": "value",
            "op": "in",
            "expected": [
              "efferocytosis",
              "phagocytosis"
            ]
          },
          {
            "name": "myeloid_compartment_abundance_gate",
            "role": "gate",
            "field": "$.myeloid_compartment_abundance",
            "kind": "value",
            "op": "in",
            "expected": [
              "expands_in_covid"
            ]
          },
          {
            "name": "myeloid_expansion_localization_gate",
            "role": "gate",
            "field": "$.myeloid_expansion_localization",
            "kind": "value",
            "op": "in",
            "expected": [
              "recruited_driven_resident_dysfunction"
            ]
          },
          {
            "name": "dominant_signature_gate",
            "role": "gate",
            "field": "$.dominant_signature",
            "kind": "value",
            "op": "in",
            "expected": [
              "technical_or_agonal_artifact"
            ]
          },
          {
            "name": "dominant_signature_direction_gate",
            "role": "gate",
            "field": "$.dominant_signature_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_in_covid"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-6-pi",
      "modelName": "claude-opus-4-6",
      "harness": "pi",
      "task": "q6_myeloid_subset_restricted_de",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 3,
      "passed": true,
      "score": null,
      "durationS": 1626,
      "cost": 1.625898,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "myeloid_subset_with_strongest_change_gate",
            "role": "gate",
            "field": "$.myeloid_subset_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "subset_gamma"
            ]
          },
          {
            "name": "gene_with_strongest_change_gate",
            "role": "gate",
            "field": "$.gene_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "AXL",
              "PROS1"
            ]
          },
          {
            "name": "direction_of_change_gate",
            "role": "gate",
            "field": "$.direction_of_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "down_in_covid"
            ]
          },
          {
            "name": "biological_function_interpretation_gate",
            "role": "gate",
            "field": "$.biological_function_interpretation",
            "kind": "value",
            "op": "in",
            "expected": [
              "efferocytosis",
              "phagocytosis"
            ]
          },
          {
            "name": "myeloid_compartment_abundance_gate",
            "role": "gate",
            "field": "$.myeloid_compartment_abundance",
            "kind": "value",
            "op": "in",
            "expected": [
              "expands_in_covid"
            ]
          },
          {
            "name": "myeloid_expansion_localization_gate",
            "role": "gate",
            "field": "$.myeloid_expansion_localization",
            "kind": "value",
            "op": "in",
            "expected": [
              "recruited_driven_resident_dysfunction"
            ]
          },
          {
            "name": "dominant_signature_gate",
            "role": "gate",
            "field": "$.dominant_signature",
            "kind": "value",
            "op": "in",
            "expected": [
              "technical_or_agonal_artifact"
            ]
          },
          {
            "name": "dominant_signature_direction_gate",
            "role": "gate",
            "field": "$.dominant_signature_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_in_covid"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-7-claude-code",
      "modelName": "claude-opus-4-7",
      "harness": "claude-code",
      "task": "q6_myeloid_subset_restricted_de",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 1987,
      "cost": 4.062115,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "myeloid_subset_with_strongest_change_gate",
            "role": "gate",
            "field": "$.myeloid_subset_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "subset_gamma"
            ]
          },
          {
            "name": "gene_with_strongest_change_gate",
            "role": "gate",
            "field": "$.gene_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "AXL",
              "PROS1"
            ]
          },
          {
            "name": "direction_of_change_gate",
            "role": "gate",
            "field": "$.direction_of_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "down_in_covid"
            ]
          },
          {
            "name": "biological_function_interpretation_gate",
            "role": "gate",
            "field": "$.biological_function_interpretation",
            "kind": "value",
            "op": "in",
            "expected": [
              "efferocytosis",
              "phagocytosis"
            ]
          },
          {
            "name": "myeloid_compartment_abundance_gate",
            "role": "gate",
            "field": "$.myeloid_compartment_abundance",
            "kind": "value",
            "op": "in",
            "expected": [
              "expands_in_covid"
            ]
          },
          {
            "name": "myeloid_expansion_localization_gate",
            "role": "gate",
            "field": "$.myeloid_expansion_localization",
            "kind": "value",
            "op": "in",
            "expected": [
              "recruited_driven_resident_dysfunction"
            ]
          },
          {
            "name": "dominant_signature_gate",
            "role": "gate",
            "field": "$.dominant_signature",
            "kind": "value",
            "op": "in",
            "expected": [
              "technical_or_agonal_artifact"
            ]
          },
          {
            "name": "dominant_signature_direction_gate",
            "role": "gate",
            "field": "$.dominant_signature_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_in_covid"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-7-claude-code",
      "modelName": "claude-opus-4-7",
      "harness": "claude-code",
      "task": "q6_myeloid_subset_restricted_de",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 1471,
      "cost": 4.3262462500000005,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "myeloid_subset_with_strongest_change_gate",
            "role": "gate",
            "field": "$.myeloid_subset_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "subset_gamma"
            ]
          },
          {
            "name": "gene_with_strongest_change_gate",
            "role": "gate",
            "field": "$.gene_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "AXL",
              "PROS1"
            ]
          },
          {
            "name": "direction_of_change_gate",
            "role": "gate",
            "field": "$.direction_of_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "down_in_covid"
            ]
          },
          {
            "name": "biological_function_interpretation_gate",
            "role": "gate",
            "field": "$.biological_function_interpretation",
            "kind": "value",
            "op": "in",
            "expected": [
              "efferocytosis",
              "phagocytosis"
            ]
          },
          {
            "name": "myeloid_compartment_abundance_gate",
            "role": "gate",
            "field": "$.myeloid_compartment_abundance",
            "kind": "value",
            "op": "in",
            "expected": [
              "expands_in_covid"
            ]
          },
          {
            "name": "myeloid_expansion_localization_gate",
            "role": "gate",
            "field": "$.myeloid_expansion_localization",
            "kind": "value",
            "op": "in",
            "expected": [
              "recruited_driven_resident_dysfunction"
            ]
          },
          {
            "name": "dominant_signature_gate",
            "role": "gate",
            "field": "$.dominant_signature",
            "kind": "value",
            "op": "in",
            "expected": [
              "technical_or_agonal_artifact"
            ]
          },
          {
            "name": "dominant_signature_direction_gate",
            "role": "gate",
            "field": "$.dominant_signature_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_in_covid"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-7-claude-code",
      "modelName": "claude-opus-4-7",
      "harness": "claude-code",
      "task": "q6_myeloid_subset_restricted_de",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1142,
      "cost": 3.3973032499999998,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "myeloid_subset_with_strongest_change_gate",
            "role": "gate",
            "field": "$.myeloid_subset_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "subset_gamma"
            ]
          },
          {
            "name": "gene_with_strongest_change_gate",
            "role": "gate",
            "field": "$.gene_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "AXL",
              "PROS1"
            ]
          },
          {
            "name": "direction_of_change_gate",
            "role": "gate",
            "field": "$.direction_of_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "down_in_covid"
            ]
          },
          {
            "name": "biological_function_interpretation_gate",
            "role": "gate",
            "field": "$.biological_function_interpretation",
            "kind": "value",
            "op": "in",
            "expected": [
              "efferocytosis",
              "phagocytosis"
            ]
          },
          {
            "name": "myeloid_compartment_abundance_gate",
            "role": "gate",
            "field": "$.myeloid_compartment_abundance",
            "kind": "value",
            "op": "in",
            "expected": [
              "expands_in_covid"
            ]
          },
          {
            "name": "myeloid_expansion_localization_gate",
            "role": "gate",
            "field": "$.myeloid_expansion_localization",
            "kind": "value",
            "op": "in",
            "expected": [
              "recruited_driven_resident_dysfunction"
            ]
          },
          {
            "name": "dominant_signature_gate",
            "role": "gate",
            "field": "$.dominant_signature",
            "kind": "value",
            "op": "in",
            "expected": [
              "technical_or_agonal_artifact"
            ]
          },
          {
            "name": "dominant_signature_direction_gate",
            "role": "gate",
            "field": "$.dominant_signature_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_in_covid"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-7-pi",
      "modelName": "claude-opus-4-7",
      "harness": "pi",
      "task": "q6_myeloid_subset_restricted_de",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 900,
      "cost": 1.02000525,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "myeloid_subset_with_strongest_change_gate",
            "role": "gate",
            "field": "$.myeloid_subset_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "subset_gamma"
            ]
          },
          {
            "name": "gene_with_strongest_change_gate",
            "role": "gate",
            "field": "$.gene_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "AXL",
              "PROS1"
            ]
          },
          {
            "name": "direction_of_change_gate",
            "role": "gate",
            "field": "$.direction_of_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "down_in_covid"
            ]
          },
          {
            "name": "biological_function_interpretation_gate",
            "role": "gate",
            "field": "$.biological_function_interpretation",
            "kind": "value",
            "op": "in",
            "expected": [
              "efferocytosis",
              "phagocytosis"
            ]
          },
          {
            "name": "myeloid_compartment_abundance_gate",
            "role": "gate",
            "field": "$.myeloid_compartment_abundance",
            "kind": "value",
            "op": "in",
            "expected": [
              "expands_in_covid"
            ]
          },
          {
            "name": "myeloid_expansion_localization_gate",
            "role": "gate",
            "field": "$.myeloid_expansion_localization",
            "kind": "value",
            "op": "in",
            "expected": [
              "recruited_driven_resident_dysfunction"
            ]
          },
          {
            "name": "dominant_signature_gate",
            "role": "gate",
            "field": "$.dominant_signature",
            "kind": "value",
            "op": "in",
            "expected": [
              "technical_or_agonal_artifact"
            ]
          },
          {
            "name": "dominant_signature_direction_gate",
            "role": "gate",
            "field": "$.dominant_signature_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_in_covid"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-7-pi",
      "modelName": "claude-opus-4-7",
      "harness": "pi",
      "task": "q6_myeloid_subset_restricted_de",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 464,
      "cost": 0.9251775000000001,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "myeloid_subset_with_strongest_change_gate",
            "role": "gate",
            "field": "$.myeloid_subset_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "subset_gamma"
            ]
          },
          {
            "name": "gene_with_strongest_change_gate",
            "role": "gate",
            "field": "$.gene_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "AXL",
              "PROS1"
            ]
          },
          {
            "name": "direction_of_change_gate",
            "role": "gate",
            "field": "$.direction_of_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "down_in_covid"
            ]
          },
          {
            "name": "biological_function_interpretation_gate",
            "role": "gate",
            "field": "$.biological_function_interpretation",
            "kind": "value",
            "op": "in",
            "expected": [
              "efferocytosis",
              "phagocytosis"
            ]
          },
          {
            "name": "myeloid_compartment_abundance_gate",
            "role": "gate",
            "field": "$.myeloid_compartment_abundance",
            "kind": "value",
            "op": "in",
            "expected": [
              "expands_in_covid"
            ]
          },
          {
            "name": "myeloid_expansion_localization_gate",
            "role": "gate",
            "field": "$.myeloid_expansion_localization",
            "kind": "value",
            "op": "in",
            "expected": [
              "recruited_driven_resident_dysfunction"
            ]
          },
          {
            "name": "dominant_signature_gate",
            "role": "gate",
            "field": "$.dominant_signature",
            "kind": "value",
            "op": "in",
            "expected": [
              "technical_or_agonal_artifact"
            ]
          },
          {
            "name": "dominant_signature_direction_gate",
            "role": "gate",
            "field": "$.dominant_signature_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_in_covid"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-7-pi",
      "modelName": "claude-opus-4-7",
      "harness": "pi",
      "task": "q6_myeloid_subset_restricted_de",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 590,
      "cost": 0.8895885,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "myeloid_subset_with_strongest_change_gate",
            "role": "gate",
            "field": "$.myeloid_subset_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "subset_gamma"
            ]
          },
          {
            "name": "gene_with_strongest_change_gate",
            "role": "gate",
            "field": "$.gene_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "AXL",
              "PROS1"
            ]
          },
          {
            "name": "direction_of_change_gate",
            "role": "gate",
            "field": "$.direction_of_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "down_in_covid"
            ]
          },
          {
            "name": "biological_function_interpretation_gate",
            "role": "gate",
            "field": "$.biological_function_interpretation",
            "kind": "value",
            "op": "in",
            "expected": [
              "efferocytosis",
              "phagocytosis"
            ]
          },
          {
            "name": "myeloid_compartment_abundance_gate",
            "role": "gate",
            "field": "$.myeloid_compartment_abundance",
            "kind": "value",
            "op": "in",
            "expected": [
              "expands_in_covid"
            ]
          },
          {
            "name": "myeloid_expansion_localization_gate",
            "role": "gate",
            "field": "$.myeloid_expansion_localization",
            "kind": "value",
            "op": "in",
            "expected": [
              "recruited_driven_resident_dysfunction"
            ]
          },
          {
            "name": "dominant_signature_gate",
            "role": "gate",
            "field": "$.dominant_signature",
            "kind": "value",
            "op": "in",
            "expected": [
              "technical_or_agonal_artifact"
            ]
          },
          {
            "name": "dominant_signature_direction_gate",
            "role": "gate",
            "field": "$.dominant_signature_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_in_covid"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-8-claude-code",
      "modelName": "claude-opus-4-8",
      "harness": "claude-code",
      "task": "q6_myeloid_subset_restricted_de",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 1,
      "passed": true,
      "score": null,
      "durationS": 2255,
      "cost": 7.2938315,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "myeloid_subset_with_strongest_change_gate",
            "role": "gate",
            "field": "$.myeloid_subset_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "subset_gamma"
            ]
          },
          {
            "name": "gene_with_strongest_change_gate",
            "role": "gate",
            "field": "$.gene_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "AXL",
              "PROS1"
            ]
          },
          {
            "name": "direction_of_change_gate",
            "role": "gate",
            "field": "$.direction_of_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "down_in_covid"
            ]
          },
          {
            "name": "biological_function_interpretation_gate",
            "role": "gate",
            "field": "$.biological_function_interpretation",
            "kind": "value",
            "op": "in",
            "expected": [
              "efferocytosis",
              "phagocytosis"
            ]
          },
          {
            "name": "myeloid_compartment_abundance_gate",
            "role": "gate",
            "field": "$.myeloid_compartment_abundance",
            "kind": "value",
            "op": "in",
            "expected": [
              "expands_in_covid"
            ]
          },
          {
            "name": "myeloid_expansion_localization_gate",
            "role": "gate",
            "field": "$.myeloid_expansion_localization",
            "kind": "value",
            "op": "in",
            "expected": [
              "recruited_driven_resident_dysfunction"
            ]
          },
          {
            "name": "dominant_signature_gate",
            "role": "gate",
            "field": "$.dominant_signature",
            "kind": "value",
            "op": "in",
            "expected": [
              "technical_or_agonal_artifact"
            ]
          },
          {
            "name": "dominant_signature_direction_gate",
            "role": "gate",
            "field": "$.dominant_signature_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_in_covid"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-8-claude-code",
      "modelName": "claude-opus-4-8",
      "harness": "claude-code",
      "task": "q6_myeloid_subset_restricted_de",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 2,
      "passed": true,
      "score": null,
      "durationS": 2545,
      "cost": 6.220426000000001,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "myeloid_subset_with_strongest_change_gate",
            "role": "gate",
            "field": "$.myeloid_subset_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "subset_gamma"
            ]
          },
          {
            "name": "gene_with_strongest_change_gate",
            "role": "gate",
            "field": "$.gene_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "AXL",
              "PROS1"
            ]
          },
          {
            "name": "direction_of_change_gate",
            "role": "gate",
            "field": "$.direction_of_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "down_in_covid"
            ]
          },
          {
            "name": "biological_function_interpretation_gate",
            "role": "gate",
            "field": "$.biological_function_interpretation",
            "kind": "value",
            "op": "in",
            "expected": [
              "efferocytosis",
              "phagocytosis"
            ]
          },
          {
            "name": "myeloid_compartment_abundance_gate",
            "role": "gate",
            "field": "$.myeloid_compartment_abundance",
            "kind": "value",
            "op": "in",
            "expected": [
              "expands_in_covid"
            ]
          },
          {
            "name": "myeloid_expansion_localization_gate",
            "role": "gate",
            "field": "$.myeloid_expansion_localization",
            "kind": "value",
            "op": "in",
            "expected": [
              "recruited_driven_resident_dysfunction"
            ]
          },
          {
            "name": "dominant_signature_gate",
            "role": "gate",
            "field": "$.dominant_signature",
            "kind": "value",
            "op": "in",
            "expected": [
              "technical_or_agonal_artifact"
            ]
          },
          {
            "name": "dominant_signature_direction_gate",
            "role": "gate",
            "field": "$.dominant_signature_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_in_covid"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-8-claude-code",
      "modelName": "claude-opus-4-8",
      "harness": "claude-code",
      "task": "q6_myeloid_subset_restricted_de",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 897,
      "cost": 2.8098360000000002,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "myeloid_subset_with_strongest_change_gate",
            "role": "gate",
            "field": "$.myeloid_subset_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "subset_gamma"
            ]
          },
          {
            "name": "gene_with_strongest_change_gate",
            "role": "gate",
            "field": "$.gene_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "AXL",
              "PROS1"
            ]
          },
          {
            "name": "direction_of_change_gate",
            "role": "gate",
            "field": "$.direction_of_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "down_in_covid"
            ]
          },
          {
            "name": "biological_function_interpretation_gate",
            "role": "gate",
            "field": "$.biological_function_interpretation",
            "kind": "value",
            "op": "in",
            "expected": [
              "efferocytosis",
              "phagocytosis"
            ]
          },
          {
            "name": "myeloid_compartment_abundance_gate",
            "role": "gate",
            "field": "$.myeloid_compartment_abundance",
            "kind": "value",
            "op": "in",
            "expected": [
              "expands_in_covid"
            ]
          },
          {
            "name": "myeloid_expansion_localization_gate",
            "role": "gate",
            "field": "$.myeloid_expansion_localization",
            "kind": "value",
            "op": "in",
            "expected": [
              "recruited_driven_resident_dysfunction"
            ]
          },
          {
            "name": "dominant_signature_gate",
            "role": "gate",
            "field": "$.dominant_signature",
            "kind": "value",
            "op": "in",
            "expected": [
              "technical_or_agonal_artifact"
            ]
          },
          {
            "name": "dominant_signature_direction_gate",
            "role": "gate",
            "field": "$.dominant_signature_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_in_covid"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-8-pi",
      "modelName": "claude-opus-4-8",
      "harness": "pi",
      "task": "q6_myeloid_subset_restricted_de",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 1192,
      "cost": 2.2061542500000004,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "myeloid_subset_with_strongest_change_gate",
            "role": "gate",
            "field": "$.myeloid_subset_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "subset_gamma"
            ]
          },
          {
            "name": "gene_with_strongest_change_gate",
            "role": "gate",
            "field": "$.gene_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "AXL",
              "PROS1"
            ]
          },
          {
            "name": "direction_of_change_gate",
            "role": "gate",
            "field": "$.direction_of_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "down_in_covid"
            ]
          },
          {
            "name": "biological_function_interpretation_gate",
            "role": "gate",
            "field": "$.biological_function_interpretation",
            "kind": "value",
            "op": "in",
            "expected": [
              "efferocytosis",
              "phagocytosis"
            ]
          },
          {
            "name": "myeloid_compartment_abundance_gate",
            "role": "gate",
            "field": "$.myeloid_compartment_abundance",
            "kind": "value",
            "op": "in",
            "expected": [
              "expands_in_covid"
            ]
          },
          {
            "name": "myeloid_expansion_localization_gate",
            "role": "gate",
            "field": "$.myeloid_expansion_localization",
            "kind": "value",
            "op": "in",
            "expected": [
              "recruited_driven_resident_dysfunction"
            ]
          },
          {
            "name": "dominant_signature_gate",
            "role": "gate",
            "field": "$.dominant_signature",
            "kind": "value",
            "op": "in",
            "expected": [
              "technical_or_agonal_artifact"
            ]
          },
          {
            "name": "dominant_signature_direction_gate",
            "role": "gate",
            "field": "$.dominant_signature_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_in_covid"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-8-pi",
      "modelName": "claude-opus-4-8",
      "harness": "pi",
      "task": "q6_myeloid_subset_restricted_de",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 2,
      "passed": true,
      "score": null,
      "durationS": 858,
      "cost": 1.7554220000000003,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "myeloid_subset_with_strongest_change_gate",
            "role": "gate",
            "field": "$.myeloid_subset_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "subset_gamma"
            ]
          },
          {
            "name": "gene_with_strongest_change_gate",
            "role": "gate",
            "field": "$.gene_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "AXL",
              "PROS1"
            ]
          },
          {
            "name": "direction_of_change_gate",
            "role": "gate",
            "field": "$.direction_of_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "down_in_covid"
            ]
          },
          {
            "name": "biological_function_interpretation_gate",
            "role": "gate",
            "field": "$.biological_function_interpretation",
            "kind": "value",
            "op": "in",
            "expected": [
              "efferocytosis",
              "phagocytosis"
            ]
          },
          {
            "name": "myeloid_compartment_abundance_gate",
            "role": "gate",
            "field": "$.myeloid_compartment_abundance",
            "kind": "value",
            "op": "in",
            "expected": [
              "expands_in_covid"
            ]
          },
          {
            "name": "myeloid_expansion_localization_gate",
            "role": "gate",
            "field": "$.myeloid_expansion_localization",
            "kind": "value",
            "op": "in",
            "expected": [
              "recruited_driven_resident_dysfunction"
            ]
          },
          {
            "name": "dominant_signature_gate",
            "role": "gate",
            "field": "$.dominant_signature",
            "kind": "value",
            "op": "in",
            "expected": [
              "technical_or_agonal_artifact"
            ]
          },
          {
            "name": "dominant_signature_direction_gate",
            "role": "gate",
            "field": "$.dominant_signature_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_in_covid"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-8-pi",
      "modelName": "claude-opus-4-8",
      "harness": "pi",
      "task": "q6_myeloid_subset_restricted_de",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1378,
      "cost": 1.727385,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "myeloid_subset_with_strongest_change_gate",
            "role": "gate",
            "field": "$.myeloid_subset_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "subset_gamma"
            ]
          },
          {
            "name": "gene_with_strongest_change_gate",
            "role": "gate",
            "field": "$.gene_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "AXL",
              "PROS1"
            ]
          },
          {
            "name": "direction_of_change_gate",
            "role": "gate",
            "field": "$.direction_of_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "down_in_covid"
            ]
          },
          {
            "name": "biological_function_interpretation_gate",
            "role": "gate",
            "field": "$.biological_function_interpretation",
            "kind": "value",
            "op": "in",
            "expected": [
              "efferocytosis",
              "phagocytosis"
            ]
          },
          {
            "name": "myeloid_compartment_abundance_gate",
            "role": "gate",
            "field": "$.myeloid_compartment_abundance",
            "kind": "value",
            "op": "in",
            "expected": [
              "expands_in_covid"
            ]
          },
          {
            "name": "myeloid_expansion_localization_gate",
            "role": "gate",
            "field": "$.myeloid_expansion_localization",
            "kind": "value",
            "op": "in",
            "expected": [
              "recruited_driven_resident_dysfunction"
            ]
          },
          {
            "name": "dominant_signature_gate",
            "role": "gate",
            "field": "$.dominant_signature",
            "kind": "value",
            "op": "in",
            "expected": [
              "technical_or_agonal_artifact"
            ]
          },
          {
            "name": "dominant_signature_direction_gate",
            "role": "gate",
            "field": "$.dominant_signature_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_in_covid"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-4-6-pi",
      "modelName": "claude-sonnet-4-6",
      "harness": "pi",
      "task": "q6_myeloid_subset_restricted_de",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 2156,
      "cost": 1.9726707,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "myeloid_subset_with_strongest_change_gate",
            "role": "gate",
            "field": "$.myeloid_subset_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "subset_gamma"
            ]
          },
          {
            "name": "gene_with_strongest_change_gate",
            "role": "gate",
            "field": "$.gene_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "AXL",
              "PROS1"
            ]
          },
          {
            "name": "direction_of_change_gate",
            "role": "gate",
            "field": "$.direction_of_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "down_in_covid"
            ]
          },
          {
            "name": "biological_function_interpretation_gate",
            "role": "gate",
            "field": "$.biological_function_interpretation",
            "kind": "value",
            "op": "in",
            "expected": [
              "efferocytosis",
              "phagocytosis"
            ]
          },
          {
            "name": "myeloid_compartment_abundance_gate",
            "role": "gate",
            "field": "$.myeloid_compartment_abundance",
            "kind": "value",
            "op": "in",
            "expected": [
              "expands_in_covid"
            ]
          },
          {
            "name": "myeloid_expansion_localization_gate",
            "role": "gate",
            "field": "$.myeloid_expansion_localization",
            "kind": "value",
            "op": "in",
            "expected": [
              "recruited_driven_resident_dysfunction"
            ]
          },
          {
            "name": "dominant_signature_gate",
            "role": "gate",
            "field": "$.dominant_signature",
            "kind": "value",
            "op": "in",
            "expected": [
              "technical_or_agonal_artifact"
            ]
          },
          {
            "name": "dominant_signature_direction_gate",
            "role": "gate",
            "field": "$.dominant_signature_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_in_covid"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-4-6-pi",
      "modelName": "claude-sonnet-4-6",
      "harness": "pi",
      "task": "q6_myeloid_subset_restricted_de",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 2643,
      "cost": 1.8570616500000003,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "myeloid_subset_with_strongest_change_gate",
            "role": "gate",
            "field": "$.myeloid_subset_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "subset_gamma"
            ]
          },
          {
            "name": "gene_with_strongest_change_gate",
            "role": "gate",
            "field": "$.gene_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "AXL",
              "PROS1"
            ]
          },
          {
            "name": "direction_of_change_gate",
            "role": "gate",
            "field": "$.direction_of_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "down_in_covid"
            ]
          },
          {
            "name": "biological_function_interpretation_gate",
            "role": "gate",
            "field": "$.biological_function_interpretation",
            "kind": "value",
            "op": "in",
            "expected": [
              "efferocytosis",
              "phagocytosis"
            ]
          },
          {
            "name": "myeloid_compartment_abundance_gate",
            "role": "gate",
            "field": "$.myeloid_compartment_abundance",
            "kind": "value",
            "op": "in",
            "expected": [
              "expands_in_covid"
            ]
          },
          {
            "name": "myeloid_expansion_localization_gate",
            "role": "gate",
            "field": "$.myeloid_expansion_localization",
            "kind": "value",
            "op": "in",
            "expected": [
              "recruited_driven_resident_dysfunction"
            ]
          },
          {
            "name": "dominant_signature_gate",
            "role": "gate",
            "field": "$.dominant_signature",
            "kind": "value",
            "op": "in",
            "expected": [
              "technical_or_agonal_artifact"
            ]
          },
          {
            "name": "dominant_signature_direction_gate",
            "role": "gate",
            "field": "$.dominant_signature_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_in_covid"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-4-6-pi",
      "modelName": "claude-sonnet-4-6",
      "harness": "pi",
      "task": "q6_myeloid_subset_restricted_de",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1273,
      "cost": 1.2838840500000002,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "myeloid_subset_with_strongest_change_gate",
            "role": "gate",
            "field": "$.myeloid_subset_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "subset_gamma"
            ]
          },
          {
            "name": "gene_with_strongest_change_gate",
            "role": "gate",
            "field": "$.gene_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "AXL",
              "PROS1"
            ]
          },
          {
            "name": "direction_of_change_gate",
            "role": "gate",
            "field": "$.direction_of_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "down_in_covid"
            ]
          },
          {
            "name": "biological_function_interpretation_gate",
            "role": "gate",
            "field": "$.biological_function_interpretation",
            "kind": "value",
            "op": "in",
            "expected": [
              "efferocytosis",
              "phagocytosis"
            ]
          },
          {
            "name": "myeloid_compartment_abundance_gate",
            "role": "gate",
            "field": "$.myeloid_compartment_abundance",
            "kind": "value",
            "op": "in",
            "expected": [
              "expands_in_covid"
            ]
          },
          {
            "name": "myeloid_expansion_localization_gate",
            "role": "gate",
            "field": "$.myeloid_expansion_localization",
            "kind": "value",
            "op": "in",
            "expected": [
              "recruited_driven_resident_dysfunction"
            ]
          },
          {
            "name": "dominant_signature_gate",
            "role": "gate",
            "field": "$.dominant_signature",
            "kind": "value",
            "op": "in",
            "expected": [
              "technical_or_agonal_artifact"
            ]
          },
          {
            "name": "dominant_signature_direction_gate",
            "role": "gate",
            "field": "$.dominant_signature_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_in_covid"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gemini-3-1-pro-preview-pi",
      "modelName": "gemini-3.1-pro-preview",
      "harness": "pi",
      "task": "q6_myeloid_subset_restricted_de",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 1759,
      "cost": 0.9469228,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "myeloid_subset_with_strongest_change_gate",
            "role": "gate",
            "field": "$.myeloid_subset_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "subset_gamma"
            ]
          },
          {
            "name": "gene_with_strongest_change_gate",
            "role": "gate",
            "field": "$.gene_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "AXL",
              "PROS1"
            ]
          },
          {
            "name": "direction_of_change_gate",
            "role": "gate",
            "field": "$.direction_of_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "down_in_covid"
            ]
          },
          {
            "name": "biological_function_interpretation_gate",
            "role": "gate",
            "field": "$.biological_function_interpretation",
            "kind": "value",
            "op": "in",
            "expected": [
              "efferocytosis",
              "phagocytosis"
            ]
          },
          {
            "name": "myeloid_compartment_abundance_gate",
            "role": "gate",
            "field": "$.myeloid_compartment_abundance",
            "kind": "value",
            "op": "in",
            "expected": [
              "expands_in_covid"
            ]
          },
          {
            "name": "myeloid_expansion_localization_gate",
            "role": "gate",
            "field": "$.myeloid_expansion_localization",
            "kind": "value",
            "op": "in",
            "expected": [
              "recruited_driven_resident_dysfunction"
            ]
          },
          {
            "name": "dominant_signature_gate",
            "role": "gate",
            "field": "$.dominant_signature",
            "kind": "value",
            "op": "in",
            "expected": [
              "technical_or_agonal_artifact"
            ]
          },
          {
            "name": "dominant_signature_direction_gate",
            "role": "gate",
            "field": "$.dominant_signature_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_in_covid"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gemini-3-1-pro-preview-pi",
      "modelName": "gemini-3.1-pro-preview",
      "harness": "pi",
      "task": "q6_myeloid_subset_restricted_de",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 691,
      "cost": 1.0687454000000003,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "myeloid_subset_with_strongest_change_gate",
            "role": "gate",
            "field": "$.myeloid_subset_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "subset_gamma"
            ]
          },
          {
            "name": "gene_with_strongest_change_gate",
            "role": "gate",
            "field": "$.gene_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "AXL",
              "PROS1"
            ]
          },
          {
            "name": "direction_of_change_gate",
            "role": "gate",
            "field": "$.direction_of_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "down_in_covid"
            ]
          },
          {
            "name": "biological_function_interpretation_gate",
            "role": "gate",
            "field": "$.biological_function_interpretation",
            "kind": "value",
            "op": "in",
            "expected": [
              "efferocytosis",
              "phagocytosis"
            ]
          },
          {
            "name": "myeloid_compartment_abundance_gate",
            "role": "gate",
            "field": "$.myeloid_compartment_abundance",
            "kind": "value",
            "op": "in",
            "expected": [
              "expands_in_covid"
            ]
          },
          {
            "name": "myeloid_expansion_localization_gate",
            "role": "gate",
            "field": "$.myeloid_expansion_localization",
            "kind": "value",
            "op": "in",
            "expected": [
              "recruited_driven_resident_dysfunction"
            ]
          },
          {
            "name": "dominant_signature_gate",
            "role": "gate",
            "field": "$.dominant_signature",
            "kind": "value",
            "op": "in",
            "expected": [
              "technical_or_agonal_artifact"
            ]
          },
          {
            "name": "dominant_signature_direction_gate",
            "role": "gate",
            "field": "$.dominant_signature_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_in_covid"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gemini-3-1-pro-preview-pi",
      "modelName": "gemini-3.1-pro-preview",
      "harness": "pi",
      "task": "q6_myeloid_subset_restricted_de",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1040,
      "cost": 0.5102418,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "myeloid_subset_with_strongest_change_gate",
            "role": "gate",
            "field": "$.myeloid_subset_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "subset_gamma"
            ]
          },
          {
            "name": "gene_with_strongest_change_gate",
            "role": "gate",
            "field": "$.gene_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "AXL",
              "PROS1"
            ]
          },
          {
            "name": "direction_of_change_gate",
            "role": "gate",
            "field": "$.direction_of_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "down_in_covid"
            ]
          },
          {
            "name": "biological_function_interpretation_gate",
            "role": "gate",
            "field": "$.biological_function_interpretation",
            "kind": "value",
            "op": "in",
            "expected": [
              "efferocytosis",
              "phagocytosis"
            ]
          },
          {
            "name": "myeloid_compartment_abundance_gate",
            "role": "gate",
            "field": "$.myeloid_compartment_abundance",
            "kind": "value",
            "op": "in",
            "expected": [
              "expands_in_covid"
            ]
          },
          {
            "name": "myeloid_expansion_localization_gate",
            "role": "gate",
            "field": "$.myeloid_expansion_localization",
            "kind": "value",
            "op": "in",
            "expected": [
              "recruited_driven_resident_dysfunction"
            ]
          },
          {
            "name": "dominant_signature_gate",
            "role": "gate",
            "field": "$.dominant_signature",
            "kind": "value",
            "op": "in",
            "expected": [
              "technical_or_agonal_artifact"
            ]
          },
          {
            "name": "dominant_signature_direction_gate",
            "role": "gate",
            "field": "$.dominant_signature_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_in_covid"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gemini-3-5-flash-pi",
      "modelName": "gemini-3.5-flash",
      "harness": "pi",
      "task": "q6_myeloid_subset_restricted_de",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 1520,
      "cost": 0.9948516000000002,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "myeloid_subset_with_strongest_change_gate",
            "role": "gate",
            "field": "$.myeloid_subset_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "subset_gamma"
            ]
          },
          {
            "name": "gene_with_strongest_change_gate",
            "role": "gate",
            "field": "$.gene_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "AXL",
              "PROS1"
            ]
          },
          {
            "name": "direction_of_change_gate",
            "role": "gate",
            "field": "$.direction_of_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "down_in_covid"
            ]
          },
          {
            "name": "biological_function_interpretation_gate",
            "role": "gate",
            "field": "$.biological_function_interpretation",
            "kind": "value",
            "op": "in",
            "expected": [
              "efferocytosis",
              "phagocytosis"
            ]
          },
          {
            "name": "myeloid_compartment_abundance_gate",
            "role": "gate",
            "field": "$.myeloid_compartment_abundance",
            "kind": "value",
            "op": "in",
            "expected": [
              "expands_in_covid"
            ]
          },
          {
            "name": "myeloid_expansion_localization_gate",
            "role": "gate",
            "field": "$.myeloid_expansion_localization",
            "kind": "value",
            "op": "in",
            "expected": [
              "recruited_driven_resident_dysfunction"
            ]
          },
          {
            "name": "dominant_signature_gate",
            "role": "gate",
            "field": "$.dominant_signature",
            "kind": "value",
            "op": "in",
            "expected": [
              "technical_or_agonal_artifact"
            ]
          },
          {
            "name": "dominant_signature_direction_gate",
            "role": "gate",
            "field": "$.dominant_signature_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_in_covid"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gemini-3-5-flash-pi",
      "modelName": "gemini-3.5-flash",
      "harness": "pi",
      "task": "q6_myeloid_subset_restricted_de",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 886,
      "cost": 0.5890602000000001,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "myeloid_subset_with_strongest_change_gate",
            "role": "gate",
            "field": "$.myeloid_subset_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "subset_gamma"
            ]
          },
          {
            "name": "gene_with_strongest_change_gate",
            "role": "gate",
            "field": "$.gene_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "AXL",
              "PROS1"
            ]
          },
          {
            "name": "direction_of_change_gate",
            "role": "gate",
            "field": "$.direction_of_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "down_in_covid"
            ]
          },
          {
            "name": "biological_function_interpretation_gate",
            "role": "gate",
            "field": "$.biological_function_interpretation",
            "kind": "value",
            "op": "in",
            "expected": [
              "efferocytosis",
              "phagocytosis"
            ]
          },
          {
            "name": "myeloid_compartment_abundance_gate",
            "role": "gate",
            "field": "$.myeloid_compartment_abundance",
            "kind": "value",
            "op": "in",
            "expected": [
              "expands_in_covid"
            ]
          },
          {
            "name": "myeloid_expansion_localization_gate",
            "role": "gate",
            "field": "$.myeloid_expansion_localization",
            "kind": "value",
            "op": "in",
            "expected": [
              "recruited_driven_resident_dysfunction"
            ]
          },
          {
            "name": "dominant_signature_gate",
            "role": "gate",
            "field": "$.dominant_signature",
            "kind": "value",
            "op": "in",
            "expected": [
              "technical_or_agonal_artifact"
            ]
          },
          {
            "name": "dominant_signature_direction_gate",
            "role": "gate",
            "field": "$.dominant_signature_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_in_covid"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gemini-3-5-flash-pi",
      "modelName": "gemini-3.5-flash",
      "harness": "pi",
      "task": "q6_myeloid_subset_restricted_de",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 3,
      "passed": true,
      "score": null,
      "durationS": 1068,
      "cost": 0.7921611,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "myeloid_subset_with_strongest_change_gate",
            "role": "gate",
            "field": "$.myeloid_subset_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "subset_gamma"
            ]
          },
          {
            "name": "gene_with_strongest_change_gate",
            "role": "gate",
            "field": "$.gene_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "AXL",
              "PROS1"
            ]
          },
          {
            "name": "direction_of_change_gate",
            "role": "gate",
            "field": "$.direction_of_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "down_in_covid"
            ]
          },
          {
            "name": "biological_function_interpretation_gate",
            "role": "gate",
            "field": "$.biological_function_interpretation",
            "kind": "value",
            "op": "in",
            "expected": [
              "efferocytosis",
              "phagocytosis"
            ]
          },
          {
            "name": "myeloid_compartment_abundance_gate",
            "role": "gate",
            "field": "$.myeloid_compartment_abundance",
            "kind": "value",
            "op": "in",
            "expected": [
              "expands_in_covid"
            ]
          },
          {
            "name": "myeloid_expansion_localization_gate",
            "role": "gate",
            "field": "$.myeloid_expansion_localization",
            "kind": "value",
            "op": "in",
            "expected": [
              "recruited_driven_resident_dysfunction"
            ]
          },
          {
            "name": "dominant_signature_gate",
            "role": "gate",
            "field": "$.dominant_signature",
            "kind": "value",
            "op": "in",
            "expected": [
              "technical_or_agonal_artifact"
            ]
          },
          {
            "name": "dominant_signature_direction_gate",
            "role": "gate",
            "field": "$.dominant_signature_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_in_covid"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-glm-5p2-pi",
      "modelName": "glm-5p2",
      "harness": "pi",
      "task": "q6_myeloid_subset_restricted_de",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 5365,
      "cost": 2.2082368199999993,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "myeloid_subset_with_strongest_change_gate",
            "role": "gate",
            "field": "$.myeloid_subset_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "subset_gamma"
            ]
          },
          {
            "name": "gene_with_strongest_change_gate",
            "role": "gate",
            "field": "$.gene_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "AXL",
              "PROS1"
            ]
          },
          {
            "name": "direction_of_change_gate",
            "role": "gate",
            "field": "$.direction_of_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "down_in_covid"
            ]
          },
          {
            "name": "biological_function_interpretation_gate",
            "role": "gate",
            "field": "$.biological_function_interpretation",
            "kind": "value",
            "op": "in",
            "expected": [
              "efferocytosis",
              "phagocytosis"
            ]
          },
          {
            "name": "myeloid_compartment_abundance_gate",
            "role": "gate",
            "field": "$.myeloid_compartment_abundance",
            "kind": "value",
            "op": "in",
            "expected": [
              "expands_in_covid"
            ]
          },
          {
            "name": "myeloid_expansion_localization_gate",
            "role": "gate",
            "field": "$.myeloid_expansion_localization",
            "kind": "value",
            "op": "in",
            "expected": [
              "recruited_driven_resident_dysfunction"
            ]
          },
          {
            "name": "dominant_signature_gate",
            "role": "gate",
            "field": "$.dominant_signature",
            "kind": "value",
            "op": "in",
            "expected": [
              "technical_or_agonal_artifact"
            ]
          },
          {
            "name": "dominant_signature_direction_gate",
            "role": "gate",
            "field": "$.dominant_signature_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_in_covid"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-glm-5p2-pi",
      "modelName": "glm-5p2",
      "harness": "pi",
      "task": "q6_myeloid_subset_restricted_de",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 3845,
      "cost": 2.0767889,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "myeloid_subset_with_strongest_change_gate",
            "role": "gate",
            "field": "$.myeloid_subset_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "subset_gamma"
            ]
          },
          {
            "name": "gene_with_strongest_change_gate",
            "role": "gate",
            "field": "$.gene_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "AXL",
              "PROS1"
            ]
          },
          {
            "name": "direction_of_change_gate",
            "role": "gate",
            "field": "$.direction_of_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "down_in_covid"
            ]
          },
          {
            "name": "biological_function_interpretation_gate",
            "role": "gate",
            "field": "$.biological_function_interpretation",
            "kind": "value",
            "op": "in",
            "expected": [
              "efferocytosis",
              "phagocytosis"
            ]
          },
          {
            "name": "myeloid_compartment_abundance_gate",
            "role": "gate",
            "field": "$.myeloid_compartment_abundance",
            "kind": "value",
            "op": "in",
            "expected": [
              "expands_in_covid"
            ]
          },
          {
            "name": "myeloid_expansion_localization_gate",
            "role": "gate",
            "field": "$.myeloid_expansion_localization",
            "kind": "value",
            "op": "in",
            "expected": [
              "recruited_driven_resident_dysfunction"
            ]
          },
          {
            "name": "dominant_signature_gate",
            "role": "gate",
            "field": "$.dominant_signature",
            "kind": "value",
            "op": "in",
            "expected": [
              "technical_or_agonal_artifact"
            ]
          },
          {
            "name": "dominant_signature_direction_gate",
            "role": "gate",
            "field": "$.dominant_signature_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_in_covid"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-glm-5p2-pi",
      "modelName": "glm-5p2",
      "harness": "pi",
      "task": "q6_myeloid_subset_restricted_de",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 6003,
      "cost": 2.473176539999999,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "myeloid_subset_with_strongest_change_gate",
            "role": "gate",
            "field": "$.myeloid_subset_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "subset_gamma"
            ]
          },
          {
            "name": "gene_with_strongest_change_gate",
            "role": "gate",
            "field": "$.gene_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "AXL",
              "PROS1"
            ]
          },
          {
            "name": "direction_of_change_gate",
            "role": "gate",
            "field": "$.direction_of_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "down_in_covid"
            ]
          },
          {
            "name": "biological_function_interpretation_gate",
            "role": "gate",
            "field": "$.biological_function_interpretation",
            "kind": "value",
            "op": "in",
            "expected": [
              "efferocytosis",
              "phagocytosis"
            ]
          },
          {
            "name": "myeloid_compartment_abundance_gate",
            "role": "gate",
            "field": "$.myeloid_compartment_abundance",
            "kind": "value",
            "op": "in",
            "expected": [
              "expands_in_covid"
            ]
          },
          {
            "name": "myeloid_expansion_localization_gate",
            "role": "gate",
            "field": "$.myeloid_expansion_localization",
            "kind": "value",
            "op": "in",
            "expected": [
              "recruited_driven_resident_dysfunction"
            ]
          },
          {
            "name": "dominant_signature_gate",
            "role": "gate",
            "field": "$.dominant_signature",
            "kind": "value",
            "op": "in",
            "expected": [
              "technical_or_agonal_artifact"
            ]
          },
          {
            "name": "dominant_signature_direction_gate",
            "role": "gate",
            "field": "$.dominant_signature_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_in_covid"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-4-codex",
      "modelName": "gpt-5.4",
      "harness": "openai-codex",
      "task": "q6_myeloid_subset_restricted_de",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 883,
      "cost": 2.46579,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "myeloid_subset_with_strongest_change_gate",
            "role": "gate",
            "field": "$.myeloid_subset_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "subset_gamma"
            ]
          },
          {
            "name": "gene_with_strongest_change_gate",
            "role": "gate",
            "field": "$.gene_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "AXL",
              "PROS1"
            ]
          },
          {
            "name": "direction_of_change_gate",
            "role": "gate",
            "field": "$.direction_of_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "down_in_covid"
            ]
          },
          {
            "name": "biological_function_interpretation_gate",
            "role": "gate",
            "field": "$.biological_function_interpretation",
            "kind": "value",
            "op": "in",
            "expected": [
              "efferocytosis",
              "phagocytosis"
            ]
          },
          {
            "name": "myeloid_compartment_abundance_gate",
            "role": "gate",
            "field": "$.myeloid_compartment_abundance",
            "kind": "value",
            "op": "in",
            "expected": [
              "expands_in_covid"
            ]
          },
          {
            "name": "myeloid_expansion_localization_gate",
            "role": "gate",
            "field": "$.myeloid_expansion_localization",
            "kind": "value",
            "op": "in",
            "expected": [
              "recruited_driven_resident_dysfunction"
            ]
          },
          {
            "name": "dominant_signature_gate",
            "role": "gate",
            "field": "$.dominant_signature",
            "kind": "value",
            "op": "in",
            "expected": [
              "technical_or_agonal_artifact"
            ]
          },
          {
            "name": "dominant_signature_direction_gate",
            "role": "gate",
            "field": "$.dominant_signature_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_in_covid"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-4-codex",
      "modelName": "gpt-5.4",
      "harness": "openai-codex",
      "task": "q6_myeloid_subset_restricted_de",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 646,
      "cost": 1.1252250000000001,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "myeloid_subset_with_strongest_change_gate",
            "role": "gate",
            "field": "$.myeloid_subset_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "subset_gamma"
            ]
          },
          {
            "name": "gene_with_strongest_change_gate",
            "role": "gate",
            "field": "$.gene_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "AXL",
              "PROS1"
            ]
          },
          {
            "name": "direction_of_change_gate",
            "role": "gate",
            "field": "$.direction_of_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "down_in_covid"
            ]
          },
          {
            "name": "biological_function_interpretation_gate",
            "role": "gate",
            "field": "$.biological_function_interpretation",
            "kind": "value",
            "op": "in",
            "expected": [
              "efferocytosis",
              "phagocytosis"
            ]
          },
          {
            "name": "myeloid_compartment_abundance_gate",
            "role": "gate",
            "field": "$.myeloid_compartment_abundance",
            "kind": "value",
            "op": "in",
            "expected": [
              "expands_in_covid"
            ]
          },
          {
            "name": "myeloid_expansion_localization_gate",
            "role": "gate",
            "field": "$.myeloid_expansion_localization",
            "kind": "value",
            "op": "in",
            "expected": [
              "recruited_driven_resident_dysfunction"
            ]
          },
          {
            "name": "dominant_signature_gate",
            "role": "gate",
            "field": "$.dominant_signature",
            "kind": "value",
            "op": "in",
            "expected": [
              "technical_or_agonal_artifact"
            ]
          },
          {
            "name": "dominant_signature_direction_gate",
            "role": "gate",
            "field": "$.dominant_signature_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_in_covid"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-4-codex",
      "modelName": "gpt-5.4",
      "harness": "openai-codex",
      "task": "q6_myeloid_subset_restricted_de",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1357,
      "cost": 2.3055135,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "myeloid_subset_with_strongest_change_gate",
            "role": "gate",
            "field": "$.myeloid_subset_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "subset_gamma"
            ]
          },
          {
            "name": "gene_with_strongest_change_gate",
            "role": "gate",
            "field": "$.gene_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "AXL",
              "PROS1"
            ]
          },
          {
            "name": "direction_of_change_gate",
            "role": "gate",
            "field": "$.direction_of_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "down_in_covid"
            ]
          },
          {
            "name": "biological_function_interpretation_gate",
            "role": "gate",
            "field": "$.biological_function_interpretation",
            "kind": "value",
            "op": "in",
            "expected": [
              "efferocytosis",
              "phagocytosis"
            ]
          },
          {
            "name": "myeloid_compartment_abundance_gate",
            "role": "gate",
            "field": "$.myeloid_compartment_abundance",
            "kind": "value",
            "op": "in",
            "expected": [
              "expands_in_covid"
            ]
          },
          {
            "name": "myeloid_expansion_localization_gate",
            "role": "gate",
            "field": "$.myeloid_expansion_localization",
            "kind": "value",
            "op": "in",
            "expected": [
              "recruited_driven_resident_dysfunction"
            ]
          },
          {
            "name": "dominant_signature_gate",
            "role": "gate",
            "field": "$.dominant_signature",
            "kind": "value",
            "op": "in",
            "expected": [
              "technical_or_agonal_artifact"
            ]
          },
          {
            "name": "dominant_signature_direction_gate",
            "role": "gate",
            "field": "$.dominant_signature_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_in_covid"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-4-pi",
      "modelName": "gpt-5.4",
      "harness": "pi",
      "task": "q6_myeloid_subset_restricted_de",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 1,
      "passed": true,
      "score": null,
      "durationS": 2358,
      "cost": 1.188432,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "myeloid_subset_with_strongest_change_gate",
            "role": "gate",
            "field": "$.myeloid_subset_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "subset_gamma"
            ]
          },
          {
            "name": "gene_with_strongest_change_gate",
            "role": "gate",
            "field": "$.gene_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "AXL",
              "PROS1"
            ]
          },
          {
            "name": "direction_of_change_gate",
            "role": "gate",
            "field": "$.direction_of_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "down_in_covid"
            ]
          },
          {
            "name": "biological_function_interpretation_gate",
            "role": "gate",
            "field": "$.biological_function_interpretation",
            "kind": "value",
            "op": "in",
            "expected": [
              "efferocytosis",
              "phagocytosis"
            ]
          },
          {
            "name": "myeloid_compartment_abundance_gate",
            "role": "gate",
            "field": "$.myeloid_compartment_abundance",
            "kind": "value",
            "op": "in",
            "expected": [
              "expands_in_covid"
            ]
          },
          {
            "name": "myeloid_expansion_localization_gate",
            "role": "gate",
            "field": "$.myeloid_expansion_localization",
            "kind": "value",
            "op": "in",
            "expected": [
              "recruited_driven_resident_dysfunction"
            ]
          },
          {
            "name": "dominant_signature_gate",
            "role": "gate",
            "field": "$.dominant_signature",
            "kind": "value",
            "op": "in",
            "expected": [
              "technical_or_agonal_artifact"
            ]
          },
          {
            "name": "dominant_signature_direction_gate",
            "role": "gate",
            "field": "$.dominant_signature_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_in_covid"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-4-pi",
      "modelName": "gpt-5.4",
      "harness": "pi",
      "task": "q6_myeloid_subset_restricted_de",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 5714,
      "cost": 1.1316164999999998,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "myeloid_subset_with_strongest_change_gate",
            "role": "gate",
            "field": "$.myeloid_subset_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "subset_gamma"
            ]
          },
          {
            "name": "gene_with_strongest_change_gate",
            "role": "gate",
            "field": "$.gene_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "AXL",
              "PROS1"
            ]
          },
          {
            "name": "direction_of_change_gate",
            "role": "gate",
            "field": "$.direction_of_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "down_in_covid"
            ]
          },
          {
            "name": "biological_function_interpretation_gate",
            "role": "gate",
            "field": "$.biological_function_interpretation",
            "kind": "value",
            "op": "in",
            "expected": [
              "efferocytosis",
              "phagocytosis"
            ]
          },
          {
            "name": "myeloid_compartment_abundance_gate",
            "role": "gate",
            "field": "$.myeloid_compartment_abundance",
            "kind": "value",
            "op": "in",
            "expected": [
              "expands_in_covid"
            ]
          },
          {
            "name": "myeloid_expansion_localization_gate",
            "role": "gate",
            "field": "$.myeloid_expansion_localization",
            "kind": "value",
            "op": "in",
            "expected": [
              "recruited_driven_resident_dysfunction"
            ]
          },
          {
            "name": "dominant_signature_gate",
            "role": "gate",
            "field": "$.dominant_signature",
            "kind": "value",
            "op": "in",
            "expected": [
              "technical_or_agonal_artifact"
            ]
          },
          {
            "name": "dominant_signature_direction_gate",
            "role": "gate",
            "field": "$.dominant_signature_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_in_covid"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-4-pi",
      "modelName": "gpt-5.4",
      "harness": "pi",
      "task": "q6_myeloid_subset_restricted_de",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 2204,
      "cost": 1.131359,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "myeloid_subset_with_strongest_change_gate",
            "role": "gate",
            "field": "$.myeloid_subset_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "subset_gamma"
            ]
          },
          {
            "name": "gene_with_strongest_change_gate",
            "role": "gate",
            "field": "$.gene_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "AXL",
              "PROS1"
            ]
          },
          {
            "name": "direction_of_change_gate",
            "role": "gate",
            "field": "$.direction_of_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "down_in_covid"
            ]
          },
          {
            "name": "biological_function_interpretation_gate",
            "role": "gate",
            "field": "$.biological_function_interpretation",
            "kind": "value",
            "op": "in",
            "expected": [
              "efferocytosis",
              "phagocytosis"
            ]
          },
          {
            "name": "myeloid_compartment_abundance_gate",
            "role": "gate",
            "field": "$.myeloid_compartment_abundance",
            "kind": "value",
            "op": "in",
            "expected": [
              "expands_in_covid"
            ]
          },
          {
            "name": "myeloid_expansion_localization_gate",
            "role": "gate",
            "field": "$.myeloid_expansion_localization",
            "kind": "value",
            "op": "in",
            "expected": [
              "recruited_driven_resident_dysfunction"
            ]
          },
          {
            "name": "dominant_signature_gate",
            "role": "gate",
            "field": "$.dominant_signature",
            "kind": "value",
            "op": "in",
            "expected": [
              "technical_or_agonal_artifact"
            ]
          },
          {
            "name": "dominant_signature_direction_gate",
            "role": "gate",
            "field": "$.dominant_signature_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_in_covid"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-5-codex",
      "modelName": "gpt-5.5",
      "harness": "openai-codex",
      "task": "q6_myeloid_subset_restricted_de",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 1,
      "passed": true,
      "score": null,
      "durationS": 579,
      "cost": 1.584889,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "myeloid_subset_with_strongest_change_gate",
            "role": "gate",
            "field": "$.myeloid_subset_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "subset_gamma"
            ]
          },
          {
            "name": "gene_with_strongest_change_gate",
            "role": "gate",
            "field": "$.gene_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "AXL",
              "PROS1"
            ]
          },
          {
            "name": "direction_of_change_gate",
            "role": "gate",
            "field": "$.direction_of_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "down_in_covid"
            ]
          },
          {
            "name": "biological_function_interpretation_gate",
            "role": "gate",
            "field": "$.biological_function_interpretation",
            "kind": "value",
            "op": "in",
            "expected": [
              "efferocytosis",
              "phagocytosis"
            ]
          },
          {
            "name": "myeloid_compartment_abundance_gate",
            "role": "gate",
            "field": "$.myeloid_compartment_abundance",
            "kind": "value",
            "op": "in",
            "expected": [
              "expands_in_covid"
            ]
          },
          {
            "name": "myeloid_expansion_localization_gate",
            "role": "gate",
            "field": "$.myeloid_expansion_localization",
            "kind": "value",
            "op": "in",
            "expected": [
              "recruited_driven_resident_dysfunction"
            ]
          },
          {
            "name": "dominant_signature_gate",
            "role": "gate",
            "field": "$.dominant_signature",
            "kind": "value",
            "op": "in",
            "expected": [
              "technical_or_agonal_artifact"
            ]
          },
          {
            "name": "dominant_signature_direction_gate",
            "role": "gate",
            "field": "$.dominant_signature_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_in_covid"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-5-codex",
      "modelName": "gpt-5.5",
      "harness": "openai-codex",
      "task": "q6_myeloid_subset_restricted_de",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 1028,
      "cost": 2.161449,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "myeloid_subset_with_strongest_change_gate",
            "role": "gate",
            "field": "$.myeloid_subset_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "subset_gamma"
            ]
          },
          {
            "name": "gene_with_strongest_change_gate",
            "role": "gate",
            "field": "$.gene_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "AXL",
              "PROS1"
            ]
          },
          {
            "name": "direction_of_change_gate",
            "role": "gate",
            "field": "$.direction_of_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "down_in_covid"
            ]
          },
          {
            "name": "biological_function_interpretation_gate",
            "role": "gate",
            "field": "$.biological_function_interpretation",
            "kind": "value",
            "op": "in",
            "expected": [
              "efferocytosis",
              "phagocytosis"
            ]
          },
          {
            "name": "myeloid_compartment_abundance_gate",
            "role": "gate",
            "field": "$.myeloid_compartment_abundance",
            "kind": "value",
            "op": "in",
            "expected": [
              "expands_in_covid"
            ]
          },
          {
            "name": "myeloid_expansion_localization_gate",
            "role": "gate",
            "field": "$.myeloid_expansion_localization",
            "kind": "value",
            "op": "in",
            "expected": [
              "recruited_driven_resident_dysfunction"
            ]
          },
          {
            "name": "dominant_signature_gate",
            "role": "gate",
            "field": "$.dominant_signature",
            "kind": "value",
            "op": "in",
            "expected": [
              "technical_or_agonal_artifact"
            ]
          },
          {
            "name": "dominant_signature_direction_gate",
            "role": "gate",
            "field": "$.dominant_signature_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_in_covid"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-5-codex",
      "modelName": "gpt-5.5",
      "harness": "openai-codex",
      "task": "q6_myeloid_subset_restricted_de",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 647,
      "cost": 2.232822,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "myeloid_subset_with_strongest_change_gate",
            "role": "gate",
            "field": "$.myeloid_subset_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "subset_gamma"
            ]
          },
          {
            "name": "gene_with_strongest_change_gate",
            "role": "gate",
            "field": "$.gene_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "AXL",
              "PROS1"
            ]
          },
          {
            "name": "direction_of_change_gate",
            "role": "gate",
            "field": "$.direction_of_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "down_in_covid"
            ]
          },
          {
            "name": "biological_function_interpretation_gate",
            "role": "gate",
            "field": "$.biological_function_interpretation",
            "kind": "value",
            "op": "in",
            "expected": [
              "efferocytosis",
              "phagocytosis"
            ]
          },
          {
            "name": "myeloid_compartment_abundance_gate",
            "role": "gate",
            "field": "$.myeloid_compartment_abundance",
            "kind": "value",
            "op": "in",
            "expected": [
              "expands_in_covid"
            ]
          },
          {
            "name": "myeloid_expansion_localization_gate",
            "role": "gate",
            "field": "$.myeloid_expansion_localization",
            "kind": "value",
            "op": "in",
            "expected": [
              "recruited_driven_resident_dysfunction"
            ]
          },
          {
            "name": "dominant_signature_gate",
            "role": "gate",
            "field": "$.dominant_signature",
            "kind": "value",
            "op": "in",
            "expected": [
              "technical_or_agonal_artifact"
            ]
          },
          {
            "name": "dominant_signature_direction_gate",
            "role": "gate",
            "field": "$.dominant_signature_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_in_covid"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-5-pi",
      "modelName": "gpt-5.5",
      "harness": "pi",
      "task": "q6_myeloid_subset_restricted_de",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 1,
      "passed": true,
      "score": null,
      "durationS": 1469,
      "cost": 1.6275900000000003,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "myeloid_subset_with_strongest_change_gate",
            "role": "gate",
            "field": "$.myeloid_subset_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "subset_gamma"
            ]
          },
          {
            "name": "gene_with_strongest_change_gate",
            "role": "gate",
            "field": "$.gene_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "AXL",
              "PROS1"
            ]
          },
          {
            "name": "direction_of_change_gate",
            "role": "gate",
            "field": "$.direction_of_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "down_in_covid"
            ]
          },
          {
            "name": "biological_function_interpretation_gate",
            "role": "gate",
            "field": "$.biological_function_interpretation",
            "kind": "value",
            "op": "in",
            "expected": [
              "efferocytosis",
              "phagocytosis"
            ]
          },
          {
            "name": "myeloid_compartment_abundance_gate",
            "role": "gate",
            "field": "$.myeloid_compartment_abundance",
            "kind": "value",
            "op": "in",
            "expected": [
              "expands_in_covid"
            ]
          },
          {
            "name": "myeloid_expansion_localization_gate",
            "role": "gate",
            "field": "$.myeloid_expansion_localization",
            "kind": "value",
            "op": "in",
            "expected": [
              "recruited_driven_resident_dysfunction"
            ]
          },
          {
            "name": "dominant_signature_gate",
            "role": "gate",
            "field": "$.dominant_signature",
            "kind": "value",
            "op": "in",
            "expected": [
              "technical_or_agonal_artifact"
            ]
          },
          {
            "name": "dominant_signature_direction_gate",
            "role": "gate",
            "field": "$.dominant_signature_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_in_covid"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-5-pi",
      "modelName": "gpt-5.5",
      "harness": "pi",
      "task": "q6_myeloid_subset_restricted_de",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 1240,
      "cost": 1.6197300000000003,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "myeloid_subset_with_strongest_change_gate",
            "role": "gate",
            "field": "$.myeloid_subset_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "subset_gamma"
            ]
          },
          {
            "name": "gene_with_strongest_change_gate",
            "role": "gate",
            "field": "$.gene_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "AXL",
              "PROS1"
            ]
          },
          {
            "name": "direction_of_change_gate",
            "role": "gate",
            "field": "$.direction_of_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "down_in_covid"
            ]
          },
          {
            "name": "biological_function_interpretation_gate",
            "role": "gate",
            "field": "$.biological_function_interpretation",
            "kind": "value",
            "op": "in",
            "expected": [
              "efferocytosis",
              "phagocytosis"
            ]
          },
          {
            "name": "myeloid_compartment_abundance_gate",
            "role": "gate",
            "field": "$.myeloid_compartment_abundance",
            "kind": "value",
            "op": "in",
            "expected": [
              "expands_in_covid"
            ]
          },
          {
            "name": "myeloid_expansion_localization_gate",
            "role": "gate",
            "field": "$.myeloid_expansion_localization",
            "kind": "value",
            "op": "in",
            "expected": [
              "recruited_driven_resident_dysfunction"
            ]
          },
          {
            "name": "dominant_signature_gate",
            "role": "gate",
            "field": "$.dominant_signature",
            "kind": "value",
            "op": "in",
            "expected": [
              "technical_or_agonal_artifact"
            ]
          },
          {
            "name": "dominant_signature_direction_gate",
            "role": "gate",
            "field": "$.dominant_signature_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_in_covid"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-5-pi",
      "modelName": "gpt-5.5",
      "harness": "pi",
      "task": "q6_myeloid_subset_restricted_de",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1399,
      "cost": 2.2809830000000004,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "myeloid_subset_with_strongest_change_gate",
            "role": "gate",
            "field": "$.myeloid_subset_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "subset_gamma"
            ]
          },
          {
            "name": "gene_with_strongest_change_gate",
            "role": "gate",
            "field": "$.gene_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "AXL",
              "PROS1"
            ]
          },
          {
            "name": "direction_of_change_gate",
            "role": "gate",
            "field": "$.direction_of_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "down_in_covid"
            ]
          },
          {
            "name": "biological_function_interpretation_gate",
            "role": "gate",
            "field": "$.biological_function_interpretation",
            "kind": "value",
            "op": "in",
            "expected": [
              "efferocytosis",
              "phagocytosis"
            ]
          },
          {
            "name": "myeloid_compartment_abundance_gate",
            "role": "gate",
            "field": "$.myeloid_compartment_abundance",
            "kind": "value",
            "op": "in",
            "expected": [
              "expands_in_covid"
            ]
          },
          {
            "name": "myeloid_expansion_localization_gate",
            "role": "gate",
            "field": "$.myeloid_expansion_localization",
            "kind": "value",
            "op": "in",
            "expected": [
              "recruited_driven_resident_dysfunction"
            ]
          },
          {
            "name": "dominant_signature_gate",
            "role": "gate",
            "field": "$.dominant_signature",
            "kind": "value",
            "op": "in",
            "expected": [
              "technical_or_agonal_artifact"
            ]
          },
          {
            "name": "dominant_signature_direction_gate",
            "role": "gate",
            "field": "$.dominant_signature_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_in_covid"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-20-beta-reasoning-pi",
      "modelName": "grok-4.20-beta-0309-reasoning",
      "harness": "pi",
      "task": "q6_myeloid_subset_restricted_de",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 383,
      "cost": 0.12980930000000002,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "myeloid_subset_with_strongest_change_gate",
            "role": "gate",
            "field": "$.myeloid_subset_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "subset_gamma"
            ]
          },
          {
            "name": "gene_with_strongest_change_gate",
            "role": "gate",
            "field": "$.gene_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "AXL",
              "PROS1"
            ]
          },
          {
            "name": "direction_of_change_gate",
            "role": "gate",
            "field": "$.direction_of_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "down_in_covid"
            ]
          },
          {
            "name": "biological_function_interpretation_gate",
            "role": "gate",
            "field": "$.biological_function_interpretation",
            "kind": "value",
            "op": "in",
            "expected": [
              "efferocytosis",
              "phagocytosis"
            ]
          },
          {
            "name": "myeloid_compartment_abundance_gate",
            "role": "gate",
            "field": "$.myeloid_compartment_abundance",
            "kind": "value",
            "op": "in",
            "expected": [
              "expands_in_covid"
            ]
          },
          {
            "name": "myeloid_expansion_localization_gate",
            "role": "gate",
            "field": "$.myeloid_expansion_localization",
            "kind": "value",
            "op": "in",
            "expected": [
              "recruited_driven_resident_dysfunction"
            ]
          },
          {
            "name": "dominant_signature_gate",
            "role": "gate",
            "field": "$.dominant_signature",
            "kind": "value",
            "op": "in",
            "expected": [
              "technical_or_agonal_artifact"
            ]
          },
          {
            "name": "dominant_signature_direction_gate",
            "role": "gate",
            "field": "$.dominant_signature_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_in_covid"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-20-beta-reasoning-pi",
      "modelName": "grok-4.20-beta-0309-reasoning",
      "harness": "pi",
      "task": "q6_myeloid_subset_restricted_de",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 781,
      "cost": 0.06162810000000001,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "myeloid_subset_with_strongest_change_gate",
            "role": "gate",
            "field": "$.myeloid_subset_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "subset_gamma"
            ]
          },
          {
            "name": "gene_with_strongest_change_gate",
            "role": "gate",
            "field": "$.gene_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "AXL",
              "PROS1"
            ]
          },
          {
            "name": "direction_of_change_gate",
            "role": "gate",
            "field": "$.direction_of_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "down_in_covid"
            ]
          },
          {
            "name": "biological_function_interpretation_gate",
            "role": "gate",
            "field": "$.biological_function_interpretation",
            "kind": "value",
            "op": "in",
            "expected": [
              "efferocytosis",
              "phagocytosis"
            ]
          },
          {
            "name": "myeloid_compartment_abundance_gate",
            "role": "gate",
            "field": "$.myeloid_compartment_abundance",
            "kind": "value",
            "op": "in",
            "expected": [
              "expands_in_covid"
            ]
          },
          {
            "name": "myeloid_expansion_localization_gate",
            "role": "gate",
            "field": "$.myeloid_expansion_localization",
            "kind": "value",
            "op": "in",
            "expected": [
              "recruited_driven_resident_dysfunction"
            ]
          },
          {
            "name": "dominant_signature_gate",
            "role": "gate",
            "field": "$.dominant_signature",
            "kind": "value",
            "op": "in",
            "expected": [
              "technical_or_agonal_artifact"
            ]
          },
          {
            "name": "dominant_signature_direction_gate",
            "role": "gate",
            "field": "$.dominant_signature_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_in_covid"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-20-beta-reasoning-pi",
      "modelName": "grok-4.20-beta-0309-reasoning",
      "harness": "pi",
      "task": "q6_myeloid_subset_restricted_de",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 675,
      "cost": 0.05930294999999999,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "myeloid_subset_with_strongest_change_gate",
            "role": "gate",
            "field": "$.myeloid_subset_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "subset_gamma"
            ]
          },
          {
            "name": "gene_with_strongest_change_gate",
            "role": "gate",
            "field": "$.gene_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "AXL",
              "PROS1"
            ]
          },
          {
            "name": "direction_of_change_gate",
            "role": "gate",
            "field": "$.direction_of_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "down_in_covid"
            ]
          },
          {
            "name": "biological_function_interpretation_gate",
            "role": "gate",
            "field": "$.biological_function_interpretation",
            "kind": "value",
            "op": "in",
            "expected": [
              "efferocytosis",
              "phagocytosis"
            ]
          },
          {
            "name": "myeloid_compartment_abundance_gate",
            "role": "gate",
            "field": "$.myeloid_compartment_abundance",
            "kind": "value",
            "op": "in",
            "expected": [
              "expands_in_covid"
            ]
          },
          {
            "name": "myeloid_expansion_localization_gate",
            "role": "gate",
            "field": "$.myeloid_expansion_localization",
            "kind": "value",
            "op": "in",
            "expected": [
              "recruited_driven_resident_dysfunction"
            ]
          },
          {
            "name": "dominant_signature_gate",
            "role": "gate",
            "field": "$.dominant_signature",
            "kind": "value",
            "op": "in",
            "expected": [
              "technical_or_agonal_artifact"
            ]
          },
          {
            "name": "dominant_signature_direction_gate",
            "role": "gate",
            "field": "$.dominant_signature_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_in_covid"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-3-pi",
      "modelName": "grok-4.3",
      "harness": "pi",
      "task": "q6_myeloid_subset_restricted_de",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 347,
      "cost": 0.01614555,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "myeloid_subset_with_strongest_change_gate",
            "role": "gate",
            "field": "$.myeloid_subset_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "subset_gamma"
            ]
          },
          {
            "name": "gene_with_strongest_change_gate",
            "role": "gate",
            "field": "$.gene_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "AXL",
              "PROS1"
            ]
          },
          {
            "name": "direction_of_change_gate",
            "role": "gate",
            "field": "$.direction_of_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "down_in_covid"
            ]
          },
          {
            "name": "biological_function_interpretation_gate",
            "role": "gate",
            "field": "$.biological_function_interpretation",
            "kind": "value",
            "op": "in",
            "expected": [
              "efferocytosis",
              "phagocytosis"
            ]
          },
          {
            "name": "myeloid_compartment_abundance_gate",
            "role": "gate",
            "field": "$.myeloid_compartment_abundance",
            "kind": "value",
            "op": "in",
            "expected": [
              "expands_in_covid"
            ]
          },
          {
            "name": "myeloid_expansion_localization_gate",
            "role": "gate",
            "field": "$.myeloid_expansion_localization",
            "kind": "value",
            "op": "in",
            "expected": [
              "recruited_driven_resident_dysfunction"
            ]
          },
          {
            "name": "dominant_signature_gate",
            "role": "gate",
            "field": "$.dominant_signature",
            "kind": "value",
            "op": "in",
            "expected": [
              "technical_or_agonal_artifact"
            ]
          },
          {
            "name": "dominant_signature_direction_gate",
            "role": "gate",
            "field": "$.dominant_signature_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_in_covid"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-3-pi",
      "modelName": "grok-4.3",
      "harness": "pi",
      "task": "q6_myeloid_subset_restricted_de",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 526,
      "cost": 0.028493800000000007,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "myeloid_subset_with_strongest_change_gate",
            "role": "gate",
            "field": "$.myeloid_subset_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "subset_gamma"
            ]
          },
          {
            "name": "gene_with_strongest_change_gate",
            "role": "gate",
            "field": "$.gene_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "AXL",
              "PROS1"
            ]
          },
          {
            "name": "direction_of_change_gate",
            "role": "gate",
            "field": "$.direction_of_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "down_in_covid"
            ]
          },
          {
            "name": "biological_function_interpretation_gate",
            "role": "gate",
            "field": "$.biological_function_interpretation",
            "kind": "value",
            "op": "in",
            "expected": [
              "efferocytosis",
              "phagocytosis"
            ]
          },
          {
            "name": "myeloid_compartment_abundance_gate",
            "role": "gate",
            "field": "$.myeloid_compartment_abundance",
            "kind": "value",
            "op": "in",
            "expected": [
              "expands_in_covid"
            ]
          },
          {
            "name": "myeloid_expansion_localization_gate",
            "role": "gate",
            "field": "$.myeloid_expansion_localization",
            "kind": "value",
            "op": "in",
            "expected": [
              "recruited_driven_resident_dysfunction"
            ]
          },
          {
            "name": "dominant_signature_gate",
            "role": "gate",
            "field": "$.dominant_signature",
            "kind": "value",
            "op": "in",
            "expected": [
              "technical_or_agonal_artifact"
            ]
          },
          {
            "name": "dominant_signature_direction_gate",
            "role": "gate",
            "field": "$.dominant_signature_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_in_covid"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-3-pi",
      "modelName": "grok-4.3",
      "harness": "pi",
      "task": "q6_myeloid_subset_restricted_de",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 308,
      "cost": 0.018333200000000004,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "myeloid_subset_with_strongest_change_gate",
            "role": "gate",
            "field": "$.myeloid_subset_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "subset_gamma"
            ]
          },
          {
            "name": "gene_with_strongest_change_gate",
            "role": "gate",
            "field": "$.gene_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "AXL",
              "PROS1"
            ]
          },
          {
            "name": "direction_of_change_gate",
            "role": "gate",
            "field": "$.direction_of_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "down_in_covid"
            ]
          },
          {
            "name": "biological_function_interpretation_gate",
            "role": "gate",
            "field": "$.biological_function_interpretation",
            "kind": "value",
            "op": "in",
            "expected": [
              "efferocytosis",
              "phagocytosis"
            ]
          },
          {
            "name": "myeloid_compartment_abundance_gate",
            "role": "gate",
            "field": "$.myeloid_compartment_abundance",
            "kind": "value",
            "op": "in",
            "expected": [
              "expands_in_covid"
            ]
          },
          {
            "name": "myeloid_expansion_localization_gate",
            "role": "gate",
            "field": "$.myeloid_expansion_localization",
            "kind": "value",
            "op": "in",
            "expected": [
              "recruited_driven_resident_dysfunction"
            ]
          },
          {
            "name": "dominant_signature_gate",
            "role": "gate",
            "field": "$.dominant_signature",
            "kind": "value",
            "op": "in",
            "expected": [
              "technical_or_agonal_artifact"
            ]
          },
          {
            "name": "dominant_signature_direction_gate",
            "role": "gate",
            "field": "$.dominant_signature_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_in_covid"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-kimi-k2p6-pi",
      "modelName": "kimi-k2p6",
      "harness": "pi",
      "task": "q6_myeloid_subset_restricted_de",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 2728,
      "cost": 0.6718483300000001,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "myeloid_subset_with_strongest_change_gate",
            "role": "gate",
            "field": "$.myeloid_subset_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "subset_gamma"
            ]
          },
          {
            "name": "gene_with_strongest_change_gate",
            "role": "gate",
            "field": "$.gene_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "AXL",
              "PROS1"
            ]
          },
          {
            "name": "direction_of_change_gate",
            "role": "gate",
            "field": "$.direction_of_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "down_in_covid"
            ]
          },
          {
            "name": "biological_function_interpretation_gate",
            "role": "gate",
            "field": "$.biological_function_interpretation",
            "kind": "value",
            "op": "in",
            "expected": [
              "efferocytosis",
              "phagocytosis"
            ]
          },
          {
            "name": "myeloid_compartment_abundance_gate",
            "role": "gate",
            "field": "$.myeloid_compartment_abundance",
            "kind": "value",
            "op": "in",
            "expected": [
              "expands_in_covid"
            ]
          },
          {
            "name": "myeloid_expansion_localization_gate",
            "role": "gate",
            "field": "$.myeloid_expansion_localization",
            "kind": "value",
            "op": "in",
            "expected": [
              "recruited_driven_resident_dysfunction"
            ]
          },
          {
            "name": "dominant_signature_gate",
            "role": "gate",
            "field": "$.dominant_signature",
            "kind": "value",
            "op": "in",
            "expected": [
              "technical_or_agonal_artifact"
            ]
          },
          {
            "name": "dominant_signature_direction_gate",
            "role": "gate",
            "field": "$.dominant_signature_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_in_covid"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-kimi-k2p6-pi",
      "modelName": "kimi-k2p6",
      "harness": "pi",
      "task": "q6_myeloid_subset_restricted_de",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 3527,
      "cost": 1.2195491299999999,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "myeloid_subset_with_strongest_change_gate",
            "role": "gate",
            "field": "$.myeloid_subset_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "subset_gamma"
            ]
          },
          {
            "name": "gene_with_strongest_change_gate",
            "role": "gate",
            "field": "$.gene_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "AXL",
              "PROS1"
            ]
          },
          {
            "name": "direction_of_change_gate",
            "role": "gate",
            "field": "$.direction_of_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "down_in_covid"
            ]
          },
          {
            "name": "biological_function_interpretation_gate",
            "role": "gate",
            "field": "$.biological_function_interpretation",
            "kind": "value",
            "op": "in",
            "expected": [
              "efferocytosis",
              "phagocytosis"
            ]
          },
          {
            "name": "myeloid_compartment_abundance_gate",
            "role": "gate",
            "field": "$.myeloid_compartment_abundance",
            "kind": "value",
            "op": "in",
            "expected": [
              "expands_in_covid"
            ]
          },
          {
            "name": "myeloid_expansion_localization_gate",
            "role": "gate",
            "field": "$.myeloid_expansion_localization",
            "kind": "value",
            "op": "in",
            "expected": [
              "recruited_driven_resident_dysfunction"
            ]
          },
          {
            "name": "dominant_signature_gate",
            "role": "gate",
            "field": "$.dominant_signature",
            "kind": "value",
            "op": "in",
            "expected": [
              "technical_or_agonal_artifact"
            ]
          },
          {
            "name": "dominant_signature_direction_gate",
            "role": "gate",
            "field": "$.dominant_signature_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_in_covid"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-kimi-k2p6-pi",
      "modelName": "kimi-k2p6",
      "harness": "pi",
      "task": "q6_myeloid_subset_restricted_de",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1568,
      "cost": 0.5100215499999999,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "myeloid_subset_with_strongest_change_gate",
            "role": "gate",
            "field": "$.myeloid_subset_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "subset_gamma"
            ]
          },
          {
            "name": "gene_with_strongest_change_gate",
            "role": "gate",
            "field": "$.gene_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "AXL",
              "PROS1"
            ]
          },
          {
            "name": "direction_of_change_gate",
            "role": "gate",
            "field": "$.direction_of_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "down_in_covid"
            ]
          },
          {
            "name": "biological_function_interpretation_gate",
            "role": "gate",
            "field": "$.biological_function_interpretation",
            "kind": "value",
            "op": "in",
            "expected": [
              "efferocytosis",
              "phagocytosis"
            ]
          },
          {
            "name": "myeloid_compartment_abundance_gate",
            "role": "gate",
            "field": "$.myeloid_compartment_abundance",
            "kind": "value",
            "op": "in",
            "expected": [
              "expands_in_covid"
            ]
          },
          {
            "name": "myeloid_expansion_localization_gate",
            "role": "gate",
            "field": "$.myeloid_expansion_localization",
            "kind": "value",
            "op": "in",
            "expected": [
              "recruited_driven_resident_dysfunction"
            ]
          },
          {
            "name": "dominant_signature_gate",
            "role": "gate",
            "field": "$.dominant_signature",
            "kind": "value",
            "op": "in",
            "expected": [
              "technical_or_agonal_artifact"
            ]
          },
          {
            "name": "dominant_signature_direction_gate",
            "role": "gate",
            "field": "$.dominant_signature_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_in_covid"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-6-claude-code",
      "modelName": "claude-opus-4-6",
      "harness": "claude-code",
      "task": "snrnaseq_per_donor_composition_shifts_lethal_covid_lung",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 1498,
      "cost": 1.15100975,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dict_match",
            "role": "gate",
            "field": "cell_type_direction_map",
            "kind": "dict",
            "expected": {
              "Myeloid": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "Epithelia": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Fibroblasts": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "T_NK": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "B_plasma": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Endothelial": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Mast": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "APC": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Neuronal": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              }
            }
          },
          {
            "name": "over_merge_immune_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Immune",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_leukocyte_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Leukocyte",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_lymphoid_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Lymphoid",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_stromal_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Stromal",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-6-claude-code",
      "modelName": "claude-opus-4-6",
      "harness": "claude-code",
      "task": "snrnaseq_per_donor_composition_shifts_lethal_covid_lung",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 1854,
      "cost": 1.29487225,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dict_match",
            "role": "gate",
            "field": "cell_type_direction_map",
            "kind": "dict",
            "expected": {
              "Myeloid": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "Epithelia": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Fibroblasts": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "T_NK": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "B_plasma": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Endothelial": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Mast": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "APC": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Neuronal": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              }
            }
          },
          {
            "name": "over_merge_immune_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Immune",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_leukocyte_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Leukocyte",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_lymphoid_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Lymphoid",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_stromal_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Stromal",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-6-claude-code",
      "modelName": "claude-opus-4-6",
      "harness": "claude-code",
      "task": "snrnaseq_per_donor_composition_shifts_lethal_covid_lung",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1085,
      "cost": 0.7191685,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dict_match",
            "role": "gate",
            "field": "cell_type_direction_map",
            "kind": "dict",
            "expected": {
              "Myeloid": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "Epithelia": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Fibroblasts": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "T_NK": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "B_plasma": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Endothelial": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Mast": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "APC": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Neuronal": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              }
            }
          },
          {
            "name": "over_merge_immune_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Immune",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_leukocyte_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Leukocyte",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_lymphoid_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Lymphoid",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_stromal_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Stromal",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-6-pi",
      "modelName": "claude-opus-4-6",
      "harness": "pi",
      "task": "snrnaseq_per_donor_composition_shifts_lethal_covid_lung",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 1004,
      "cost": 0.4875995,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dict_match",
            "role": "gate",
            "field": "cell_type_direction_map",
            "kind": "dict",
            "expected": {
              "Myeloid": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "Epithelia": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Fibroblasts": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "T_NK": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "B_plasma": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Endothelial": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Mast": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "APC": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Neuronal": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              }
            }
          },
          {
            "name": "over_merge_immune_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Immune",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_leukocyte_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Leukocyte",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_lymphoid_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Lymphoid",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_stromal_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Stromal",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-6-pi",
      "modelName": "claude-opus-4-6",
      "harness": "pi",
      "task": "snrnaseq_per_donor_composition_shifts_lethal_covid_lung",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 883,
      "cost": 0.5460135,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dict_match",
            "role": "gate",
            "field": "cell_type_direction_map",
            "kind": "dict",
            "expected": {
              "Myeloid": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "Epithelia": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Fibroblasts": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "T_NK": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "B_plasma": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Endothelial": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Mast": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "APC": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Neuronal": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              }
            }
          },
          {
            "name": "over_merge_immune_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Immune",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_leukocyte_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Leukocyte",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_lymphoid_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Lymphoid",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_stromal_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Stromal",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-6-pi",
      "modelName": "claude-opus-4-6",
      "harness": "pi",
      "task": "snrnaseq_per_donor_composition_shifts_lethal_covid_lung",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1082,
      "cost": 0.42023025,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dict_match",
            "role": "gate",
            "field": "cell_type_direction_map",
            "kind": "dict",
            "expected": {
              "Myeloid": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "Epithelia": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Fibroblasts": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "T_NK": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "B_plasma": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Endothelial": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Mast": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "APC": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Neuronal": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              }
            }
          },
          {
            "name": "over_merge_immune_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Immune",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_leukocyte_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Leukocyte",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_lymphoid_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Lymphoid",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_stromal_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Stromal",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-7-claude-code",
      "modelName": "claude-opus-4-7",
      "harness": "claude-code",
      "task": "snrnaseq_per_donor_composition_shifts_lethal_covid_lung",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 279,
      "cost": 0.7271762500000001,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dict_match",
            "role": "gate",
            "field": "cell_type_direction_map",
            "kind": "dict",
            "expected": {
              "Myeloid": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "Epithelia": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Fibroblasts": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "T_NK": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "B_plasma": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Endothelial": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Mast": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "APC": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Neuronal": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              }
            }
          },
          {
            "name": "over_merge_immune_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Immune",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_leukocyte_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Leukocyte",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_lymphoid_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Lymphoid",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_stromal_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Stromal",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-7-claude-code",
      "modelName": "claude-opus-4-7",
      "harness": "claude-code",
      "task": "snrnaseq_per_donor_composition_shifts_lethal_covid_lung",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 1489,
      "cost": 2.5563662500000004,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dict_match",
            "role": "gate",
            "field": "cell_type_direction_map",
            "kind": "dict",
            "expected": {
              "Myeloid": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "Epithelia": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Fibroblasts": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "T_NK": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "B_plasma": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Endothelial": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Mast": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "APC": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Neuronal": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              }
            }
          },
          {
            "name": "over_merge_immune_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Immune",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_leukocyte_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Leukocyte",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_lymphoid_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Lymphoid",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_stromal_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Stromal",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-7-claude-code",
      "modelName": "claude-opus-4-7",
      "harness": "claude-code",
      "task": "snrnaseq_per_donor_composition_shifts_lethal_covid_lung",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1593,
      "cost": 2.8640947500000005,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dict_match",
            "role": "gate",
            "field": "cell_type_direction_map",
            "kind": "dict",
            "expected": {
              "Myeloid": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "Epithelia": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Fibroblasts": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "T_NK": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "B_plasma": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Endothelial": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Mast": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "APC": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Neuronal": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              }
            }
          },
          {
            "name": "over_merge_immune_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Immune",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_leukocyte_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Leukocyte",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_lymphoid_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Lymphoid",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_stromal_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Stromal",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-7-pi",
      "modelName": "claude-opus-4-7",
      "harness": "pi",
      "task": "snrnaseq_per_donor_composition_shifts_lethal_covid_lung",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 745,
      "cost": 0.6427324999999999,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dict_match",
            "role": "gate",
            "field": "cell_type_direction_map",
            "kind": "dict",
            "expected": {
              "Myeloid": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "Epithelia": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Fibroblasts": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "T_NK": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "B_plasma": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Endothelial": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Mast": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "APC": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Neuronal": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              }
            }
          },
          {
            "name": "over_merge_immune_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Immune",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_leukocyte_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Leukocyte",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_lymphoid_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Lymphoid",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_stromal_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Stromal",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-7-pi",
      "modelName": "claude-opus-4-7",
      "harness": "pi",
      "task": "snrnaseq_per_donor_composition_shifts_lethal_covid_lung",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 1898,
      "cost": 1.4955025,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dict_match",
            "role": "gate",
            "field": "cell_type_direction_map",
            "kind": "dict",
            "expected": {
              "Myeloid": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "Epithelia": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Fibroblasts": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "T_NK": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "B_plasma": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Endothelial": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Mast": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "APC": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Neuronal": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              }
            }
          },
          {
            "name": "over_merge_immune_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Immune",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_leukocyte_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Leukocyte",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_lymphoid_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Lymphoid",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_stromal_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Stromal",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-7-pi",
      "modelName": "claude-opus-4-7",
      "harness": "pi",
      "task": "snrnaseq_per_donor_composition_shifts_lethal_covid_lung",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 993,
      "cost": 0.8033975,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dict_match",
            "role": "gate",
            "field": "cell_type_direction_map",
            "kind": "dict",
            "expected": {
              "Myeloid": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "Epithelia": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Fibroblasts": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "T_NK": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "B_plasma": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Endothelial": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Mast": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "APC": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Neuronal": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              }
            }
          },
          {
            "name": "over_merge_immune_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Immune",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_leukocyte_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Leukocyte",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_lymphoid_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Lymphoid",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_stromal_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Stromal",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-8-claude-code",
      "modelName": "claude-opus-4-8",
      "harness": "claude-code",
      "task": "snrnaseq_per_donor_composition_shifts_lethal_covid_lung",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 602,
      "cost": 2.64867375,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dict_match",
            "role": "gate",
            "field": "cell_type_direction_map",
            "kind": "dict",
            "expected": {
              "Myeloid": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "Epithelia": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Fibroblasts": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "T_NK": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "B_plasma": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Endothelial": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Mast": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "APC": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Neuronal": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              }
            }
          },
          {
            "name": "over_merge_immune_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Immune",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_leukocyte_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Leukocyte",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_lymphoid_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Lymphoid",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_stromal_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Stromal",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-8-claude-code",
      "modelName": "claude-opus-4-8",
      "harness": "claude-code",
      "task": "snrnaseq_per_donor_composition_shifts_lethal_covid_lung",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 3925,
      "cost": 8.2431605,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dict_match",
            "role": "gate",
            "field": "cell_type_direction_map",
            "kind": "dict",
            "expected": {
              "Myeloid": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "Epithelia": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Fibroblasts": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "T_NK": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "B_plasma": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Endothelial": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Mast": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "APC": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Neuronal": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              }
            }
          },
          {
            "name": "over_merge_immune_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Immune",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_leukocyte_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Leukocyte",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_lymphoid_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Lymphoid",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_stromal_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Stromal",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-8-claude-code",
      "modelName": "claude-opus-4-8",
      "harness": "claude-code",
      "task": "snrnaseq_per_donor_composition_shifts_lethal_covid_lung",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 2373,
      "cost": 6.50156775,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dict_match",
            "role": "gate",
            "field": "cell_type_direction_map",
            "kind": "dict",
            "expected": {
              "Myeloid": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "Epithelia": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Fibroblasts": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "T_NK": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "B_plasma": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Endothelial": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Mast": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "APC": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Neuronal": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              }
            }
          },
          {
            "name": "over_merge_immune_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Immune",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_leukocyte_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Leukocyte",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_lymphoid_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Lymphoid",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_stromal_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Stromal",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-8-pi",
      "modelName": "claude-opus-4-8",
      "harness": "pi",
      "task": "snrnaseq_per_donor_composition_shifts_lethal_covid_lung",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 1692,
      "cost": 2.1568585,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dict_match",
            "role": "gate",
            "field": "cell_type_direction_map",
            "kind": "dict",
            "expected": {
              "Myeloid": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "Epithelia": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Fibroblasts": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "T_NK": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "B_plasma": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Endothelial": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Mast": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "APC": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Neuronal": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              }
            }
          },
          {
            "name": "over_merge_immune_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Immune",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_leukocyte_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Leukocyte",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_lymphoid_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Lymphoid",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_stromal_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Stromal",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-8-pi",
      "modelName": "claude-opus-4-8",
      "harness": "pi",
      "task": "snrnaseq_per_donor_composition_shifts_lethal_covid_lung",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 1011,
      "cost": 1.56112775,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dict_match",
            "role": "gate",
            "field": "cell_type_direction_map",
            "kind": "dict",
            "expected": {
              "Myeloid": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "Epithelia": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Fibroblasts": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "T_NK": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "B_plasma": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Endothelial": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Mast": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "APC": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Neuronal": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              }
            }
          },
          {
            "name": "over_merge_immune_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Immune",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_leukocyte_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Leukocyte",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_lymphoid_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Lymphoid",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_stromal_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Stromal",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-4-8-pi",
      "modelName": "claude-opus-4-8",
      "harness": "pi",
      "task": "snrnaseq_per_donor_composition_shifts_lethal_covid_lung",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1372,
      "cost": 2.14115225,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dict_match",
            "role": "gate",
            "field": "cell_type_direction_map",
            "kind": "dict",
            "expected": {
              "Myeloid": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "Epithelia": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Fibroblasts": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "T_NK": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "B_plasma": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Endothelial": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Mast": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "APC": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Neuronal": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              }
            }
          },
          {
            "name": "over_merge_immune_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Immune",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_leukocyte_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Leukocyte",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_lymphoid_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Lymphoid",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_stromal_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Stromal",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-4-6-pi",
      "modelName": "claude-sonnet-4-6",
      "harness": "pi",
      "task": "snrnaseq_per_donor_composition_shifts_lethal_covid_lung",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 1910,
      "cost": 0.81648825,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dict_match",
            "role": "gate",
            "field": "cell_type_direction_map",
            "kind": "dict",
            "expected": {
              "Myeloid": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "Epithelia": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Fibroblasts": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "T_NK": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "B_plasma": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Endothelial": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Mast": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "APC": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Neuronal": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              }
            }
          },
          {
            "name": "over_merge_immune_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Immune",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_leukocyte_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Leukocyte",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_lymphoid_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Lymphoid",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_stromal_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Stromal",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-4-6-pi",
      "modelName": "claude-sonnet-4-6",
      "harness": "pi",
      "task": "snrnaseq_per_donor_composition_shifts_lethal_covid_lung",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 1915,
      "cost": 1.2787640999999996,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dict_match",
            "role": "gate",
            "field": "cell_type_direction_map",
            "kind": "dict",
            "expected": {
              "Myeloid": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "Epithelia": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Fibroblasts": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "T_NK": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "B_plasma": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Endothelial": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Mast": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "APC": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Neuronal": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              }
            }
          },
          {
            "name": "over_merge_immune_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Immune",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_leukocyte_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Leukocyte",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_lymphoid_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Lymphoid",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_stromal_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Stromal",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-4-6-pi",
      "modelName": "claude-sonnet-4-6",
      "harness": "pi",
      "task": "snrnaseq_per_donor_composition_shifts_lethal_covid_lung",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1426,
      "cost": 0.90906735,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dict_match",
            "role": "gate",
            "field": "cell_type_direction_map",
            "kind": "dict",
            "expected": {
              "Myeloid": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "Epithelia": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Fibroblasts": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "T_NK": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "B_plasma": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Endothelial": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Mast": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "APC": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Neuronal": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              }
            }
          },
          {
            "name": "over_merge_immune_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Immune",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_leukocyte_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Leukocyte",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_lymphoid_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Lymphoid",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_stromal_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Stromal",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gemini-3-1-pro-preview-pi",
      "modelName": "gemini-3.1-pro-preview",
      "harness": "pi",
      "task": "snrnaseq_per_donor_composition_shifts_lethal_covid_lung",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 1745,
      "cost": 0.6431580000000001,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dict_match",
            "role": "gate",
            "field": "cell_type_direction_map",
            "kind": "dict",
            "expected": {
              "Myeloid": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "Epithelia": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Fibroblasts": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "T_NK": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "B_plasma": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Endothelial": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Mast": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "APC": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Neuronal": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              }
            }
          },
          {
            "name": "over_merge_immune_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Immune",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_leukocyte_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Leukocyte",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_lymphoid_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Lymphoid",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_stromal_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Stromal",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gemini-3-1-pro-preview-pi",
      "modelName": "gemini-3.1-pro-preview",
      "harness": "pi",
      "task": "snrnaseq_per_donor_composition_shifts_lethal_covid_lung",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 907,
      "cost": 0.5819086,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dict_match",
            "role": "gate",
            "field": "cell_type_direction_map",
            "kind": "dict",
            "expected": {
              "Myeloid": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "Epithelia": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Fibroblasts": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "T_NK": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "B_plasma": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Endothelial": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Mast": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "APC": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Neuronal": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              }
            }
          },
          {
            "name": "over_merge_immune_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Immune",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_leukocyte_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Leukocyte",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_lymphoid_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Lymphoid",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_stromal_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Stromal",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gemini-3-1-pro-preview-pi",
      "modelName": "gemini-3.1-pro-preview",
      "harness": "pi",
      "task": "snrnaseq_per_donor_composition_shifts_lethal_covid_lung",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 2576,
      "cost": 0.6685778,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dict_match",
            "role": "gate",
            "field": "cell_type_direction_map",
            "kind": "dict",
            "expected": {
              "Myeloid": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "Epithelia": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Fibroblasts": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "T_NK": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "B_plasma": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Endothelial": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Mast": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "APC": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Neuronal": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              }
            }
          },
          {
            "name": "over_merge_immune_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Immune",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_leukocyte_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Leukocyte",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_lymphoid_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Lymphoid",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_stromal_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Stromal",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gemini-3-5-flash-pi",
      "modelName": "gemini-3.5-flash",
      "harness": "pi",
      "task": "snrnaseq_per_donor_composition_shifts_lethal_covid_lung",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 1,
      "passed": true,
      "score": null,
      "durationS": 344,
      "cost": 1.0950556499999997,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dict_match",
            "role": "gate",
            "field": "cell_type_direction_map",
            "kind": "dict",
            "expected": {
              "Myeloid": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "Epithelia": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Fibroblasts": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "T_NK": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "B_plasma": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Endothelial": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Mast": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "APC": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Neuronal": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              }
            }
          },
          {
            "name": "over_merge_immune_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Immune",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_leukocyte_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Leukocyte",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_lymphoid_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Lymphoid",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_stromal_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Stromal",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gemini-3-5-flash-pi",
      "modelName": "gemini-3.5-flash",
      "harness": "pi",
      "task": "snrnaseq_per_donor_composition_shifts_lethal_covid_lung",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 2,
      "passed": true,
      "score": null,
      "durationS": 71,
      "cost": 0.1466208,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dict_match",
            "role": "gate",
            "field": "cell_type_direction_map",
            "kind": "dict",
            "expected": {
              "Myeloid": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "Epithelia": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Fibroblasts": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "T_NK": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "B_plasma": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Endothelial": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Mast": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "APC": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Neuronal": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              }
            }
          },
          {
            "name": "over_merge_immune_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Immune",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_leukocyte_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Leukocyte",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_lymphoid_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Lymphoid",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_stromal_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Stromal",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gemini-3-5-flash-pi",
      "modelName": "gemini-3.5-flash",
      "harness": "pi",
      "task": "snrnaseq_per_donor_composition_shifts_lethal_covid_lung",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1056,
      "cost": 0.9284010000000001,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dict_match",
            "role": "gate",
            "field": "cell_type_direction_map",
            "kind": "dict",
            "expected": {
              "Myeloid": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "Epithelia": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Fibroblasts": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "T_NK": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "B_plasma": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Endothelial": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Mast": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "APC": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Neuronal": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              }
            }
          },
          {
            "name": "over_merge_immune_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Immune",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_leukocyte_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Leukocyte",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_lymphoid_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Lymphoid",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_stromal_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Stromal",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-glm-5p2-pi",
      "modelName": "glm-5p2",
      "harness": "pi",
      "task": "snrnaseq_per_donor_composition_shifts_lethal_covid_lung",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 5646,
      "cost": 2.5551274399999997,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dict_match",
            "role": "gate",
            "field": "cell_type_direction_map",
            "kind": "dict",
            "expected": {
              "Myeloid": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "Epithelia": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Fibroblasts": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "T_NK": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "B_plasma": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Endothelial": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Mast": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "APC": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Neuronal": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              }
            }
          },
          {
            "name": "over_merge_immune_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Immune",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_leukocyte_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Leukocyte",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_lymphoid_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Lymphoid",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_stromal_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Stromal",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-glm-5p2-pi",
      "modelName": "glm-5p2",
      "harness": "pi",
      "task": "snrnaseq_per_donor_composition_shifts_lethal_covid_lung",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 3177,
      "cost": 1.24044946,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dict_match",
            "role": "gate",
            "field": "cell_type_direction_map",
            "kind": "dict",
            "expected": {
              "Myeloid": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "Epithelia": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Fibroblasts": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "T_NK": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "B_plasma": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Endothelial": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Mast": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "APC": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Neuronal": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              }
            }
          },
          {
            "name": "over_merge_immune_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Immune",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_leukocyte_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Leukocyte",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_lymphoid_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Lymphoid",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_stromal_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Stromal",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-glm-5p2-pi",
      "modelName": "glm-5p2",
      "harness": "pi",
      "task": "snrnaseq_per_donor_composition_shifts_lethal_covid_lung",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 3879,
      "cost": 2.4653647399999996,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dict_match",
            "role": "gate",
            "field": "cell_type_direction_map",
            "kind": "dict",
            "expected": {
              "Myeloid": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "Epithelia": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Fibroblasts": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "T_NK": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "B_plasma": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Endothelial": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Mast": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "APC": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Neuronal": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              }
            }
          },
          {
            "name": "over_merge_immune_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Immune",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_leukocyte_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Leukocyte",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_lymphoid_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Lymphoid",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_stromal_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Stromal",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-4-codex",
      "modelName": "gpt-5.4",
      "harness": "openai-codex",
      "task": "snrnaseq_per_donor_composition_shifts_lethal_covid_lung",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 896,
      "cost": 1.963077,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dict_match",
            "role": "gate",
            "field": "cell_type_direction_map",
            "kind": "dict",
            "expected": {
              "Myeloid": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "Epithelia": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Fibroblasts": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "T_NK": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "B_plasma": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Endothelial": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Mast": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "APC": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Neuronal": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              }
            }
          },
          {
            "name": "over_merge_immune_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Immune",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_leukocyte_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Leukocyte",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_lymphoid_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Lymphoid",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_stromal_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Stromal",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-4-codex",
      "modelName": "gpt-5.4",
      "harness": "openai-codex",
      "task": "snrnaseq_per_donor_composition_shifts_lethal_covid_lung",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 1143,
      "cost": 2.6284905,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dict_match",
            "role": "gate",
            "field": "cell_type_direction_map",
            "kind": "dict",
            "expected": {
              "Myeloid": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "Epithelia": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Fibroblasts": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "T_NK": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "B_plasma": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Endothelial": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Mast": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "APC": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Neuronal": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              }
            }
          },
          {
            "name": "over_merge_immune_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Immune",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_leukocyte_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Leukocyte",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_lymphoid_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Lymphoid",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_stromal_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Stromal",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-4-codex",
      "modelName": "gpt-5.4",
      "harness": "openai-codex",
      "task": "snrnaseq_per_donor_composition_shifts_lethal_covid_lung",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 906,
      "cost": 0.8479485,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dict_match",
            "role": "gate",
            "field": "cell_type_direction_map",
            "kind": "dict",
            "expected": {
              "Myeloid": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "Epithelia": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Fibroblasts": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "T_NK": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "B_plasma": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Endothelial": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Mast": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "APC": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Neuronal": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              }
            }
          },
          {
            "name": "over_merge_immune_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Immune",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_leukocyte_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Leukocyte",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_lymphoid_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Lymphoid",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_stromal_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Stromal",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-4-pi",
      "modelName": "gpt-5.4",
      "harness": "pi",
      "task": "snrnaseq_per_donor_composition_shifts_lethal_covid_lung",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 5322,
      "cost": 0.8721615000000003,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dict_match",
            "role": "gate",
            "field": "cell_type_direction_map",
            "kind": "dict",
            "expected": {
              "Myeloid": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "Epithelia": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Fibroblasts": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "T_NK": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "B_plasma": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Endothelial": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Mast": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "APC": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Neuronal": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              }
            }
          },
          {
            "name": "over_merge_immune_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Immune",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_leukocyte_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Leukocyte",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_lymphoid_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Lymphoid",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_stromal_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Stromal",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-4-pi",
      "modelName": "gpt-5.4",
      "harness": "pi",
      "task": "snrnaseq_per_donor_composition_shifts_lethal_covid_lung",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 2796,
      "cost": 1.0458405000000002,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dict_match",
            "role": "gate",
            "field": "cell_type_direction_map",
            "kind": "dict",
            "expected": {
              "Myeloid": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "Epithelia": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Fibroblasts": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "T_NK": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "B_plasma": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Endothelial": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Mast": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "APC": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Neuronal": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              }
            }
          },
          {
            "name": "over_merge_immune_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Immune",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_leukocyte_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Leukocyte",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_lymphoid_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Lymphoid",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_stromal_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Stromal",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-4-pi",
      "modelName": "gpt-5.4",
      "harness": "pi",
      "task": "snrnaseq_per_donor_composition_shifts_lethal_covid_lung",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 2619,
      "cost": 0.641306,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dict_match",
            "role": "gate",
            "field": "cell_type_direction_map",
            "kind": "dict",
            "expected": {
              "Myeloid": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "Epithelia": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Fibroblasts": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "T_NK": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "B_plasma": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Endothelial": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Mast": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "APC": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Neuronal": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              }
            }
          },
          {
            "name": "over_merge_immune_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Immune",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_leukocyte_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Leukocyte",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_lymphoid_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Lymphoid",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_stromal_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Stromal",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-5-codex",
      "modelName": "gpt-5.5",
      "harness": "openai-codex",
      "task": "snrnaseq_per_donor_composition_shifts_lethal_covid_lung",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 1770,
      "cost": 1.5955270000000001,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dict_match",
            "role": "gate",
            "field": "cell_type_direction_map",
            "kind": "dict",
            "expected": {
              "Myeloid": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "Epithelia": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Fibroblasts": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "T_NK": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "B_plasma": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Endothelial": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Mast": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "APC": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Neuronal": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              }
            }
          },
          {
            "name": "over_merge_immune_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Immune",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_leukocyte_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Leukocyte",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_lymphoid_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Lymphoid",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_stromal_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Stromal",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-5-codex",
      "modelName": "gpt-5.5",
      "harness": "openai-codex",
      "task": "snrnaseq_per_donor_composition_shifts_lethal_covid_lung",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 1342,
      "cost": 2.11691,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dict_match",
            "role": "gate",
            "field": "cell_type_direction_map",
            "kind": "dict",
            "expected": {
              "Myeloid": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "Epithelia": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Fibroblasts": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "T_NK": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "B_plasma": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Endothelial": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Mast": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "APC": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Neuronal": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              }
            }
          },
          {
            "name": "over_merge_immune_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Immune",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_leukocyte_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Leukocyte",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_lymphoid_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Lymphoid",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_stromal_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Stromal",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-5-codex",
      "modelName": "gpt-5.5",
      "harness": "openai-codex",
      "task": "snrnaseq_per_donor_composition_shifts_lethal_covid_lung",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1022,
      "cost": 2.070126,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dict_match",
            "role": "gate",
            "field": "cell_type_direction_map",
            "kind": "dict",
            "expected": {
              "Myeloid": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "Epithelia": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Fibroblasts": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "T_NK": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "B_plasma": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Endothelial": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Mast": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "APC": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Neuronal": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              }
            }
          },
          {
            "name": "over_merge_immune_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Immune",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_leukocyte_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Leukocyte",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_lymphoid_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Lymphoid",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_stromal_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Stromal",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-5-pi",
      "modelName": "gpt-5.5",
      "harness": "pi",
      "task": "snrnaseq_per_donor_composition_shifts_lethal_covid_lung",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 717,
      "cost": 1.3658020000000002,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dict_match",
            "role": "gate",
            "field": "cell_type_direction_map",
            "kind": "dict",
            "expected": {
              "Myeloid": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "Epithelia": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Fibroblasts": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "T_NK": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "B_plasma": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Endothelial": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Mast": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "APC": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Neuronal": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              }
            }
          },
          {
            "name": "over_merge_immune_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Immune",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_leukocyte_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Leukocyte",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_lymphoid_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Lymphoid",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_stromal_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Stromal",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-5-pi",
      "modelName": "gpt-5.5",
      "harness": "pi",
      "task": "snrnaseq_per_donor_composition_shifts_lethal_covid_lung",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 3390,
      "cost": 1.8389130000000005,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dict_match",
            "role": "gate",
            "field": "cell_type_direction_map",
            "kind": "dict",
            "expected": {
              "Myeloid": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "Epithelia": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Fibroblasts": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "T_NK": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "B_plasma": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Endothelial": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Mast": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "APC": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Neuronal": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              }
            }
          },
          {
            "name": "over_merge_immune_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Immune",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_leukocyte_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Leukocyte",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_lymphoid_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Lymphoid",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_stromal_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Stromal",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-5-pi",
      "modelName": "gpt-5.5",
      "harness": "pi",
      "task": "snrnaseq_per_donor_composition_shifts_lethal_covid_lung",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1958,
      "cost": 1.729458,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dict_match",
            "role": "gate",
            "field": "cell_type_direction_map",
            "kind": "dict",
            "expected": {
              "Myeloid": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "Epithelia": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Fibroblasts": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "T_NK": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "B_plasma": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Endothelial": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Mast": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "APC": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Neuronal": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              }
            }
          },
          {
            "name": "over_merge_immune_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Immune",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_leukocyte_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Leukocyte",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_lymphoid_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Lymphoid",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_stromal_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Stromal",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-20-beta-reasoning-pi",
      "modelName": "grok-4.20-beta-0309-reasoning",
      "harness": "pi",
      "task": "snrnaseq_per_donor_composition_shifts_lethal_covid_lung",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 1500,
      "cost": 0.0407091,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dict_match",
            "role": "gate",
            "field": "cell_type_direction_map",
            "kind": "dict",
            "expected": {
              "Myeloid": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "Epithelia": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Fibroblasts": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "T_NK": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "B_plasma": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Endothelial": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Mast": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "APC": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Neuronal": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              }
            }
          },
          {
            "name": "over_merge_immune_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Immune",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_leukocyte_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Leukocyte",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_lymphoid_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Lymphoid",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_stromal_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Stromal",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-20-beta-reasoning-pi",
      "modelName": "grok-4.20-beta-0309-reasoning",
      "harness": "pi",
      "task": "snrnaseq_per_donor_composition_shifts_lethal_covid_lung",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 385,
      "cost": 0.0385134,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dict_match",
            "role": "gate",
            "field": "cell_type_direction_map",
            "kind": "dict",
            "expected": {
              "Myeloid": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "Epithelia": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Fibroblasts": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "T_NK": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "B_plasma": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Endothelial": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Mast": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "APC": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Neuronal": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              }
            }
          },
          {
            "name": "over_merge_immune_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Immune",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_leukocyte_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Leukocyte",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_lymphoid_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Lymphoid",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_stromal_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Stromal",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-20-beta-reasoning-pi",
      "modelName": "grok-4.20-beta-0309-reasoning",
      "harness": "pi",
      "task": "snrnaseq_per_donor_composition_shifts_lethal_covid_lung",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 968,
      "cost": 0.06545015000000001,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dict_match",
            "role": "gate",
            "field": "cell_type_direction_map",
            "kind": "dict",
            "expected": {
              "Myeloid": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "Epithelia": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Fibroblasts": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "T_NK": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "B_plasma": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Endothelial": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Mast": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "APC": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Neuronal": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              }
            }
          },
          {
            "name": "over_merge_immune_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Immune",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_leukocyte_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Leukocyte",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_lymphoid_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Lymphoid",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_stromal_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Stromal",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-3-pi",
      "modelName": "grok-4.3",
      "harness": "pi",
      "task": "snrnaseq_per_donor_composition_shifts_lethal_covid_lung",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 803,
      "cost": 0.023822199999999998,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dict_match",
            "role": "gate",
            "field": "cell_type_direction_map",
            "kind": "dict",
            "expected": {
              "Myeloid": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "Epithelia": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Fibroblasts": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "T_NK": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "B_plasma": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Endothelial": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Mast": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "APC": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Neuronal": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              }
            }
          },
          {
            "name": "over_merge_immune_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Immune",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_leukocyte_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Leukocyte",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_lymphoid_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Lymphoid",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_stromal_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Stromal",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-3-pi",
      "modelName": "grok-4.3",
      "harness": "pi",
      "task": "snrnaseq_per_donor_composition_shifts_lethal_covid_lung",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 390,
      "cost": 0.020901200000000005,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dict_match",
            "role": "gate",
            "field": "cell_type_direction_map",
            "kind": "dict",
            "expected": {
              "Myeloid": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "Epithelia": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Fibroblasts": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "T_NK": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "B_plasma": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Endothelial": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Mast": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "APC": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Neuronal": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              }
            }
          },
          {
            "name": "over_merge_immune_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Immune",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_leukocyte_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Leukocyte",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_lymphoid_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Lymphoid",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_stromal_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Stromal",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-3-pi",
      "modelName": "grok-4.3",
      "harness": "pi",
      "task": "snrnaseq_per_donor_composition_shifts_lethal_covid_lung",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 449,
      "cost": 0.021307350000000003,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dict_match",
            "role": "gate",
            "field": "cell_type_direction_map",
            "kind": "dict",
            "expected": {
              "Myeloid": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "Epithelia": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Fibroblasts": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "T_NK": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "B_plasma": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Endothelial": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Mast": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "APC": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Neuronal": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              }
            }
          },
          {
            "name": "over_merge_immune_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Immune",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_leukocyte_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Leukocyte",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_lymphoid_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Lymphoid",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_stromal_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Stromal",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-kimi-k2p6-pi",
      "modelName": "kimi-k2p6",
      "harness": "pi",
      "task": "snrnaseq_per_donor_composition_shifts_lethal_covid_lung",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 1849,
      "cost": 0.34760215,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dict_match",
            "role": "gate",
            "field": "cell_type_direction_map",
            "kind": "dict",
            "expected": {
              "Myeloid": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "Epithelia": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Fibroblasts": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "T_NK": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "B_plasma": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Endothelial": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Mast": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "APC": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Neuronal": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              }
            }
          },
          {
            "name": "over_merge_immune_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Immune",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_leukocyte_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Leukocyte",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_lymphoid_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Lymphoid",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_stromal_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Stromal",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-kimi-k2p6-pi",
      "modelName": "kimi-k2p6",
      "harness": "pi",
      "task": "snrnaseq_per_donor_composition_shifts_lethal_covid_lung",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 801,
      "cost": 0.22482285999999996,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dict_match",
            "role": "gate",
            "field": "cell_type_direction_map",
            "kind": "dict",
            "expected": {
              "Myeloid": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "Epithelia": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Fibroblasts": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "T_NK": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "B_plasma": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Endothelial": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Mast": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "APC": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Neuronal": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              }
            }
          },
          {
            "name": "over_merge_immune_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Immune",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_leukocyte_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Leukocyte",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_lymphoid_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Lymphoid",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_stromal_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Stromal",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-kimi-k2p6-pi",
      "modelName": "kimi-k2p6",
      "harness": "pi",
      "task": "snrnaseq_per_donor_composition_shifts_lethal_covid_lung",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 2090,
      "cost": 0.8448669799999998,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dict_match",
            "role": "gate",
            "field": "cell_type_direction_map",
            "kind": "dict",
            "expected": {
              "Myeloid": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "Epithelia": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Fibroblasts": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "T_NK": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "B_plasma": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Endothelial": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Mast": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "APC": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Neuronal": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              }
            }
          },
          {
            "name": "over_merge_immune_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Immune",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_leukocyte_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Leukocyte",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_lymphoid_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Lymphoid",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_stromal_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Stromal",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-5-claude-code",
      "modelName": "claude-sonnet-5",
      "harness": "claude-code",
      "task": "gse155381_q4_pseudotime_delay",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 1,
      "passed": true,
      "score": null,
      "durationS": 2197.484311,
      "cost": 6.096605350000001,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "pseudotime_direction_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "E17_chronological_match_weighted",
            "role": "additive",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pluripotency_exit_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pseudotime_direction_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_synchronous_with_monkey_EPI",
              "chimera_human_EPI_developmentally_advanced",
              "chimera_human_EPI_delayed_at_pre_implantation_only_resolved_post_implantation"
            ]
          },
          {
            "name": "E17_chronological_match_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "resembles_monkey_E17_Gast_EPI",
              "resembles_chimera_monkey_E17_EPI_specifically",
              "chimera_human_E17_EPI_more_advanced_than_monkey_E17_Gast"
            ]
          },
          {
            "name": "pluripotency_exit_paper_inverted_or_prior_driven",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_advances_pluripotency_exit_prematurely",
              "naive_pluripotency_program_specifically_engaged",
              "pluripotency_core_retained_via_KLF17_naive_marker_specifically"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-5-claude-code",
      "modelName": "claude-sonnet-5",
      "harness": "claude-code",
      "task": "gse155381_q4_pseudotime_delay",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 2,
      "passed": true,
      "score": null,
      "durationS": 2622.391372,
      "cost": 8.411136699999998,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "pseudotime_direction_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "E17_chronological_match_weighted",
            "role": "additive",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pluripotency_exit_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pseudotime_direction_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_synchronous_with_monkey_EPI",
              "chimera_human_EPI_developmentally_advanced",
              "chimera_human_EPI_delayed_at_pre_implantation_only_resolved_post_implantation"
            ]
          },
          {
            "name": "E17_chronological_match_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "resembles_monkey_E17_Gast_EPI",
              "resembles_chimera_monkey_E17_EPI_specifically",
              "chimera_human_E17_EPI_more_advanced_than_monkey_E17_Gast"
            ]
          },
          {
            "name": "pluripotency_exit_paper_inverted_or_prior_driven",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_advances_pluripotency_exit_prematurely",
              "naive_pluripotency_program_specifically_engaged",
              "pluripotency_core_retained_via_KLF17_naive_marker_specifically"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-5-claude-code",
      "modelName": "claude-sonnet-5",
      "harness": "claude-code",
      "task": "gse155381_q4_pseudotime_delay",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 2000.278511,
      "cost": 8.600373949999996,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "pseudotime_direction_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "E17_chronological_match_weighted",
            "role": "additive",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pluripotency_exit_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pseudotime_direction_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_synchronous_with_monkey_EPI",
              "chimera_human_EPI_developmentally_advanced",
              "chimera_human_EPI_delayed_at_pre_implantation_only_resolved_post_implantation"
            ]
          },
          {
            "name": "E17_chronological_match_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "resembles_monkey_E17_Gast_EPI",
              "resembles_chimera_monkey_E17_EPI_specifically",
              "chimera_human_E17_EPI_more_advanced_than_monkey_E17_Gast"
            ]
          },
          {
            "name": "pluripotency_exit_paper_inverted_or_prior_driven",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_advances_pluripotency_exit_prematurely",
              "naive_pluripotency_program_specifically_engaged",
              "pluripotency_core_retained_via_KLF17_naive_marker_specifically"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-5-claude-code",
      "modelName": "claude-sonnet-5",
      "harness": "claude-code",
      "task": "gse155381_q5_chimera_monkey_perturbation",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 1501.867914,
      "cost": 3.3364853000000005,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "deg_direction_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "comparison_subset_choice_weighted",
            "role": "additive",
            "field": "$.comparison_subset_choice",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "panel_resolved_strong_target_ccn1_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN1",
              "CYR61"
            ]
          },
          {
            "name": "panel_resolved_strong_target_ccn2_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN2",
              "CTGF"
            ]
          },
          {
            "name": "hippo_pathway_weighted",
            "role": "additive",
            "field": "$.hippo_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "tgf_beta_pathway_weighted",
            "role": "additive",
            "field": "$.tgf_beta_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "deg_direction_batch_effect_uncorrected",
            "role": "hard_fail",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_dominated_global",
              "up_dominated_in_chosen_subset"
            ]
          },
          {
            "name": "silent_panel_dropout_paired_with_directional_pathway_claim",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "hippo_pathway_direction": {
                "op": "in",
                "values": [
                  "hippo_pathway_upregulated",
                  "hippo_pathway_unchanged",
                  "wnt_pathway_specifically_perturbed",
                  "stress_response_pathway_dominantly_engaged",
                  "no_pathway_signal_above_confound",
                  "yap_tead_direct_target_panel_downregulated",
                  "yap_tead_panel_dependent_direction",
                  "hippo_pathway_downregulated"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-5-claude-code",
      "modelName": "claude-sonnet-5",
      "harness": "claude-code",
      "task": "gse155381_q5_chimera_monkey_perturbation",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 1652.05181,
      "cost": 3.680094,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "deg_direction_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "comparison_subset_choice_weighted",
            "role": "additive",
            "field": "$.comparison_subset_choice",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "panel_resolved_strong_target_ccn1_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN1",
              "CYR61"
            ]
          },
          {
            "name": "panel_resolved_strong_target_ccn2_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN2",
              "CTGF"
            ]
          },
          {
            "name": "hippo_pathway_weighted",
            "role": "additive",
            "field": "$.hippo_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "tgf_beta_pathway_weighted",
            "role": "additive",
            "field": "$.tgf_beta_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "deg_direction_batch_effect_uncorrected",
            "role": "hard_fail",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_dominated_global",
              "up_dominated_in_chosen_subset"
            ]
          },
          {
            "name": "silent_panel_dropout_paired_with_directional_pathway_claim",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "hippo_pathway_direction": {
                "op": "in",
                "values": [
                  "hippo_pathway_upregulated",
                  "hippo_pathway_unchanged",
                  "wnt_pathway_specifically_perturbed",
                  "stress_response_pathway_dominantly_engaged",
                  "no_pathway_signal_above_confound",
                  "yap_tead_direct_target_panel_downregulated",
                  "yap_tead_panel_dependent_direction",
                  "hippo_pathway_downregulated"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-5-claude-code",
      "modelName": "claude-sonnet-5",
      "harness": "claude-code",
      "task": "gse155381_q5_chimera_monkey_perturbation",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1436.666103,
      "cost": 3.787110549999999,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "deg_direction_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "comparison_subset_choice_weighted",
            "role": "additive",
            "field": "$.comparison_subset_choice",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "panel_resolved_strong_target_ccn1_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN1",
              "CYR61"
            ]
          },
          {
            "name": "panel_resolved_strong_target_ccn2_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN2",
              "CTGF"
            ]
          },
          {
            "name": "hippo_pathway_weighted",
            "role": "additive",
            "field": "$.hippo_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "tgf_beta_pathway_weighted",
            "role": "additive",
            "field": "$.tgf_beta_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "deg_direction_batch_effect_uncorrected",
            "role": "hard_fail",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_dominated_global",
              "up_dominated_in_chosen_subset"
            ]
          },
          {
            "name": "silent_panel_dropout_paired_with_directional_pathway_claim",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "hippo_pathway_direction": {
                "op": "in",
                "values": [
                  "hippo_pathway_upregulated",
                  "hippo_pathway_unchanged",
                  "wnt_pathway_specifically_perturbed",
                  "stress_response_pathway_dominantly_engaged",
                  "no_pathway_signal_above_confound",
                  "yap_tead_direct_target_panel_downregulated",
                  "yap_tead_panel_dependent_direction",
                  "hippo_pathway_downregulated"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-5-claude-code",
      "modelName": "claude-sonnet-5",
      "harness": "claude-code",
      "task": "gse155381_q7_xspecies_epi_communication",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 2104.533678,
      "cost": 7.016268100000003,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "richness_weighted",
            "role": "additive",
            "field": "$.xspecies_signaling_richness",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_weighted",
            "role": "additive",
            "field": "$.dominant_signaling_axis",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_wnt_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "WNT_FZD_LRP_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_fgf_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "FGF_FGFR_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_notch_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "NOTCH_DLL_JAG_family"
                ]
              }
            }
          },
          {
            "name": "signaling_directionality_weighted",
            "role": "additive",
            "field": "$.signaling_directionality",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "top_signal_interaction_class_weighted",
            "role": "additive",
            "field": "$.top_signal_interaction_class",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-5-claude-code",
      "modelName": "claude-sonnet-5",
      "harness": "claude-code",
      "task": "gse155381_q7_xspecies_epi_communication",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 2481.429871,
      "cost": 9.548241099999998,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "richness_weighted",
            "role": "additive",
            "field": "$.xspecies_signaling_richness",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_weighted",
            "role": "additive",
            "field": "$.dominant_signaling_axis",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_wnt_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "WNT_FZD_LRP_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_fgf_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "FGF_FGFR_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_notch_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "NOTCH_DLL_JAG_family"
                ]
              }
            }
          },
          {
            "name": "signaling_directionality_weighted",
            "role": "additive",
            "field": "$.signaling_directionality",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "top_signal_interaction_class_weighted",
            "role": "additive",
            "field": "$.top_signal_interaction_class",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-5-claude-code",
      "modelName": "claude-sonnet-5",
      "harness": "claude-code",
      "task": "gse155381_q7_xspecies_epi_communication",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 2920.205175,
      "cost": 7.247540749999999,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "richness_weighted",
            "role": "additive",
            "field": "$.xspecies_signaling_richness",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_weighted",
            "role": "additive",
            "field": "$.dominant_signaling_axis",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_wnt_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "WNT_FZD_LRP_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_fgf_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "FGF_FGFR_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_notch_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "NOTCH_DLL_JAG_family"
                ]
              }
            }
          },
          {
            "name": "signaling_directionality_weighted",
            "role": "additive",
            "field": "$.signaling_directionality",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "top_signal_interaction_class_weighted",
            "role": "additive",
            "field": "$.top_signal_interaction_class",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-5-claude-code",
      "modelName": "claude-sonnet-5",
      "harness": "claude-code",
      "task": "gse155381_q8_bidirectional_convergence",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 2060.832132,
      "cost": 8.568951850000005,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "chimera_human_identity_shift_target_weighted",
            "role": "additive",
            "field": "$.chimera_human_identity_shift_target",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_in_vivo_state_assignment_weighted",
            "role": "additive",
            "field": "$.chimera_human_in_vivo_state_assignment",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_shift_magnitude_weighted",
            "role": "additive",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_monkey_identity_shift_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "both_arms_null_contradicted_by_calibration",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "chimera_human_identity_shift_target": {
                "op": "equals",
                "values": [
                  "chimera_human_maintains_origin_identity_no_measurable_shift"
                ]
              },
              "chimera_monkey_identity_shift": {
                "op": "equals",
                "values": [
                  "chimera_monkey_unchanged_relative_to_control"
                ]
              }
            }
          },
          {
            "name": "chimera_monkey_paper_claim_on_confounded_input",
            "role": "hard_fail",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "value",
            "expected": "chimera_monkey_shifts_toward_chimera_human_identity"
          },
          {
            "name": "chimera_human_magnitude_wrong_attribution_signature",
            "role": "hard_fail",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "value",
            "op": "in",
            "expected": [
              "dramatic_complete_replacement_of_origin_identity",
              "magnitude_dominated_by_culture_or_metabolic_axis",
              "magnitude_dominated_by_developmental_stage_axis",
              "magnitude_dominated_by_cell_cycle_axis",
              "magnitude_dominated_by_batch_effect_axis"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-5-claude-code",
      "modelName": "claude-sonnet-5",
      "harness": "claude-code",
      "task": "gse155381_q8_bidirectional_convergence",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 3582.136926,
      "cost": 22.13638899999998,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "chimera_human_identity_shift_target_weighted",
            "role": "additive",
            "field": "$.chimera_human_identity_shift_target",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_in_vivo_state_assignment_weighted",
            "role": "additive",
            "field": "$.chimera_human_in_vivo_state_assignment",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_shift_magnitude_weighted",
            "role": "additive",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_monkey_identity_shift_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "both_arms_null_contradicted_by_calibration",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "chimera_human_identity_shift_target": {
                "op": "equals",
                "values": [
                  "chimera_human_maintains_origin_identity_no_measurable_shift"
                ]
              },
              "chimera_monkey_identity_shift": {
                "op": "equals",
                "values": [
                  "chimera_monkey_unchanged_relative_to_control"
                ]
              }
            }
          },
          {
            "name": "chimera_monkey_paper_claim_on_confounded_input",
            "role": "hard_fail",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "value",
            "expected": "chimera_monkey_shifts_toward_chimera_human_identity"
          },
          {
            "name": "chimera_human_magnitude_wrong_attribution_signature",
            "role": "hard_fail",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "value",
            "op": "in",
            "expected": [
              "dramatic_complete_replacement_of_origin_identity",
              "magnitude_dominated_by_culture_or_metabolic_axis",
              "magnitude_dominated_by_developmental_stage_axis",
              "magnitude_dominated_by_cell_cycle_axis",
              "magnitude_dominated_by_batch_effect_axis"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-5-claude-code",
      "modelName": "claude-sonnet-5",
      "harness": "claude-code",
      "task": "gse155381_q8_bidirectional_convergence",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 2211.901386,
      "cost": 17.035178649999995,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "chimera_human_identity_shift_target_weighted",
            "role": "additive",
            "field": "$.chimera_human_identity_shift_target",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_in_vivo_state_assignment_weighted",
            "role": "additive",
            "field": "$.chimera_human_in_vivo_state_assignment",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_shift_magnitude_weighted",
            "role": "additive",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_monkey_identity_shift_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "both_arms_null_contradicted_by_calibration",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "chimera_human_identity_shift_target": {
                "op": "equals",
                "values": [
                  "chimera_human_maintains_origin_identity_no_measurable_shift"
                ]
              },
              "chimera_monkey_identity_shift": {
                "op": "equals",
                "values": [
                  "chimera_monkey_unchanged_relative_to_control"
                ]
              }
            }
          },
          {
            "name": "chimera_monkey_paper_claim_on_confounded_input",
            "role": "hard_fail",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "value",
            "expected": "chimera_monkey_shifts_toward_chimera_human_identity"
          },
          {
            "name": "chimera_human_magnitude_wrong_attribution_signature",
            "role": "hard_fail",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "value",
            "op": "in",
            "expected": [
              "dramatic_complete_replacement_of_origin_identity",
              "magnitude_dominated_by_culture_or_metabolic_axis",
              "magnitude_dominated_by_developmental_stage_axis",
              "magnitude_dominated_by_cell_cycle_axis",
              "magnitude_dominated_by_batch_effect_axis"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-5-claude-code",
      "modelName": "claude-sonnet-5",
      "harness": "claude-code",
      "task": "gse171524_ectopic_airway_rare_cell",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 3545.170236,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "focus_population_identity_gate",
            "role": "gate",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "chemosensory_marker_expressing_population_alpha"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "focus_population_marker_calls",
            "kind": "list",
            "matchKey": "marker",
            "k": 3,
            "expected": [
              {
                "key": "POU2F3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "CHAT",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "TRPM5",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "LRMP",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNAT3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "AVIL",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNG13",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              }
            ]
          },
          {
            "name": "focus_population_condition_distribution_gate",
            "role": "gate",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_enriched_with_substantial_control_presence",
              "covid_predominant",
              "covid_only",
              "covid_enriched_trend_not_significant"
            ]
          },
          {
            "name": "focus_population_location_gate",
            "role": "gate",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "ectopic_parenchyma_in_covid",
              "parenchyma_only",
              "bronchiolized_or_peribronchial_parenchyma",
              "airway_and_parenchyma"
            ]
          },
          {
            "name": "focus_population_recovery_mode_gate",
            "role": "gate",
            "field": "$.focus_population_recovery_mode",
            "kind": "value",
            "op": "in",
            "expected": [
              "continuum_not_discrete_cluster"
            ]
          },
          {
            "name": "focus_population_identity_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "ionocyte_like_beta",
              "neuroendocrine_like_gamma",
              "basal_stem_progenitor_like_delta",
              "aberrant_basaloid_transitional_like_epsilon",
              "neuronal_or_neural_gene_minor_population_zeta",
              "deuterosomal_ciliated_progenitor_like_eta",
              "hillock_transitional_like_theta",
              "low_quality_doublet_or_debris",
              "no_rare_cell_recovered",
              "not_assessed"
            ]
          },
          {
            "name": "cf_wrong_lineage_marker_called_positive",
            "role": "hard_fail",
            "field": "$.focus_population_marker_calls[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "focus_population_condition_distribution_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "equally_distributed",
              "control_predominant",
              "not_assessed"
            ]
          },
          {
            "name": "focus_population_location_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "airway_only",
              "not_assessed"
            ]
          },
          {
            "name": "cross_axis_canonical_identity_with_airway_only_location_incoherence",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "focus_population_identity": {
                "op": "in",
                "values": [
                  "chemosensory_marker_expressing_population_alpha"
                ]
              },
              "focus_population_location": {
                "op": "in",
                "values": [
                  "airway_only"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-5-claude-code",
      "modelName": "claude-sonnet-5",
      "harness": "claude-code",
      "task": "gse171524_ectopic_airway_rare_cell",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 4972.44932,
      "cost": 4.994773249999999,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "focus_population_identity_gate",
            "role": "gate",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "chemosensory_marker_expressing_population_alpha"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "focus_population_marker_calls",
            "kind": "list",
            "matchKey": "marker",
            "k": 3,
            "expected": [
              {
                "key": "POU2F3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "CHAT",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "TRPM5",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "LRMP",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNAT3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "AVIL",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNG13",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              }
            ]
          },
          {
            "name": "focus_population_condition_distribution_gate",
            "role": "gate",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_enriched_with_substantial_control_presence",
              "covid_predominant",
              "covid_only",
              "covid_enriched_trend_not_significant"
            ]
          },
          {
            "name": "focus_population_location_gate",
            "role": "gate",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "ectopic_parenchyma_in_covid",
              "parenchyma_only",
              "bronchiolized_or_peribronchial_parenchyma",
              "airway_and_parenchyma"
            ]
          },
          {
            "name": "focus_population_recovery_mode_gate",
            "role": "gate",
            "field": "$.focus_population_recovery_mode",
            "kind": "value",
            "op": "in",
            "expected": [
              "continuum_not_discrete_cluster"
            ]
          },
          {
            "name": "focus_population_identity_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "ionocyte_like_beta",
              "neuroendocrine_like_gamma",
              "basal_stem_progenitor_like_delta",
              "aberrant_basaloid_transitional_like_epsilon",
              "neuronal_or_neural_gene_minor_population_zeta",
              "deuterosomal_ciliated_progenitor_like_eta",
              "hillock_transitional_like_theta",
              "low_quality_doublet_or_debris",
              "no_rare_cell_recovered",
              "not_assessed"
            ]
          },
          {
            "name": "cf_wrong_lineage_marker_called_positive",
            "role": "hard_fail",
            "field": "$.focus_population_marker_calls[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "focus_population_condition_distribution_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "equally_distributed",
              "control_predominant",
              "not_assessed"
            ]
          },
          {
            "name": "focus_population_location_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "airway_only",
              "not_assessed"
            ]
          },
          {
            "name": "cross_axis_canonical_identity_with_airway_only_location_incoherence",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "focus_population_identity": {
                "op": "in",
                "values": [
                  "chemosensory_marker_expressing_population_alpha"
                ]
              },
              "focus_population_location": {
                "op": "in",
                "values": [
                  "airway_only"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-5-claude-code",
      "modelName": "claude-sonnet-5",
      "harness": "claude-code",
      "task": "gse171524_ectopic_airway_rare_cell",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 8622.731302,
      "cost": 10.045180699999996,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "focus_population_identity_gate",
            "role": "gate",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "chemosensory_marker_expressing_population_alpha"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "focus_population_marker_calls",
            "kind": "list",
            "matchKey": "marker",
            "k": 3,
            "expected": [
              {
                "key": "POU2F3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "CHAT",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "TRPM5",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "LRMP",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNAT3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "AVIL",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNG13",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              }
            ]
          },
          {
            "name": "focus_population_condition_distribution_gate",
            "role": "gate",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_enriched_with_substantial_control_presence",
              "covid_predominant",
              "covid_only",
              "covid_enriched_trend_not_significant"
            ]
          },
          {
            "name": "focus_population_location_gate",
            "role": "gate",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "ectopic_parenchyma_in_covid",
              "parenchyma_only",
              "bronchiolized_or_peribronchial_parenchyma",
              "airway_and_parenchyma"
            ]
          },
          {
            "name": "focus_population_recovery_mode_gate",
            "role": "gate",
            "field": "$.focus_population_recovery_mode",
            "kind": "value",
            "op": "in",
            "expected": [
              "continuum_not_discrete_cluster"
            ]
          },
          {
            "name": "focus_population_identity_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "ionocyte_like_beta",
              "neuroendocrine_like_gamma",
              "basal_stem_progenitor_like_delta",
              "aberrant_basaloid_transitional_like_epsilon",
              "neuronal_or_neural_gene_minor_population_zeta",
              "deuterosomal_ciliated_progenitor_like_eta",
              "hillock_transitional_like_theta",
              "low_quality_doublet_or_debris",
              "no_rare_cell_recovered",
              "not_assessed"
            ]
          },
          {
            "name": "cf_wrong_lineage_marker_called_positive",
            "role": "hard_fail",
            "field": "$.focus_population_marker_calls[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "focus_population_condition_distribution_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "equally_distributed",
              "control_predominant",
              "not_assessed"
            ]
          },
          {
            "name": "focus_population_location_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "airway_only",
              "not_assessed"
            ]
          },
          {
            "name": "cross_axis_canonical_identity_with_airway_only_location_incoherence",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "focus_population_identity": {
                "op": "in",
                "values": [
                  "chemosensory_marker_expressing_population_alpha"
                ]
              },
              "focus_population_location": {
                "op": "in",
                "values": [
                  "airway_only"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-5-claude-code",
      "modelName": "claude-sonnet-5",
      "harness": "claude-code",
      "task": "gse171524_pathological_fibroblast_master_regulator_druggable_targets",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 4822.478364,
      "cost": 2.0810937499999995,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "disease_associated_fibroblast_subtype_weighted",
            "role": "additive",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "mr_tf_junb_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_alpha"
            ]
          },
          {
            "name": "mr_tf_jund_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_beta"
            ]
          },
          {
            "name": "drug_mmp14_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_alpha"
            ]
          },
          {
            "name": "drug_stat3_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_beta"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_weighted",
            "role": "additive",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "disease_associated_fibroblast_subtype_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "value",
            "op": "in",
            "expected": [
              "subtype_lambda",
              "subtype_epsilon",
              "subtype_iota",
              "subtype_beta",
              "subtype_eta",
              "subtype_zeta",
              "subtype_not_assessed"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "simple_de_ranking_by_log_fc",
              "marker_based_de_only",
              "not_computed"
            ]
          },
          {
            "name": "top_master_regulator_transcription_factors_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_not_computed"
            ]
          },
          {
            "name": "top_druggable_proteins_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_not_assessed"
            ]
          },
          {
            "name": "cross_axis_circular_marker_in_druggable_proteins",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_gamma",
              "drug_target_delta",
              "drug_target_eta"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-5-claude-code",
      "modelName": "claude-sonnet-5",
      "harness": "claude-code",
      "task": "gse171524_pathological_fibroblast_master_regulator_druggable_targets",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 6036.060679,
      "cost": 2.3576580499999995,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "disease_associated_fibroblast_subtype_weighted",
            "role": "additive",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "mr_tf_junb_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_alpha"
            ]
          },
          {
            "name": "mr_tf_jund_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_beta"
            ]
          },
          {
            "name": "drug_mmp14_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_alpha"
            ]
          },
          {
            "name": "drug_stat3_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_beta"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_weighted",
            "role": "additive",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "disease_associated_fibroblast_subtype_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "value",
            "op": "in",
            "expected": [
              "subtype_lambda",
              "subtype_epsilon",
              "subtype_iota",
              "subtype_beta",
              "subtype_eta",
              "subtype_zeta",
              "subtype_not_assessed"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "simple_de_ranking_by_log_fc",
              "marker_based_de_only",
              "not_computed"
            ]
          },
          {
            "name": "top_master_regulator_transcription_factors_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_not_computed"
            ]
          },
          {
            "name": "top_druggable_proteins_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_not_assessed"
            ]
          },
          {
            "name": "cross_axis_circular_marker_in_druggable_proteins",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_gamma",
              "drug_target_delta",
              "drug_target_eta"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-5-claude-code",
      "modelName": "claude-sonnet-5",
      "harness": "claude-code",
      "task": "gse171524_pathological_fibroblast_master_regulator_druggable_targets",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 9254.720351,
      "cost": 2.5862114000000007,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "disease_associated_fibroblast_subtype_weighted",
            "role": "additive",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "mr_tf_junb_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_alpha"
            ]
          },
          {
            "name": "mr_tf_jund_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_beta"
            ]
          },
          {
            "name": "drug_mmp14_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_alpha"
            ]
          },
          {
            "name": "drug_stat3_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_beta"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_weighted",
            "role": "additive",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "disease_associated_fibroblast_subtype_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "value",
            "op": "in",
            "expected": [
              "subtype_lambda",
              "subtype_epsilon",
              "subtype_iota",
              "subtype_beta",
              "subtype_eta",
              "subtype_zeta",
              "subtype_not_assessed"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "simple_de_ranking_by_log_fc",
              "marker_based_de_only",
              "not_computed"
            ]
          },
          {
            "name": "top_master_regulator_transcription_factors_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_not_computed"
            ]
          },
          {
            "name": "top_druggable_proteins_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_not_assessed"
            ]
          },
          {
            "name": "cross_axis_circular_marker_in_druggable_proteins",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_gamma",
              "drug_target_delta",
              "drug_target_eta"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-5-claude-code",
      "modelName": "claude-sonnet-5",
      "harness": "claude-code",
      "task": "gse171524_plasma_bcr_neutralizing_clonotype",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 1401.287655,
      "cost": 2.4024415500000003,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dominant_pairing_heavy_gate",
            "role": "gate",
            "field": "$.dominant_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV4-34"
            ]
          },
          {
            "name": "dominant_pairing_light_gate",
            "role": "gate",
            "field": "$.dominant_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGLV3-19"
            ]
          },
          {
            "name": "dominant_neutralizer_match_gate",
            "role": "gate",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_known_neutralizer_match"
            ]
          },
          {
            "name": "neutralizer_pairing_heavy_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV1-18"
            ]
          },
          {
            "name": "neutralizer_pairing_light_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGKV3-20"
            ]
          },
          {
            "name": "neutralizer_recurrence_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "observed_in_2_to_5_donors"
            ]
          },
          {
            "name": "neutralizer_rank_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "top_5_pairing",
              "top_6_to_20_pairing"
            ]
          },
          {
            "name": "dominant_clone_isotype_gate",
            "role": "gate",
            "field": "$.dominant_clone_isotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "IgG"
            ]
          },
          {
            "name": "blineage_class_switch_contrast_gate",
            "role": "gate",
            "field": "$.blineage_class_switch_contrast",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_class_switched_control_higher_unswitched",
              "class_switched_in_both_cohorts"
            ]
          },
          {
            "name": "dominant_match_narrative_trap_or_null",
            "role": "hard_fail",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_rbd_neutralizer",
              "matches_known_autoreactive_clone",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_heavy",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_light",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_recurrence_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_rank_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "cross_axis_found_dominant_but_failed_to_scan_for_embedded_neutralizer",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_pairing_neutralizer_match": {
                "op": "in",
                "values": [
                  "no_known_neutralizer_match"
                ]
              },
              "neutralizer_pairing_heavy": {
                "op": "in",
                "values": [
                  "none_detected"
                ]
              }
            }
          },
          {
            "name": "neutralizer_pairing_not_exactly_the_s309_pair",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "op": "not",
            "expected": {
              "op": "and",
              "args": [
                {
                  "op": "field",
                  "name": "neutralizer_pairing_heavy",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGHV1-18"
                    ]
                  }
                },
                {
                  "op": "field",
                  "name": "neutralizer_pairing_light",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGKV3-20"
                    ]
                  }
                }
              ]
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-5-claude-code",
      "modelName": "claude-sonnet-5",
      "harness": "claude-code",
      "task": "gse171524_plasma_bcr_neutralizing_clonotype",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 1554.215438,
      "cost": 2.1628610500000005,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dominant_pairing_heavy_gate",
            "role": "gate",
            "field": "$.dominant_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV4-34"
            ]
          },
          {
            "name": "dominant_pairing_light_gate",
            "role": "gate",
            "field": "$.dominant_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGLV3-19"
            ]
          },
          {
            "name": "dominant_neutralizer_match_gate",
            "role": "gate",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_known_neutralizer_match"
            ]
          },
          {
            "name": "neutralizer_pairing_heavy_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV1-18"
            ]
          },
          {
            "name": "neutralizer_pairing_light_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGKV3-20"
            ]
          },
          {
            "name": "neutralizer_recurrence_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "observed_in_2_to_5_donors"
            ]
          },
          {
            "name": "neutralizer_rank_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "top_5_pairing",
              "top_6_to_20_pairing"
            ]
          },
          {
            "name": "dominant_clone_isotype_gate",
            "role": "gate",
            "field": "$.dominant_clone_isotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "IgG"
            ]
          },
          {
            "name": "blineage_class_switch_contrast_gate",
            "role": "gate",
            "field": "$.blineage_class_switch_contrast",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_class_switched_control_higher_unswitched",
              "class_switched_in_both_cohorts"
            ]
          },
          {
            "name": "dominant_match_narrative_trap_or_null",
            "role": "hard_fail",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_rbd_neutralizer",
              "matches_known_autoreactive_clone",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_heavy",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_light",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_recurrence_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_rank_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "cross_axis_found_dominant_but_failed_to_scan_for_embedded_neutralizer",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_pairing_neutralizer_match": {
                "op": "in",
                "values": [
                  "no_known_neutralizer_match"
                ]
              },
              "neutralizer_pairing_heavy": {
                "op": "in",
                "values": [
                  "none_detected"
                ]
              }
            }
          },
          {
            "name": "neutralizer_pairing_not_exactly_the_s309_pair",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "op": "not",
            "expected": {
              "op": "and",
              "args": [
                {
                  "op": "field",
                  "name": "neutralizer_pairing_heavy",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGHV1-18"
                    ]
                  }
                },
                {
                  "op": "field",
                  "name": "neutralizer_pairing_light",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGKV3-20"
                    ]
                  }
                }
              ]
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-5-claude-code",
      "modelName": "claude-sonnet-5",
      "harness": "claude-code",
      "task": "gse171524_plasma_bcr_neutralizing_clonotype",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1056.676071,
      "cost": 2.9763178500000005,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dominant_pairing_heavy_gate",
            "role": "gate",
            "field": "$.dominant_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV4-34"
            ]
          },
          {
            "name": "dominant_pairing_light_gate",
            "role": "gate",
            "field": "$.dominant_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGLV3-19"
            ]
          },
          {
            "name": "dominant_neutralizer_match_gate",
            "role": "gate",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_known_neutralizer_match"
            ]
          },
          {
            "name": "neutralizer_pairing_heavy_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV1-18"
            ]
          },
          {
            "name": "neutralizer_pairing_light_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGKV3-20"
            ]
          },
          {
            "name": "neutralizer_recurrence_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "observed_in_2_to_5_donors"
            ]
          },
          {
            "name": "neutralizer_rank_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "top_5_pairing",
              "top_6_to_20_pairing"
            ]
          },
          {
            "name": "dominant_clone_isotype_gate",
            "role": "gate",
            "field": "$.dominant_clone_isotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "IgG"
            ]
          },
          {
            "name": "blineage_class_switch_contrast_gate",
            "role": "gate",
            "field": "$.blineage_class_switch_contrast",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_class_switched_control_higher_unswitched",
              "class_switched_in_both_cohorts"
            ]
          },
          {
            "name": "dominant_match_narrative_trap_or_null",
            "role": "hard_fail",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_rbd_neutralizer",
              "matches_known_autoreactive_clone",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_heavy",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_light",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_recurrence_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_rank_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "cross_axis_found_dominant_but_failed_to_scan_for_embedded_neutralizer",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_pairing_neutralizer_match": {
                "op": "in",
                "values": [
                  "no_known_neutralizer_match"
                ]
              },
              "neutralizer_pairing_heavy": {
                "op": "in",
                "values": [
                  "none_detected"
                ]
              }
            }
          },
          {
            "name": "neutralizer_pairing_not_exactly_the_s309_pair",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "op": "not",
            "expected": {
              "op": "and",
              "args": [
                {
                  "op": "field",
                  "name": "neutralizer_pairing_heavy",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGHV1-18"
                    ]
                  }
                },
                {
                  "op": "field",
                  "name": "neutralizer_pairing_light",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGKV3-20"
                    ]
                  }
                }
              ]
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-5-claude-code",
      "modelName": "claude-sonnet-5",
      "harness": "claude-code",
      "task": "q6_myeloid_subset_restricted_de",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 4122.269049,
      "cost": 4.84731135,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "myeloid_subset_with_strongest_change_gate",
            "role": "gate",
            "field": "$.myeloid_subset_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "subset_gamma"
            ]
          },
          {
            "name": "gene_with_strongest_change_gate",
            "role": "gate",
            "field": "$.gene_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "AXL",
              "PROS1"
            ]
          },
          {
            "name": "direction_of_change_gate",
            "role": "gate",
            "field": "$.direction_of_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "down_in_covid"
            ]
          },
          {
            "name": "biological_function_interpretation_gate",
            "role": "gate",
            "field": "$.biological_function_interpretation",
            "kind": "value",
            "op": "in",
            "expected": [
              "efferocytosis",
              "phagocytosis"
            ]
          },
          {
            "name": "myeloid_compartment_abundance_gate",
            "role": "gate",
            "field": "$.myeloid_compartment_abundance",
            "kind": "value",
            "op": "in",
            "expected": [
              "expands_in_covid"
            ]
          },
          {
            "name": "myeloid_expansion_localization_gate",
            "role": "gate",
            "field": "$.myeloid_expansion_localization",
            "kind": "value",
            "op": "in",
            "expected": [
              "recruited_driven_resident_dysfunction"
            ]
          },
          {
            "name": "dominant_signature_gate",
            "role": "gate",
            "field": "$.dominant_signature",
            "kind": "value",
            "op": "in",
            "expected": [
              "technical_or_agonal_artifact"
            ]
          },
          {
            "name": "dominant_signature_direction_gate",
            "role": "gate",
            "field": "$.dominant_signature_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_in_covid"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-5-claude-code",
      "modelName": "claude-sonnet-5",
      "harness": "claude-code",
      "task": "q6_myeloid_subset_restricted_de",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 3303.081078,
      "cost": 4.8636082499999995,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "myeloid_subset_with_strongest_change_gate",
            "role": "gate",
            "field": "$.myeloid_subset_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "subset_gamma"
            ]
          },
          {
            "name": "gene_with_strongest_change_gate",
            "role": "gate",
            "field": "$.gene_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "AXL",
              "PROS1"
            ]
          },
          {
            "name": "direction_of_change_gate",
            "role": "gate",
            "field": "$.direction_of_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "down_in_covid"
            ]
          },
          {
            "name": "biological_function_interpretation_gate",
            "role": "gate",
            "field": "$.biological_function_interpretation",
            "kind": "value",
            "op": "in",
            "expected": [
              "efferocytosis",
              "phagocytosis"
            ]
          },
          {
            "name": "myeloid_compartment_abundance_gate",
            "role": "gate",
            "field": "$.myeloid_compartment_abundance",
            "kind": "value",
            "op": "in",
            "expected": [
              "expands_in_covid"
            ]
          },
          {
            "name": "myeloid_expansion_localization_gate",
            "role": "gate",
            "field": "$.myeloid_expansion_localization",
            "kind": "value",
            "op": "in",
            "expected": [
              "recruited_driven_resident_dysfunction"
            ]
          },
          {
            "name": "dominant_signature_gate",
            "role": "gate",
            "field": "$.dominant_signature",
            "kind": "value",
            "op": "in",
            "expected": [
              "technical_or_agonal_artifact"
            ]
          },
          {
            "name": "dominant_signature_direction_gate",
            "role": "gate",
            "field": "$.dominant_signature_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_in_covid"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-5-claude-code",
      "modelName": "claude-sonnet-5",
      "harness": "claude-code",
      "task": "q6_myeloid_subset_restricted_de",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 4768.019547,
      "cost": 5.302013150000001,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "myeloid_subset_with_strongest_change_gate",
            "role": "gate",
            "field": "$.myeloid_subset_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "subset_gamma"
            ]
          },
          {
            "name": "gene_with_strongest_change_gate",
            "role": "gate",
            "field": "$.gene_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "AXL",
              "PROS1"
            ]
          },
          {
            "name": "direction_of_change_gate",
            "role": "gate",
            "field": "$.direction_of_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "down_in_covid"
            ]
          },
          {
            "name": "biological_function_interpretation_gate",
            "role": "gate",
            "field": "$.biological_function_interpretation",
            "kind": "value",
            "op": "in",
            "expected": [
              "efferocytosis",
              "phagocytosis"
            ]
          },
          {
            "name": "myeloid_compartment_abundance_gate",
            "role": "gate",
            "field": "$.myeloid_compartment_abundance",
            "kind": "value",
            "op": "in",
            "expected": [
              "expands_in_covid"
            ]
          },
          {
            "name": "myeloid_expansion_localization_gate",
            "role": "gate",
            "field": "$.myeloid_expansion_localization",
            "kind": "value",
            "op": "in",
            "expected": [
              "recruited_driven_resident_dysfunction"
            ]
          },
          {
            "name": "dominant_signature_gate",
            "role": "gate",
            "field": "$.dominant_signature",
            "kind": "value",
            "op": "in",
            "expected": [
              "technical_or_agonal_artifact"
            ]
          },
          {
            "name": "dominant_signature_direction_gate",
            "role": "gate",
            "field": "$.dominant_signature_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_in_covid"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-5-claude-code",
      "modelName": "claude-sonnet-5",
      "harness": "claude-code",
      "task": "snrnaseq_per_donor_composition_shifts_lethal_covid_lung",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 4472.973182,
      "cost": 2.26998245,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dict_match",
            "role": "gate",
            "field": "cell_type_direction_map",
            "kind": "dict",
            "expected": {
              "Myeloid": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "Epithelia": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Fibroblasts": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "T_NK": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "B_plasma": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Endothelial": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Mast": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "APC": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Neuronal": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              }
            }
          },
          {
            "name": "over_merge_immune_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Immune",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_leukocyte_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Leukocyte",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_lymphoid_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Lymphoid",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_stromal_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Stromal",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-5-claude-code",
      "modelName": "claude-sonnet-5",
      "harness": "claude-code",
      "task": "snrnaseq_per_donor_composition_shifts_lethal_covid_lung",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 4132.548935,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dict_match",
            "role": "gate",
            "field": "cell_type_direction_map",
            "kind": "dict",
            "expected": {
              "Myeloid": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "Epithelia": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Fibroblasts": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "T_NK": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "B_plasma": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Endothelial": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Mast": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "APC": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Neuronal": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              }
            }
          },
          {
            "name": "over_merge_immune_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Immune",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_leukocyte_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Leukocyte",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_lymphoid_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Lymphoid",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_stromal_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Stromal",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-5-claude-code",
      "modelName": "claude-sonnet-5",
      "harness": "claude-code",
      "task": "snrnaseq_per_donor_composition_shifts_lethal_covid_lung",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 2990.845934,
      "cost": 3.5005523499999995,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dict_match",
            "role": "gate",
            "field": "cell_type_direction_map",
            "kind": "dict",
            "expected": {
              "Myeloid": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "Epithelia": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Fibroblasts": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "T_NK": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "B_plasma": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Endothelial": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Mast": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "APC": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Neuronal": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              }
            }
          },
          {
            "name": "over_merge_immune_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Immune",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_leukocyte_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Leukocyte",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_lymphoid_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Lymphoid",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_stromal_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Stromal",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-5-claude-code",
      "modelName": "claude-sonnet-5",
      "harness": "claude-code",
      "task": "10x_5p_melanoma_cluster_enriched_apc_states_v2",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 3009.659251,
      "cost": 17.9474,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "top_3_cluster_enriched_apc_states",
            "kind": "list",
            "matchKey": "apc_state",
            "k": 3,
            "expected": [
              {
                "key": "plasmacytoid_DC",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "T_cell_stimulatory_role_supported",
                      "role_direction_unresolved"
                    ],
                    "canonical": "T_cell_stimulatory_role_supported"
                  }
                }
              },
              {
                "key": "C1qhigh_lipid_associated_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "C1qhigh_inflammatory_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "plasma_cell",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "cohort_significant_per_patient_inconsistent"
                    ],
                    "canonical": "cohort_significant_per_patient_inconsistent"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CD80_CD86": {
                "op": "in",
                "value": [
                  "mreg_DC",
                  "shared_across_APC_states"
                ]
              },
              "PD_L1_in_APC_clusters": {
                "op": "in",
                "value": [
                  "C1qhigh_macrophage_either_subtype",
                  "shared_across_APC_states",
                  "mreg_DC"
                ]
              },
              "type_I_IFN": {
                "op": "in",
                "value": [
                  "plasmacytoid_DC",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "cf_mreg_dc_claimed_enriched",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "mreg_DC",
              "mature_DC_enriched_in_immunoregulatory_molecules"
            ]
          },
          {
            "name": "cf_framework_offcompartment_or_generic_substitution",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "cDC1",
              "Maier_mregDC",
              "Cheng_C1QC_TAM",
              "Cheng_SPP1_TAM",
              "Cheng_FOLR2_TAM",
              "Villani_pDC",
              "Zhang_inflammatory_monocyte",
              "LAMP3_pos_DC",
              "myeloid_derived_suppressor_cell",
              "MDSC",
              "tumor_associated_neutrophil",
              "mast_cell",
              "APC",
              "antigen_presenting_cell",
              "macrophage",
              "dendritic_cell",
              "tumor_associated_macrophage",
              "TAM"
            ]
          },
          {
            "name": "cf_direction_commit_on_unresolvable_slot",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_not_resolvable_hedge",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].functional_attribution_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_type_I_IFN_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.type_I_IFN",
            "kind": "value",
            "op": "in",
            "expected": [
              "mreg_DC",
              "C1qhigh_macrophage_either_subtype"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-5-claude-code",
      "modelName": "claude-sonnet-5",
      "harness": "claude-code",
      "task": "10x_5p_melanoma_cluster_enriched_apc_states_v2",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 3345.965942,
      "cost": 6.069106850000002,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "top_3_cluster_enriched_apc_states",
            "kind": "list",
            "matchKey": "apc_state",
            "k": 3,
            "expected": [
              {
                "key": "plasmacytoid_DC",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "T_cell_stimulatory_role_supported",
                      "role_direction_unresolved"
                    ],
                    "canonical": "T_cell_stimulatory_role_supported"
                  }
                }
              },
              {
                "key": "C1qhigh_lipid_associated_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "C1qhigh_inflammatory_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "plasma_cell",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "cohort_significant_per_patient_inconsistent"
                    ],
                    "canonical": "cohort_significant_per_patient_inconsistent"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CD80_CD86": {
                "op": "in",
                "value": [
                  "mreg_DC",
                  "shared_across_APC_states"
                ]
              },
              "PD_L1_in_APC_clusters": {
                "op": "in",
                "value": [
                  "C1qhigh_macrophage_either_subtype",
                  "shared_across_APC_states",
                  "mreg_DC"
                ]
              },
              "type_I_IFN": {
                "op": "in",
                "value": [
                  "plasmacytoid_DC",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "cf_mreg_dc_claimed_enriched",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "mreg_DC",
              "mature_DC_enriched_in_immunoregulatory_molecules"
            ]
          },
          {
            "name": "cf_framework_offcompartment_or_generic_substitution",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "cDC1",
              "Maier_mregDC",
              "Cheng_C1QC_TAM",
              "Cheng_SPP1_TAM",
              "Cheng_FOLR2_TAM",
              "Villani_pDC",
              "Zhang_inflammatory_monocyte",
              "LAMP3_pos_DC",
              "myeloid_derived_suppressor_cell",
              "MDSC",
              "tumor_associated_neutrophil",
              "mast_cell",
              "APC",
              "antigen_presenting_cell",
              "macrophage",
              "dendritic_cell",
              "tumor_associated_macrophage",
              "TAM"
            ]
          },
          {
            "name": "cf_direction_commit_on_unresolvable_slot",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_not_resolvable_hedge",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].functional_attribution_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_type_I_IFN_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.type_I_IFN",
            "kind": "value",
            "op": "in",
            "expected": [
              "mreg_DC",
              "C1qhigh_macrophage_either_subtype"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-5-claude-code",
      "modelName": "claude-sonnet-5",
      "harness": "claude-code",
      "task": "10x_5p_melanoma_cluster_enriched_apc_states_v2",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 3504.660791,
      "cost": 5.590457299999999,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "top_3_cluster_enriched_apc_states",
            "kind": "list",
            "matchKey": "apc_state",
            "k": 3,
            "expected": [
              {
                "key": "plasmacytoid_DC",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "T_cell_stimulatory_role_supported",
                      "role_direction_unresolved"
                    ],
                    "canonical": "T_cell_stimulatory_role_supported"
                  }
                }
              },
              {
                "key": "C1qhigh_lipid_associated_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "C1qhigh_inflammatory_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "plasma_cell",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "cohort_significant_per_patient_inconsistent"
                    ],
                    "canonical": "cohort_significant_per_patient_inconsistent"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CD80_CD86": {
                "op": "in",
                "value": [
                  "mreg_DC",
                  "shared_across_APC_states"
                ]
              },
              "PD_L1_in_APC_clusters": {
                "op": "in",
                "value": [
                  "C1qhigh_macrophage_either_subtype",
                  "shared_across_APC_states",
                  "mreg_DC"
                ]
              },
              "type_I_IFN": {
                "op": "in",
                "value": [
                  "plasmacytoid_DC",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "cf_mreg_dc_claimed_enriched",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "mreg_DC",
              "mature_DC_enriched_in_immunoregulatory_molecules"
            ]
          },
          {
            "name": "cf_framework_offcompartment_or_generic_substitution",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "cDC1",
              "Maier_mregDC",
              "Cheng_C1QC_TAM",
              "Cheng_SPP1_TAM",
              "Cheng_FOLR2_TAM",
              "Villani_pDC",
              "Zhang_inflammatory_monocyte",
              "LAMP3_pos_DC",
              "myeloid_derived_suppressor_cell",
              "MDSC",
              "tumor_associated_neutrophil",
              "mast_cell",
              "APC",
              "antigen_presenting_cell",
              "macrophage",
              "dendritic_cell",
              "tumor_associated_macrophage",
              "TAM"
            ]
          },
          {
            "name": "cf_direction_commit_on_unresolvable_slot",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_not_resolvable_hedge",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].functional_attribution_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_type_I_IFN_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.type_I_IFN",
            "kind": "value",
            "op": "in",
            "expected": [
              "mreg_DC",
              "C1qhigh_macrophage_either_subtype"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-5-claude-code",
      "modelName": "claude-sonnet-5",
      "harness": "claude-code",
      "task": "10x_5p_melanoma_cluster_tumor_state_causation_v5",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 3132.437399,
      "cost": 8.230613000000002,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "enriched_tumor_states",
            "kind": "list",
            "matchKey": "tumor_state",
            "k": 3,
            "expected": [
              {
                "key": "immune_response",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "direction_unresolved_cross_sectional"
                    ],
                    "canonical": "direction_unresolved_cross_sectional"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "IFN_response_axis",
                      "broad_antigen_presentation_MHC"
                    ],
                    "canonical": "IFN_response_axis"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "enrichment_diminished_by_decontamination",
                      "threshold_sensitive"
                    ],
                    "canonical": "enrichment_diminished_by_decontamination"
                  }
                }
              },
              {
                "key": "stress_hypoxia",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "data_supports_cell_intrinsic_cause"
                    ],
                    "canonical": "data_supports_cell_intrinsic_cause"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "HIF_hypoxia_program_specific"
                    ],
                    "canonical": "HIF_hypoxia_program_specific"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "robust_across_methods",
                      "robust_after_decontamination"
                    ],
                    "canonical": "robust_across_methods"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CXCL9_CXCL10": {
                "op": "in",
                "value": [
                  "immune_response_tumor_state"
                ]
              },
              "CCL5": {
                "op": "in",
                "value": [
                  "shared_across_tumor_states",
                  "partner_lymphocyte_derived",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "gate_conjugate_gate_confounder",
            "role": "gate",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "partner_lymphocyte_capture"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "antigen_presentation_decomposition",
            "kind": "dict",
            "expected": {
              "MHC_class_I": {
                "op": "in",
                "value": [
                  "co_regulated_with_dominant_program",
                  "tumor_intrinsic_enriched_independent"
                ]
              },
              "MHC_class_II": {
                "op": "in",
                "value": [
                  "partner_cell_attributable",
                  "flat_not_enriched",
                  "method_sensitive_no_robust_call",
                  "co_regulated_with_dominant_program"
                ]
              }
            }
          },
          {
            "name": "cf_non_robust_state_claimed_enriched",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "transitory_melanocytic",
              "intermediate_melanocytic",
              "melanocytic",
              "MITF_high",
              "neural_crest_like",
              "mitotic",
              "stress_p53",
              "undifferentiated",
              "dedifferentiated",
              "mesenchymal_tumor",
              "AXL_high_tumor",
              "invasive_tumor",
              "senescence_like",
              "EMT_transition",
              "cancer_stem_cell_like",
              "oxidative_stress_tumor"
            ]
          },
          {
            "name": "cf_framework_generic_or_offtissue_substitution",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "Tsoi_undifferentiated",
              "Tsoi_neural_crest_like",
              "Tsoi_transitory",
              "Tsoi_melanocytic",
              "Rambow_NCSC",
              "Wouters_mesenchymal",
              "Pozniak_TCF4_dependent",
              "tumor_cell",
              "melanoma_cell",
              "malignant_cell",
              "tumor_subpopulation",
              "keratinocyte",
              "hepatocyte",
              "lung_adenocarcinoma_cell",
              "colon_adenocarcinoma_cell",
              "Tirosh_cell_cycle_program",
              "Jerby_Arnon_program",
              "glioblastoma_cell",
              "breast_carcinoma_cell"
            ]
          },
          {
            "name": "cf_immune_response_direction_committed",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_immune_robustness_recall",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_ambiguous_hedge",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].causation_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_stress_generic_stress_response_evidence",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_conjugate_confounder_contamination_denied",
            "role": "hard_fail",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected"
            ]
          },
          {
            "name": "cf_ccl5_recall_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CCL5",
            "kind": "avoid",
            "expected": [
              "immune_response_tumor_state"
            ]
          },
          {
            "name": "cf_cxcl9_cxcl10_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CXCL9_CXCL10",
            "kind": "avoid",
            "expected": [
              "stress_hypoxia_tumor_state"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-5-claude-code",
      "modelName": "claude-sonnet-5",
      "harness": "claude-code",
      "task": "10x_5p_melanoma_cluster_tumor_state_causation_v5",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 3078.142045,
      "cost": 11.777687599999997,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "enriched_tumor_states",
            "kind": "list",
            "matchKey": "tumor_state",
            "k": 3,
            "expected": [
              {
                "key": "immune_response",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "direction_unresolved_cross_sectional"
                    ],
                    "canonical": "direction_unresolved_cross_sectional"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "IFN_response_axis",
                      "broad_antigen_presentation_MHC"
                    ],
                    "canonical": "IFN_response_axis"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "enrichment_diminished_by_decontamination",
                      "threshold_sensitive"
                    ],
                    "canonical": "enrichment_diminished_by_decontamination"
                  }
                }
              },
              {
                "key": "stress_hypoxia",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "data_supports_cell_intrinsic_cause"
                    ],
                    "canonical": "data_supports_cell_intrinsic_cause"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "HIF_hypoxia_program_specific"
                    ],
                    "canonical": "HIF_hypoxia_program_specific"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "robust_across_methods",
                      "robust_after_decontamination"
                    ],
                    "canonical": "robust_across_methods"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CXCL9_CXCL10": {
                "op": "in",
                "value": [
                  "immune_response_tumor_state"
                ]
              },
              "CCL5": {
                "op": "in",
                "value": [
                  "shared_across_tumor_states",
                  "partner_lymphocyte_derived",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "gate_conjugate_gate_confounder",
            "role": "gate",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "partner_lymphocyte_capture"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "antigen_presentation_decomposition",
            "kind": "dict",
            "expected": {
              "MHC_class_I": {
                "op": "in",
                "value": [
                  "co_regulated_with_dominant_program",
                  "tumor_intrinsic_enriched_independent"
                ]
              },
              "MHC_class_II": {
                "op": "in",
                "value": [
                  "partner_cell_attributable",
                  "flat_not_enriched",
                  "method_sensitive_no_robust_call",
                  "co_regulated_with_dominant_program"
                ]
              }
            }
          },
          {
            "name": "cf_non_robust_state_claimed_enriched",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "transitory_melanocytic",
              "intermediate_melanocytic",
              "melanocytic",
              "MITF_high",
              "neural_crest_like",
              "mitotic",
              "stress_p53",
              "undifferentiated",
              "dedifferentiated",
              "mesenchymal_tumor",
              "AXL_high_tumor",
              "invasive_tumor",
              "senescence_like",
              "EMT_transition",
              "cancer_stem_cell_like",
              "oxidative_stress_tumor"
            ]
          },
          {
            "name": "cf_framework_generic_or_offtissue_substitution",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "Tsoi_undifferentiated",
              "Tsoi_neural_crest_like",
              "Tsoi_transitory",
              "Tsoi_melanocytic",
              "Rambow_NCSC",
              "Wouters_mesenchymal",
              "Pozniak_TCF4_dependent",
              "tumor_cell",
              "melanoma_cell",
              "malignant_cell",
              "tumor_subpopulation",
              "keratinocyte",
              "hepatocyte",
              "lung_adenocarcinoma_cell",
              "colon_adenocarcinoma_cell",
              "Tirosh_cell_cycle_program",
              "Jerby_Arnon_program",
              "glioblastoma_cell",
              "breast_carcinoma_cell"
            ]
          },
          {
            "name": "cf_immune_response_direction_committed",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_immune_robustness_recall",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_ambiguous_hedge",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].causation_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_stress_generic_stress_response_evidence",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_conjugate_confounder_contamination_denied",
            "role": "hard_fail",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected"
            ]
          },
          {
            "name": "cf_ccl5_recall_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CCL5",
            "kind": "avoid",
            "expected": [
              "immune_response_tumor_state"
            ]
          },
          {
            "name": "cf_cxcl9_cxcl10_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CXCL9_CXCL10",
            "kind": "avoid",
            "expected": [
              "stress_hypoxia_tumor_state"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-5-claude-code",
      "modelName": "claude-sonnet-5",
      "harness": "claude-code",
      "task": "10x_5p_melanoma_cluster_tumor_state_causation_v5",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 3000.766175,
      "cost": 14.906066849999997,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "enriched_tumor_states",
            "kind": "list",
            "matchKey": "tumor_state",
            "k": 3,
            "expected": [
              {
                "key": "immune_response",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "direction_unresolved_cross_sectional"
                    ],
                    "canonical": "direction_unresolved_cross_sectional"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "IFN_response_axis",
                      "broad_antigen_presentation_MHC"
                    ],
                    "canonical": "IFN_response_axis"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "enrichment_diminished_by_decontamination",
                      "threshold_sensitive"
                    ],
                    "canonical": "enrichment_diminished_by_decontamination"
                  }
                }
              },
              {
                "key": "stress_hypoxia",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "data_supports_cell_intrinsic_cause"
                    ],
                    "canonical": "data_supports_cell_intrinsic_cause"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "HIF_hypoxia_program_specific"
                    ],
                    "canonical": "HIF_hypoxia_program_specific"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "robust_across_methods",
                      "robust_after_decontamination"
                    ],
                    "canonical": "robust_across_methods"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CXCL9_CXCL10": {
                "op": "in",
                "value": [
                  "immune_response_tumor_state"
                ]
              },
              "CCL5": {
                "op": "in",
                "value": [
                  "shared_across_tumor_states",
                  "partner_lymphocyte_derived",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "gate_conjugate_gate_confounder",
            "role": "gate",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "partner_lymphocyte_capture"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "antigen_presentation_decomposition",
            "kind": "dict",
            "expected": {
              "MHC_class_I": {
                "op": "in",
                "value": [
                  "co_regulated_with_dominant_program",
                  "tumor_intrinsic_enriched_independent"
                ]
              },
              "MHC_class_II": {
                "op": "in",
                "value": [
                  "partner_cell_attributable",
                  "flat_not_enriched",
                  "method_sensitive_no_robust_call",
                  "co_regulated_with_dominant_program"
                ]
              }
            }
          },
          {
            "name": "cf_non_robust_state_claimed_enriched",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "transitory_melanocytic",
              "intermediate_melanocytic",
              "melanocytic",
              "MITF_high",
              "neural_crest_like",
              "mitotic",
              "stress_p53",
              "undifferentiated",
              "dedifferentiated",
              "mesenchymal_tumor",
              "AXL_high_tumor",
              "invasive_tumor",
              "senescence_like",
              "EMT_transition",
              "cancer_stem_cell_like",
              "oxidative_stress_tumor"
            ]
          },
          {
            "name": "cf_framework_generic_or_offtissue_substitution",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "Tsoi_undifferentiated",
              "Tsoi_neural_crest_like",
              "Tsoi_transitory",
              "Tsoi_melanocytic",
              "Rambow_NCSC",
              "Wouters_mesenchymal",
              "Pozniak_TCF4_dependent",
              "tumor_cell",
              "melanoma_cell",
              "malignant_cell",
              "tumor_subpopulation",
              "keratinocyte",
              "hepatocyte",
              "lung_adenocarcinoma_cell",
              "colon_adenocarcinoma_cell",
              "Tirosh_cell_cycle_program",
              "Jerby_Arnon_program",
              "glioblastoma_cell",
              "breast_carcinoma_cell"
            ]
          },
          {
            "name": "cf_immune_response_direction_committed",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_immune_robustness_recall",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_ambiguous_hedge",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].causation_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_stress_generic_stress_response_evidence",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_conjugate_confounder_contamination_denied",
            "role": "hard_fail",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected"
            ]
          },
          {
            "name": "cf_ccl5_recall_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CCL5",
            "kind": "avoid",
            "expected": [
              "immune_response_tumor_state"
            ]
          },
          {
            "name": "cf_cxcl9_cxcl10_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CXCL9_CXCL10",
            "kind": "avoid",
            "expected": [
              "stress_hypoxia_tumor_state"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-5-claude-code",
      "modelName": "claude-sonnet-5",
      "harness": "claude-code",
      "task": "10x_5p_melanoma_cluster_vs_cd39_tumor_reactivity_dominance",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 2073.024215,
      "cost": 5.060930499999998,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "population_findings",
            "kind": "list",
            "matchKey": "capture_gate",
            "k": 4,
            "expected": [
              {
                "key": "tumor_cell_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "apc_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "combined_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "unselected_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "naive",
                      "central_memory",
                      "effector_memory"
                    ],
                    "canonical": "naive"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "low"
                    ],
                    "canonical": "low"
                  },
                  "purity_caveat": {
                    "op": "equals",
                    "values": [
                      "pure_T_cell"
                    ],
                    "canonical": "pure_T_cell"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "no_reactivity_evidence"
                    ],
                    "canonical": "no_reactivity_evidence"
                  }
                }
              },
              {
                "key": "marker_sorted_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "equals",
                    "values": [
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "terminal_exhausted_LAG3_high"
                  },
                  "clonal_expansion_level": {
                    "op": "in",
                    "values": [
                      "high",
                      "intermediate"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "marker_on_non_T_lineages",
                      "single_cell_gating_loses_target"
                    ],
                    "canonical": "marker_on_non_T_lineages"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "marker_or_exhaustion_defined_only"
                    ],
                    "canonical": "marker_or_exhaustion_defined_only"
                  }
                }
              }
            ]
          },
          {
            "name": "best_capture_approach_gate",
            "role": "gate",
            "field": "best_capture_approach",
            "kind": "value",
            "expected": "heterotypic_cell_conjugate"
          },
          {
            "name": "best_selection_marker_gate",
            "role": "gate",
            "field": "best_selection_marker",
            "kind": "value",
            "expected": "none"
          },
          {
            "name": "reactivity_criterion_gate",
            "role": "gate",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "clonal_expansion_and_physical_engagement"
          },
          {
            "name": "cross_patient_robustness_gate",
            "role": "gate",
            "field": "cross_patient_robustness",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_across_patients",
              "marginal_in_some_patients"
            ]
          },
          {
            "name": "marker_selection_consequence_gate",
            "role": "gate",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "discards_clonally_expanded_reactive_cells"
          },
          {
            "name": "superiority_claim_basis_gate",
            "role": "gate",
            "field": "superiority_claim_basis",
            "kind": "value",
            "expected": "requires_functional_reactivity_assay"
          },
          {
            "name": "conjugate_enriched_state_gate",
            "role": "gate",
            "field": "conjugate_enriched_state",
            "kind": "value",
            "expected": "progenitor_exhausted_TCF7_pos_stem_like"
          },
          {
            "name": "cf_circular_exhaustion_criterion",
            "role": "hard_fail",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "exhaustion_signature_intensity"
          },
          {
            "name": "cf_marker_sort_prior",
            "role": "hard_fail",
            "field": "best_selection_marker",
            "kind": "value",
            "op": "in",
            "expected": [
              "CD39",
              "PD1",
              "PD1_high",
              "TIM3",
              "PD1_CD39",
              "CD39_CD103"
            ]
          },
          {
            "name": "cf_single_cell_capture_prior",
            "role": "hard_fail",
            "field": "best_capture_approach",
            "kind": "value",
            "op": "in",
            "expected": [
              "dissociated_single_cell",
              "dissociated_bulk_population"
            ]
          },
          {
            "name": "cf_off_data_capture_gate",
            "role": "hard_fail",
            "field": "$.population_findings[*].capture_gate",
            "kind": "avoid",
            "expected": [
              "dissociated_bulk",
              "matched_peripheral_blood",
              "matched_lymph_node",
              "post_hoc_expanded_clonotype_group"
            ]
          },
          {
            "name": "cf_conjugate_mischaracterized",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_marker_singlet_called_engaged",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "contains",
            "matchKey": "capture_gate",
            "entity": "marker_sorted_singlet",
            "valueField": "reactivity_evidence",
            "expected": [
              "engagement_and_clonal_expansion"
            ]
          },
          {
            "name": "cf_no_engaged_reactive_population",
            "role": "hard_fail",
            "field": "$.population_findings[*].reactivity_evidence",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_marker_captures_all_reactive",
            "role": "hard_fail",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "captures_full_reactive_repertoire"
          },
          {
            "name": "cf_superiority_overclaimed_from_phenotype",
            "role": "hard_fail",
            "field": "superiority_claim_basis",
            "kind": "value",
            "op": "in",
            "expected": [
              "established_by_scrna_phenotype",
              "established_by_reactivity_proxy_scores"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-5-claude-code",
      "modelName": "claude-sonnet-5",
      "harness": "claude-code",
      "task": "10x_5p_melanoma_cluster_vs_cd39_tumor_reactivity_dominance",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 1820.944964,
      "cost": 5.554823699999999,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "population_findings",
            "kind": "list",
            "matchKey": "capture_gate",
            "k": 4,
            "expected": [
              {
                "key": "tumor_cell_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "apc_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "combined_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "unselected_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "naive",
                      "central_memory",
                      "effector_memory"
                    ],
                    "canonical": "naive"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "low"
                    ],
                    "canonical": "low"
                  },
                  "purity_caveat": {
                    "op": "equals",
                    "values": [
                      "pure_T_cell"
                    ],
                    "canonical": "pure_T_cell"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "no_reactivity_evidence"
                    ],
                    "canonical": "no_reactivity_evidence"
                  }
                }
              },
              {
                "key": "marker_sorted_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "equals",
                    "values": [
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "terminal_exhausted_LAG3_high"
                  },
                  "clonal_expansion_level": {
                    "op": "in",
                    "values": [
                      "high",
                      "intermediate"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "marker_on_non_T_lineages",
                      "single_cell_gating_loses_target"
                    ],
                    "canonical": "marker_on_non_T_lineages"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "marker_or_exhaustion_defined_only"
                    ],
                    "canonical": "marker_or_exhaustion_defined_only"
                  }
                }
              }
            ]
          },
          {
            "name": "best_capture_approach_gate",
            "role": "gate",
            "field": "best_capture_approach",
            "kind": "value",
            "expected": "heterotypic_cell_conjugate"
          },
          {
            "name": "best_selection_marker_gate",
            "role": "gate",
            "field": "best_selection_marker",
            "kind": "value",
            "expected": "none"
          },
          {
            "name": "reactivity_criterion_gate",
            "role": "gate",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "clonal_expansion_and_physical_engagement"
          },
          {
            "name": "cross_patient_robustness_gate",
            "role": "gate",
            "field": "cross_patient_robustness",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_across_patients",
              "marginal_in_some_patients"
            ]
          },
          {
            "name": "marker_selection_consequence_gate",
            "role": "gate",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "discards_clonally_expanded_reactive_cells"
          },
          {
            "name": "superiority_claim_basis_gate",
            "role": "gate",
            "field": "superiority_claim_basis",
            "kind": "value",
            "expected": "requires_functional_reactivity_assay"
          },
          {
            "name": "conjugate_enriched_state_gate",
            "role": "gate",
            "field": "conjugate_enriched_state",
            "kind": "value",
            "expected": "progenitor_exhausted_TCF7_pos_stem_like"
          },
          {
            "name": "cf_circular_exhaustion_criterion",
            "role": "hard_fail",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "exhaustion_signature_intensity"
          },
          {
            "name": "cf_marker_sort_prior",
            "role": "hard_fail",
            "field": "best_selection_marker",
            "kind": "value",
            "op": "in",
            "expected": [
              "CD39",
              "PD1",
              "PD1_high",
              "TIM3",
              "PD1_CD39",
              "CD39_CD103"
            ]
          },
          {
            "name": "cf_single_cell_capture_prior",
            "role": "hard_fail",
            "field": "best_capture_approach",
            "kind": "value",
            "op": "in",
            "expected": [
              "dissociated_single_cell",
              "dissociated_bulk_population"
            ]
          },
          {
            "name": "cf_off_data_capture_gate",
            "role": "hard_fail",
            "field": "$.population_findings[*].capture_gate",
            "kind": "avoid",
            "expected": [
              "dissociated_bulk",
              "matched_peripheral_blood",
              "matched_lymph_node",
              "post_hoc_expanded_clonotype_group"
            ]
          },
          {
            "name": "cf_conjugate_mischaracterized",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_marker_singlet_called_engaged",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "contains",
            "matchKey": "capture_gate",
            "entity": "marker_sorted_singlet",
            "valueField": "reactivity_evidence",
            "expected": [
              "engagement_and_clonal_expansion"
            ]
          },
          {
            "name": "cf_no_engaged_reactive_population",
            "role": "hard_fail",
            "field": "$.population_findings[*].reactivity_evidence",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_marker_captures_all_reactive",
            "role": "hard_fail",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "captures_full_reactive_repertoire"
          },
          {
            "name": "cf_superiority_overclaimed_from_phenotype",
            "role": "hard_fail",
            "field": "superiority_claim_basis",
            "kind": "value",
            "op": "in",
            "expected": [
              "established_by_scrna_phenotype",
              "established_by_reactivity_proxy_scores"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-5-claude-code",
      "modelName": "claude-sonnet-5",
      "harness": "claude-code",
      "task": "10x_5p_melanoma_cluster_vs_cd39_tumor_reactivity_dominance",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1983.445027,
      "cost": 4.9684664000000005,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "population_findings",
            "kind": "list",
            "matchKey": "capture_gate",
            "k": 4,
            "expected": [
              {
                "key": "tumor_cell_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "apc_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "combined_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "unselected_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "naive",
                      "central_memory",
                      "effector_memory"
                    ],
                    "canonical": "naive"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "low"
                    ],
                    "canonical": "low"
                  },
                  "purity_caveat": {
                    "op": "equals",
                    "values": [
                      "pure_T_cell"
                    ],
                    "canonical": "pure_T_cell"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "no_reactivity_evidence"
                    ],
                    "canonical": "no_reactivity_evidence"
                  }
                }
              },
              {
                "key": "marker_sorted_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "equals",
                    "values": [
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "terminal_exhausted_LAG3_high"
                  },
                  "clonal_expansion_level": {
                    "op": "in",
                    "values": [
                      "high",
                      "intermediate"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "marker_on_non_T_lineages",
                      "single_cell_gating_loses_target"
                    ],
                    "canonical": "marker_on_non_T_lineages"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "marker_or_exhaustion_defined_only"
                    ],
                    "canonical": "marker_or_exhaustion_defined_only"
                  }
                }
              }
            ]
          },
          {
            "name": "best_capture_approach_gate",
            "role": "gate",
            "field": "best_capture_approach",
            "kind": "value",
            "expected": "heterotypic_cell_conjugate"
          },
          {
            "name": "best_selection_marker_gate",
            "role": "gate",
            "field": "best_selection_marker",
            "kind": "value",
            "expected": "none"
          },
          {
            "name": "reactivity_criterion_gate",
            "role": "gate",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "clonal_expansion_and_physical_engagement"
          },
          {
            "name": "cross_patient_robustness_gate",
            "role": "gate",
            "field": "cross_patient_robustness",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_across_patients",
              "marginal_in_some_patients"
            ]
          },
          {
            "name": "marker_selection_consequence_gate",
            "role": "gate",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "discards_clonally_expanded_reactive_cells"
          },
          {
            "name": "superiority_claim_basis_gate",
            "role": "gate",
            "field": "superiority_claim_basis",
            "kind": "value",
            "expected": "requires_functional_reactivity_assay"
          },
          {
            "name": "conjugate_enriched_state_gate",
            "role": "gate",
            "field": "conjugate_enriched_state",
            "kind": "value",
            "expected": "progenitor_exhausted_TCF7_pos_stem_like"
          },
          {
            "name": "cf_circular_exhaustion_criterion",
            "role": "hard_fail",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "exhaustion_signature_intensity"
          },
          {
            "name": "cf_marker_sort_prior",
            "role": "hard_fail",
            "field": "best_selection_marker",
            "kind": "value",
            "op": "in",
            "expected": [
              "CD39",
              "PD1",
              "PD1_high",
              "TIM3",
              "PD1_CD39",
              "CD39_CD103"
            ]
          },
          {
            "name": "cf_single_cell_capture_prior",
            "role": "hard_fail",
            "field": "best_capture_approach",
            "kind": "value",
            "op": "in",
            "expected": [
              "dissociated_single_cell",
              "dissociated_bulk_population"
            ]
          },
          {
            "name": "cf_off_data_capture_gate",
            "role": "hard_fail",
            "field": "$.population_findings[*].capture_gate",
            "kind": "avoid",
            "expected": [
              "dissociated_bulk",
              "matched_peripheral_blood",
              "matched_lymph_node",
              "post_hoc_expanded_clonotype_group"
            ]
          },
          {
            "name": "cf_conjugate_mischaracterized",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_marker_singlet_called_engaged",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "contains",
            "matchKey": "capture_gate",
            "entity": "marker_sorted_singlet",
            "valueField": "reactivity_evidence",
            "expected": [
              "engagement_and_clonal_expansion"
            ]
          },
          {
            "name": "cf_no_engaged_reactive_population",
            "role": "hard_fail",
            "field": "$.population_findings[*].reactivity_evidence",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_marker_captures_all_reactive",
            "role": "hard_fail",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "captures_full_reactive_repertoire"
          },
          {
            "name": "cf_superiority_overclaimed_from_phenotype",
            "role": "hard_fail",
            "field": "superiority_claim_basis",
            "kind": "value",
            "op": "in",
            "expected": [
              "established_by_scrna_phenotype",
              "established_by_reactivity_proxy_scores"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-5-claude-code",
      "modelName": "claude-sonnet-5",
      "harness": "claude-code",
      "task": "10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 7718.58152,
      "cost": 12.763899449999998,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_singlet_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "TOXhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "LAG3high_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "GZMKhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "intermediate_exhausted_gzmk"
                    ],
                    "canonical": "intermediate_exhausted_gzmk"
                  }
                }
              },
              {
                "key": "TCF7_stem_like_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "stem_like_exhausted"
                    ],
                    "canonical": "stem_like_exhausted"
                  }
                }
              },
              {
                "key": "MKI67_pos_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              },
              {
                "key": "MKI67high_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_tumor_cluster_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "Tn",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "Tn_Tmem",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "early_Tem_GZMK",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              },
              {
                "key": "Tem_GZMH",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              }
            ]
          },
          {
            "name": "clonotype_expansion_direction_data_true",
            "role": "gate",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_substantially_more_expanded_than_cluster",
              "singlet_modestly_more_expanded_than_cluster"
            ]
          },
          {
            "name": "biological_inference_data_true",
            "role": "gate",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_antigen_experienced_cluster_bystander"
            ]
          },
          {
            "name": "singlet_top3_contains_naive_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Tn",
              "Tn_Tmem",
              "early_Tem_GZMK",
              "Tem_GZMH"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "TOXhigh_Tex",
              "LAG3high_Tex",
              "GZMKhigh_Tex",
              "TCF7_stem_like_Tex",
              "MKI67_pos_Tex_Tprol",
              "MKI67high_Tex_Tprol"
            ]
          },
          {
            "name": "singlet_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "TOXhigh_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "TCF7_stem_like_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "MKI67_state_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "Tn_or_Tn_Tmem_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_stemlike_precursor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_reported_canonical_expansion_direction",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_substantially_more_expanded_than_singlet",
              "cluster_modestly_more_expanded_than_singlet"
            ]
          },
          {
            "name": "cf_expansion_lazy_defer",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "expansion_pattern_not_detectable"
            ]
          },
          {
            "name": "cf_reported_canonical_expected_biology",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_antigen_experienced_singlet_bystander"
            ]
          },
          {
            "name": "cf_biological_inference_null_artifact_or_lineage",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "differences_reflect_dissociation_or_sorting_artifact",
              "cluster_and_singlet_capture_same_biology",
              "cluster_and_singlet_capture_distinct_cd8_lineages"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "biological_relations",
            "kind": "dict",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_greater_than",
                  "greater_than"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "less_than",
                  "much_less_than"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than",
                  "approximately_equal"
                ]
              },
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal"
                ]
              },
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal",
                  "greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_exhaustion_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_naive_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_proliferation_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_expansion_richness_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_multichain_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_intermediate_gzmk_overgeneralized",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_melanocytic_conjugate_overcall",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "values": [
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_relations_mass_hedge",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-5-claude-code",
      "modelName": "claude-sonnet-5",
      "harness": "claude-code",
      "task": "10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 4360.113724,
      "cost": 6.486939749999999,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_singlet_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "TOXhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "LAG3high_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "GZMKhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "intermediate_exhausted_gzmk"
                    ],
                    "canonical": "intermediate_exhausted_gzmk"
                  }
                }
              },
              {
                "key": "TCF7_stem_like_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "stem_like_exhausted"
                    ],
                    "canonical": "stem_like_exhausted"
                  }
                }
              },
              {
                "key": "MKI67_pos_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              },
              {
                "key": "MKI67high_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_tumor_cluster_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "Tn",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "Tn_Tmem",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "early_Tem_GZMK",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              },
              {
                "key": "Tem_GZMH",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              }
            ]
          },
          {
            "name": "clonotype_expansion_direction_data_true",
            "role": "gate",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_substantially_more_expanded_than_cluster",
              "singlet_modestly_more_expanded_than_cluster"
            ]
          },
          {
            "name": "biological_inference_data_true",
            "role": "gate",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_antigen_experienced_cluster_bystander"
            ]
          },
          {
            "name": "singlet_top3_contains_naive_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Tn",
              "Tn_Tmem",
              "early_Tem_GZMK",
              "Tem_GZMH"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "TOXhigh_Tex",
              "LAG3high_Tex",
              "GZMKhigh_Tex",
              "TCF7_stem_like_Tex",
              "MKI67_pos_Tex_Tprol",
              "MKI67high_Tex_Tprol"
            ]
          },
          {
            "name": "singlet_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "TOXhigh_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "TCF7_stem_like_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "MKI67_state_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "Tn_or_Tn_Tmem_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_stemlike_precursor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_reported_canonical_expansion_direction",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_substantially_more_expanded_than_singlet",
              "cluster_modestly_more_expanded_than_singlet"
            ]
          },
          {
            "name": "cf_expansion_lazy_defer",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "expansion_pattern_not_detectable"
            ]
          },
          {
            "name": "cf_reported_canonical_expected_biology",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_antigen_experienced_singlet_bystander"
            ]
          },
          {
            "name": "cf_biological_inference_null_artifact_or_lineage",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "differences_reflect_dissociation_or_sorting_artifact",
              "cluster_and_singlet_capture_same_biology",
              "cluster_and_singlet_capture_distinct_cd8_lineages"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "biological_relations",
            "kind": "dict",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_greater_than",
                  "greater_than"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "less_than",
                  "much_less_than"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than",
                  "approximately_equal"
                ]
              },
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal"
                ]
              },
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal",
                  "greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_exhaustion_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_naive_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_proliferation_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_expansion_richness_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_multichain_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_intermediate_gzmk_overgeneralized",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_melanocytic_conjugate_overcall",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "values": [
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_relations_mass_hedge",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-5-claude-code",
      "modelName": "claude-sonnet-5",
      "harness": "claude-code",
      "task": "10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 2865.75452,
      "cost": 6.364385549999999,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_singlet_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "TOXhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "LAG3high_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "GZMKhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "intermediate_exhausted_gzmk"
                    ],
                    "canonical": "intermediate_exhausted_gzmk"
                  }
                }
              },
              {
                "key": "TCF7_stem_like_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "stem_like_exhausted"
                    ],
                    "canonical": "stem_like_exhausted"
                  }
                }
              },
              {
                "key": "MKI67_pos_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              },
              {
                "key": "MKI67high_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_tumor_cluster_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "Tn",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "Tn_Tmem",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "early_Tem_GZMK",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              },
              {
                "key": "Tem_GZMH",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              }
            ]
          },
          {
            "name": "clonotype_expansion_direction_data_true",
            "role": "gate",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_substantially_more_expanded_than_cluster",
              "singlet_modestly_more_expanded_than_cluster"
            ]
          },
          {
            "name": "biological_inference_data_true",
            "role": "gate",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_antigen_experienced_cluster_bystander"
            ]
          },
          {
            "name": "singlet_top3_contains_naive_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Tn",
              "Tn_Tmem",
              "early_Tem_GZMK",
              "Tem_GZMH"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "TOXhigh_Tex",
              "LAG3high_Tex",
              "GZMKhigh_Tex",
              "TCF7_stem_like_Tex",
              "MKI67_pos_Tex_Tprol",
              "MKI67high_Tex_Tprol"
            ]
          },
          {
            "name": "singlet_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "TOXhigh_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "TCF7_stem_like_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "MKI67_state_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "Tn_or_Tn_Tmem_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_stemlike_precursor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_reported_canonical_expansion_direction",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_substantially_more_expanded_than_singlet",
              "cluster_modestly_more_expanded_than_singlet"
            ]
          },
          {
            "name": "cf_expansion_lazy_defer",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "expansion_pattern_not_detectable"
            ]
          },
          {
            "name": "cf_reported_canonical_expected_biology",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_antigen_experienced_singlet_bystander"
            ]
          },
          {
            "name": "cf_biological_inference_null_artifact_or_lineage",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "differences_reflect_dissociation_or_sorting_artifact",
              "cluster_and_singlet_capture_same_biology",
              "cluster_and_singlet_capture_distinct_cd8_lineages"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "biological_relations",
            "kind": "dict",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_greater_than",
                  "greater_than"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "less_than",
                  "much_less_than"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than",
                  "approximately_equal"
                ]
              },
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal"
                ]
              },
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal",
                  "greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_exhaustion_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_naive_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_proliferation_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_expansion_richness_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_multichain_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_intermediate_gzmk_overgeneralized",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_melanocytic_conjugate_overcall",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "values": [
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_relations_mass_hedge",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-5-claude-code",
      "modelName": "claude-sonnet-5",
      "harness": "claude-code",
      "task": "10x_5p_melanoma_tcr_matched_apc_vs_tumor_exhaustion",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 1,
      "passed": true,
      "score": null,
      "durationS": 1962.780727,
      "cost": 4.736584149999999,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "state_discriminating_genes",
            "kind": "list",
            "matchKey": "gene",
            "k": 3,
            "expected": [
              {
                "key": "CTLA4",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TOX",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "ENTPD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "HAVCR2",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TIGIT",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "PDCD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "difference_attributable_to",
            "role": "gate",
            "field": "difference_attributable_to",
            "kind": "value",
            "expected": "covaries_with_apc_conjugate_state"
          },
          {
            "name": "causal_status",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "direction_unresolved_selection_confound"
          },
          {
            "name": "functional_interpretation",
            "role": "gate",
            "field": "functional_interpretation",
            "kind": "value",
            "expected": "costimulation_signature_function_unresolved"
          },
          {
            "name": "cf_contamination_gene_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "MLANA",
              "PMEL",
              "DCT",
              "TYR",
              "TYRP1",
              "SOX10",
              "MITF",
              "MAGEA4"
            ]
          },
          {
            "name": "cf_apc_partner_contamination_gene",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "HLA-DRA",
              "CD68",
              "LYZ",
              "CD14",
              "C1QA",
              "ITGAX",
              "CD1C"
            ]
          },
          {
            "name": "cf_ligand_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "CD274",
              "PDCD1LG2",
              "CD80",
              "CD86"
            ]
          },
          {
            "name": "cf_attribution_causal_or_baseline",
            "role": "hard_fail",
            "field": "$.difference_attributable_to",
            "kind": "avoid",
            "expected": [
              "apc_engagement",
              "tumor_cell_engagement",
              "clonotype_intrinsic_baseline"
            ]
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "engagement_induces_exhausted_state",
              "pre_exhausted_cells_select_into_apc_conjugates"
            ]
          },
          {
            "name": "cf_causal_fabrication",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "resolved_by_pseudotemporal_ordering",
              "resolved_by_perturbation_experiment"
            ]
          },
          {
            "name": "cf_functional_overclaim",
            "role": "hard_fail",
            "field": "$.functional_interpretation",
            "kind": "avoid",
            "expected": [
              "costimulation_licensed_reinvigoration_competent",
              "costimulation_licensed_checkpoint_responsive",
              "tumor_driven_terminal_exhaustion"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-5-claude-code",
      "modelName": "claude-sonnet-5",
      "harness": "claude-code",
      "task": "10x_5p_melanoma_tcr_matched_apc_vs_tumor_exhaustion",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 2013.404192,
      "cost": 4.205817050000001,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "state_discriminating_genes",
            "kind": "list",
            "matchKey": "gene",
            "k": 3,
            "expected": [
              {
                "key": "CTLA4",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TOX",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "ENTPD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "HAVCR2",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TIGIT",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "PDCD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "difference_attributable_to",
            "role": "gate",
            "field": "difference_attributable_to",
            "kind": "value",
            "expected": "covaries_with_apc_conjugate_state"
          },
          {
            "name": "causal_status",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "direction_unresolved_selection_confound"
          },
          {
            "name": "functional_interpretation",
            "role": "gate",
            "field": "functional_interpretation",
            "kind": "value",
            "expected": "costimulation_signature_function_unresolved"
          },
          {
            "name": "cf_contamination_gene_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "MLANA",
              "PMEL",
              "DCT",
              "TYR",
              "TYRP1",
              "SOX10",
              "MITF",
              "MAGEA4"
            ]
          },
          {
            "name": "cf_apc_partner_contamination_gene",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "HLA-DRA",
              "CD68",
              "LYZ",
              "CD14",
              "C1QA",
              "ITGAX",
              "CD1C"
            ]
          },
          {
            "name": "cf_ligand_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "CD274",
              "PDCD1LG2",
              "CD80",
              "CD86"
            ]
          },
          {
            "name": "cf_attribution_causal_or_baseline",
            "role": "hard_fail",
            "field": "$.difference_attributable_to",
            "kind": "avoid",
            "expected": [
              "apc_engagement",
              "tumor_cell_engagement",
              "clonotype_intrinsic_baseline"
            ]
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "engagement_induces_exhausted_state",
              "pre_exhausted_cells_select_into_apc_conjugates"
            ]
          },
          {
            "name": "cf_causal_fabrication",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "resolved_by_pseudotemporal_ordering",
              "resolved_by_perturbation_experiment"
            ]
          },
          {
            "name": "cf_functional_overclaim",
            "role": "hard_fail",
            "field": "$.functional_interpretation",
            "kind": "avoid",
            "expected": [
              "costimulation_licensed_reinvigoration_competent",
              "costimulation_licensed_checkpoint_responsive",
              "tumor_driven_terminal_exhaustion"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-5-claude-code",
      "modelName": "claude-sonnet-5",
      "harness": "claude-code",
      "task": "10x_5p_melanoma_tcr_matched_apc_vs_tumor_exhaustion",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1353.35928,
      "cost": 3.1203234500000003,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "state_discriminating_genes",
            "kind": "list",
            "matchKey": "gene",
            "k": 3,
            "expected": [
              {
                "key": "CTLA4",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TOX",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "ENTPD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "HAVCR2",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TIGIT",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "PDCD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "difference_attributable_to",
            "role": "gate",
            "field": "difference_attributable_to",
            "kind": "value",
            "expected": "covaries_with_apc_conjugate_state"
          },
          {
            "name": "causal_status",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "direction_unresolved_selection_confound"
          },
          {
            "name": "functional_interpretation",
            "role": "gate",
            "field": "functional_interpretation",
            "kind": "value",
            "expected": "costimulation_signature_function_unresolved"
          },
          {
            "name": "cf_contamination_gene_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "MLANA",
              "PMEL",
              "DCT",
              "TYR",
              "TYRP1",
              "SOX10",
              "MITF",
              "MAGEA4"
            ]
          },
          {
            "name": "cf_apc_partner_contamination_gene",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "HLA-DRA",
              "CD68",
              "LYZ",
              "CD14",
              "C1QA",
              "ITGAX",
              "CD1C"
            ]
          },
          {
            "name": "cf_ligand_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "CD274",
              "PDCD1LG2",
              "CD80",
              "CD86"
            ]
          },
          {
            "name": "cf_attribution_causal_or_baseline",
            "role": "hard_fail",
            "field": "$.difference_attributable_to",
            "kind": "avoid",
            "expected": [
              "apc_engagement",
              "tumor_cell_engagement",
              "clonotype_intrinsic_baseline"
            ]
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "engagement_induces_exhausted_state",
              "pre_exhausted_cells_select_into_apc_conjugates"
            ]
          },
          {
            "name": "cf_causal_fabrication",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "resolved_by_pseudotemporal_ordering",
              "resolved_by_perturbation_experiment"
            ]
          },
          {
            "name": "cf_functional_overclaim",
            "role": "hard_fail",
            "field": "$.functional_interpretation",
            "kind": "avoid",
            "expected": [
              "costimulation_licensed_reinvigoration_competent",
              "costimulation_licensed_checkpoint_responsive",
              "tumor_driven_terminal_exhaustion"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-5-claude-code",
      "modelName": "claude-sonnet-5",
      "harness": "claude-code",
      "task": "green_2_tme_context_dependence",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 4892.31385,
      "cost": 17.70988725,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "program_map",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "sig_includes_itgae",
            "role": "gate",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Itgae"
            ]
          },
          {
            "name": "declared_a_known_confounder",
            "role": "gate",
            "field": "$.excluded_confounders[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "cf_signature_gut_contaminant",
            "role": "hard_fail",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "generic_signature_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd103_program_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "<"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "per_model_regulatory_state",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "poised_unfired"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "late_activation"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "closing_committed"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_tuples",
            "kind": "list",
            "matchKey": "regulator",
            "k": 1,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "role": {
                    "op": "equals",
                    "values": [
                      "repressor"
                    ]
                  },
                  "gating_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  },
                  "evidence_basis": {
                    "op": "equals",
                    "values": [
                      "target_program_behavior"
                    ]
                  },
                  "expr_vs_activity": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "locus_differential_gate",
            "role": "gate",
            "field": "locus_differential",
            "kind": "value",
            "expected": "uniform_across_models"
          },
          {
            "name": "formation_limited_by_gate",
            "role": "gate",
            "field": "formation_limited_by",
            "kind": "value",
            "expected": "absent_induction"
          },
          {
            "name": "locus_competent_in_naive_gate",
            "role": "gate",
            "field": "locus_competent_in_naive",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "ready_silent_fraction_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 3,
            "expected": [
              {
                "key": "B16_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_KPC_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "activator_chromatin_footprint_gate",
            "role": "gate",
            "field": "activator_chromatin_footprint",
            "kind": "value",
            "expected": "inconsistent_across_formers"
          },
          {
            "name": "exhaustion_comparable_gate",
            "role": "gate",
            "field": "exhaustion_comparable_across_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_nondeterminant_gate",
            "role": "gate",
            "field": "exhaustion_explains_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_separable_program_gate",
            "role": "gate",
            "field": "exhaustion_separable_program",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "exhaustion_counterexample_gate",
            "role": "gate",
            "field": "exhaustion_counterexample_model",
            "kind": "value",
            "expected": "KPC_GP"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "exhaustion_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "cf_inconsistent_present_without_activator",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "program",
            "entity": "present",
            "valueField": "activator_state",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_no_block_but_absent",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "none",
            "valueField": "program",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_late_block_without_poised_locus",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "late_activation",
            "valueField": "locus_state",
            "expected": [
              "closing_committed",
              "active"
            ]
          },
          {
            "name": "cf_inconsistent_early_block_with_brake_released",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "early_release",
            "valueField": "brake_state",
            "expected": [
              "released"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-5-claude-code",
      "modelName": "claude-sonnet-5",
      "harness": "claude-code",
      "task": "green_2_tme_context_dependence",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 2730.209697,
      "cost": 5.8246782,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "program_map",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "sig_includes_itgae",
            "role": "gate",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Itgae"
            ]
          },
          {
            "name": "declared_a_known_confounder",
            "role": "gate",
            "field": "$.excluded_confounders[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "cf_signature_gut_contaminant",
            "role": "hard_fail",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "generic_signature_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd103_program_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "<"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "per_model_regulatory_state",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "poised_unfired"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "late_activation"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "closing_committed"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_tuples",
            "kind": "list",
            "matchKey": "regulator",
            "k": 1,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "role": {
                    "op": "equals",
                    "values": [
                      "repressor"
                    ]
                  },
                  "gating_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  },
                  "evidence_basis": {
                    "op": "equals",
                    "values": [
                      "target_program_behavior"
                    ]
                  },
                  "expr_vs_activity": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "locus_differential_gate",
            "role": "gate",
            "field": "locus_differential",
            "kind": "value",
            "expected": "uniform_across_models"
          },
          {
            "name": "formation_limited_by_gate",
            "role": "gate",
            "field": "formation_limited_by",
            "kind": "value",
            "expected": "absent_induction"
          },
          {
            "name": "locus_competent_in_naive_gate",
            "role": "gate",
            "field": "locus_competent_in_naive",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "ready_silent_fraction_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 3,
            "expected": [
              {
                "key": "B16_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_KPC_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "activator_chromatin_footprint_gate",
            "role": "gate",
            "field": "activator_chromatin_footprint",
            "kind": "value",
            "expected": "inconsistent_across_formers"
          },
          {
            "name": "exhaustion_comparable_gate",
            "role": "gate",
            "field": "exhaustion_comparable_across_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_nondeterminant_gate",
            "role": "gate",
            "field": "exhaustion_explains_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_separable_program_gate",
            "role": "gate",
            "field": "exhaustion_separable_program",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "exhaustion_counterexample_gate",
            "role": "gate",
            "field": "exhaustion_counterexample_model",
            "kind": "value",
            "expected": "KPC_GP"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "exhaustion_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "cf_inconsistent_present_without_activator",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "program",
            "entity": "present",
            "valueField": "activator_state",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_no_block_but_absent",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "none",
            "valueField": "program",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_late_block_without_poised_locus",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "late_activation",
            "valueField": "locus_state",
            "expected": [
              "closing_committed",
              "active"
            ]
          },
          {
            "name": "cf_inconsistent_early_block_with_brake_released",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "early_release",
            "valueField": "brake_state",
            "expected": [
              "released"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-5-claude-code",
      "modelName": "claude-sonnet-5",
      "harness": "claude-code",
      "task": "green_2_tme_context_dependence",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 2659.323288,
      "cost": 8.67982035,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "program_map",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "sig_includes_itgae",
            "role": "gate",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Itgae"
            ]
          },
          {
            "name": "declared_a_known_confounder",
            "role": "gate",
            "field": "$.excluded_confounders[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "cf_signature_gut_contaminant",
            "role": "hard_fail",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "generic_signature_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd103_program_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "<"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "per_model_regulatory_state",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "poised_unfired"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "late_activation"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "closing_committed"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_tuples",
            "kind": "list",
            "matchKey": "regulator",
            "k": 1,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "role": {
                    "op": "equals",
                    "values": [
                      "repressor"
                    ]
                  },
                  "gating_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  },
                  "evidence_basis": {
                    "op": "equals",
                    "values": [
                      "target_program_behavior"
                    ]
                  },
                  "expr_vs_activity": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "locus_differential_gate",
            "role": "gate",
            "field": "locus_differential",
            "kind": "value",
            "expected": "uniform_across_models"
          },
          {
            "name": "formation_limited_by_gate",
            "role": "gate",
            "field": "formation_limited_by",
            "kind": "value",
            "expected": "absent_induction"
          },
          {
            "name": "locus_competent_in_naive_gate",
            "role": "gate",
            "field": "locus_competent_in_naive",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "ready_silent_fraction_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 3,
            "expected": [
              {
                "key": "B16_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_KPC_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "activator_chromatin_footprint_gate",
            "role": "gate",
            "field": "activator_chromatin_footprint",
            "kind": "value",
            "expected": "inconsistent_across_formers"
          },
          {
            "name": "exhaustion_comparable_gate",
            "role": "gate",
            "field": "exhaustion_comparable_across_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_nondeterminant_gate",
            "role": "gate",
            "field": "exhaustion_explains_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_separable_program_gate",
            "role": "gate",
            "field": "exhaustion_separable_program",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "exhaustion_counterexample_gate",
            "role": "gate",
            "field": "exhaustion_counterexample_model",
            "kind": "value",
            "expected": "KPC_GP"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "exhaustion_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "cf_inconsistent_present_without_activator",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "program",
            "entity": "present",
            "valueField": "activator_state",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_no_block_but_absent",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "none",
            "valueField": "program",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_late_block_without_poised_locus",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "late_activation",
            "valueField": "locus_state",
            "expected": [
              "closing_committed",
              "active"
            ]
          },
          {
            "name": "cf_inconsistent_early_block_with_brake_released",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "early_release",
            "valueField": "brake_state",
            "expected": [
              "released"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-5-claude-code",
      "modelName": "claude-sonnet-5",
      "harness": "claude-code",
      "task": "green_klf2_hidden_repressor_of_trm_like_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 2760.616194,
      "cost": 8.744128850000001,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "regulator_gate",
            "role": "gate",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "KLF2",
              "Klf2"
            ]
          },
          {
            "name": "controlling_node_character_gate",
            "role": "gate",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "high_differential_expression"
          },
          {
            "name": "expression_vs_activity_gate",
            "role": "gate",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "expression_exceeds_activity"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "target_program_partition",
            "kind": "list",
            "matchKey": "gene",
            "k": 7,
            "expected": [
              {
                "key": "S1pr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "S1pr4",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Klrg1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Cx3cr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Itgae",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Hic1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "P2rx7",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              }
            ]
          },
          {
            "name": "causal_status_gate",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "association_only_sufficiency_requires_perturbation"
          },
          {
            "name": "residency_control_logic_gate",
            "role": "gate",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_release_necessary_not_sufficient_multi_regulon"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "trm_like_program_by_model",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_absent"
                    ],
                    "canonical": "niche_absent"
                  }
                }
              }
            ]
          },
          {
            "name": "cf_regulator_activator_or_wrong_node",
            "role": "hard_fail",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "RUNX3",
              "Runx3",
              "PRDM1",
              "Prdm1",
              "BLIMP1",
              "Blimp1",
              "BHLHE40",
              "Bhlhe40",
              "ZNF683",
              "Zfp683",
              "HOBIT",
              "Hobit",
              "AHR",
              "Ahr",
              "BATF",
              "Batf",
              "TOX",
              "Tox",
              "NR4A1",
              "Nr4a1",
              "NR4A2",
              "Nr4a2",
              "NR4A3",
              "Nr4a3",
              "TCF7",
              "Tcf7",
              "TCF1",
              "Tcf1",
              "BACH2",
              "Bach2",
              "TBX21",
              "Tbx21",
              "TBET",
              "Tbet",
              "FLI1",
              "Fli1",
              "NR3C1",
              "Nr3c1",
              "EOMES",
              "Eomes",
              "ZEB2",
              "Zeb2",
              "ID2",
              "Id2",
              "ID3",
              "Id3",
              "IRF4",
              "Irf4"
            ]
          },
          {
            "name": "cf_node_character_hidden_driver_recall",
            "role": "hard_fail",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "hidden_driver_activity_not_expression"
          },
          {
            "name": "cf_effect_relation_hidden_pattern",
            "role": "hard_fail",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "activity_exceeds_expression"
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "causal_status",
            "kind": "value",
            "op": "in",
            "expected": [
              "sufficiency_established",
              "necessity_established",
              "causal_established_in_dataset",
              "sufficiency_via_perturbation_in_dataset"
            ]
          },
          {
            "name": "cf_residency_logic_activator_prior",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "residency_activator_induction_primary"
          },
          {
            "name": "cf_residency_logic_single_regulon",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_controls_entire_residency_program"
          },
          {
            "name": "cf_decoy_called_target",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_distractor_model_reported",
            "role": "hard_fail",
            "field": "$.trm_like_program_by_model[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_chromatin_only_decoy",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-5-claude-code",
      "modelName": "claude-sonnet-5",
      "harness": "claude-code",
      "task": "green_klf2_hidden_repressor_of_trm_like_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 2424.270469,
      "cost": 7.355715749999998,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "regulator_gate",
            "role": "gate",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "KLF2",
              "Klf2"
            ]
          },
          {
            "name": "controlling_node_character_gate",
            "role": "gate",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "high_differential_expression"
          },
          {
            "name": "expression_vs_activity_gate",
            "role": "gate",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "expression_exceeds_activity"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "target_program_partition",
            "kind": "list",
            "matchKey": "gene",
            "k": 7,
            "expected": [
              {
                "key": "S1pr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "S1pr4",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Klrg1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Cx3cr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Itgae",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Hic1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "P2rx7",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              }
            ]
          },
          {
            "name": "causal_status_gate",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "association_only_sufficiency_requires_perturbation"
          },
          {
            "name": "residency_control_logic_gate",
            "role": "gate",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_release_necessary_not_sufficient_multi_regulon"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "trm_like_program_by_model",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_absent"
                    ],
                    "canonical": "niche_absent"
                  }
                }
              }
            ]
          },
          {
            "name": "cf_regulator_activator_or_wrong_node",
            "role": "hard_fail",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "RUNX3",
              "Runx3",
              "PRDM1",
              "Prdm1",
              "BLIMP1",
              "Blimp1",
              "BHLHE40",
              "Bhlhe40",
              "ZNF683",
              "Zfp683",
              "HOBIT",
              "Hobit",
              "AHR",
              "Ahr",
              "BATF",
              "Batf",
              "TOX",
              "Tox",
              "NR4A1",
              "Nr4a1",
              "NR4A2",
              "Nr4a2",
              "NR4A3",
              "Nr4a3",
              "TCF7",
              "Tcf7",
              "TCF1",
              "Tcf1",
              "BACH2",
              "Bach2",
              "TBX21",
              "Tbx21",
              "TBET",
              "Tbet",
              "FLI1",
              "Fli1",
              "NR3C1",
              "Nr3c1",
              "EOMES",
              "Eomes",
              "ZEB2",
              "Zeb2",
              "ID2",
              "Id2",
              "ID3",
              "Id3",
              "IRF4",
              "Irf4"
            ]
          },
          {
            "name": "cf_node_character_hidden_driver_recall",
            "role": "hard_fail",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "hidden_driver_activity_not_expression"
          },
          {
            "name": "cf_effect_relation_hidden_pattern",
            "role": "hard_fail",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "activity_exceeds_expression"
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "causal_status",
            "kind": "value",
            "op": "in",
            "expected": [
              "sufficiency_established",
              "necessity_established",
              "causal_established_in_dataset",
              "sufficiency_via_perturbation_in_dataset"
            ]
          },
          {
            "name": "cf_residency_logic_activator_prior",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "residency_activator_induction_primary"
          },
          {
            "name": "cf_residency_logic_single_regulon",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_controls_entire_residency_program"
          },
          {
            "name": "cf_decoy_called_target",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_distractor_model_reported",
            "role": "hard_fail",
            "field": "$.trm_like_program_by_model[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_chromatin_only_decoy",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-5-claude-code",
      "modelName": "claude-sonnet-5",
      "harness": "claude-code",
      "task": "green_klf2_hidden_repressor_of_trm_like_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 2271.789456,
      "cost": 7.549901600000004,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "regulator_gate",
            "role": "gate",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "KLF2",
              "Klf2"
            ]
          },
          {
            "name": "controlling_node_character_gate",
            "role": "gate",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "high_differential_expression"
          },
          {
            "name": "expression_vs_activity_gate",
            "role": "gate",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "expression_exceeds_activity"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "target_program_partition",
            "kind": "list",
            "matchKey": "gene",
            "k": 7,
            "expected": [
              {
                "key": "S1pr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "S1pr4",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Klrg1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Cx3cr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Itgae",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Hic1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "P2rx7",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              }
            ]
          },
          {
            "name": "causal_status_gate",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "association_only_sufficiency_requires_perturbation"
          },
          {
            "name": "residency_control_logic_gate",
            "role": "gate",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_release_necessary_not_sufficient_multi_regulon"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "trm_like_program_by_model",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_absent"
                    ],
                    "canonical": "niche_absent"
                  }
                }
              }
            ]
          },
          {
            "name": "cf_regulator_activator_or_wrong_node",
            "role": "hard_fail",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "RUNX3",
              "Runx3",
              "PRDM1",
              "Prdm1",
              "BLIMP1",
              "Blimp1",
              "BHLHE40",
              "Bhlhe40",
              "ZNF683",
              "Zfp683",
              "HOBIT",
              "Hobit",
              "AHR",
              "Ahr",
              "BATF",
              "Batf",
              "TOX",
              "Tox",
              "NR4A1",
              "Nr4a1",
              "NR4A2",
              "Nr4a2",
              "NR4A3",
              "Nr4a3",
              "TCF7",
              "Tcf7",
              "TCF1",
              "Tcf1",
              "BACH2",
              "Bach2",
              "TBX21",
              "Tbx21",
              "TBET",
              "Tbet",
              "FLI1",
              "Fli1",
              "NR3C1",
              "Nr3c1",
              "EOMES",
              "Eomes",
              "ZEB2",
              "Zeb2",
              "ID2",
              "Id2",
              "ID3",
              "Id3",
              "IRF4",
              "Irf4"
            ]
          },
          {
            "name": "cf_node_character_hidden_driver_recall",
            "role": "hard_fail",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "hidden_driver_activity_not_expression"
          },
          {
            "name": "cf_effect_relation_hidden_pattern",
            "role": "hard_fail",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "activity_exceeds_expression"
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "causal_status",
            "kind": "value",
            "op": "in",
            "expected": [
              "sufficiency_established",
              "necessity_established",
              "causal_established_in_dataset",
              "sufficiency_via_perturbation_in_dataset"
            ]
          },
          {
            "name": "cf_residency_logic_activator_prior",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "residency_activator_induction_primary"
          },
          {
            "name": "cf_residency_logic_single_regulon",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_controls_entire_residency_program"
          },
          {
            "name": "cf_decoy_called_target",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_distractor_model_reported",
            "role": "hard_fail",
            "field": "$.trm_like_program_by_model[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_chromatin_only_decoy",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-5-claude-code",
      "modelName": "claude-sonnet-5",
      "harness": "claude-code",
      "task": "green_shared_vs_distinct_regulators_trm_vs_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 2516.623371,
      "cost": 6.59823425,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_program_map",
            "kind": "list",
            "matchKey": "regulator",
            "k": 3,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "both"
                    ],
                    "canonical": "both"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "represses"
                    ],
                    "canonical": "represses"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Batf",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "trm_like_til_specific"
                    ],
                    "canonical": "trm_like_til_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "chromatin_over_rna"
                    ],
                    "canonical": "chromatin_over_rna"
                  }
                }
              },
              {
                "key": "Ahr",
                "fields": {
                  "program_specificity": {
                    "op": "in",
                    "values": [
                      "trm_specific",
                      "both"
                    ],
                    "canonical": "trm_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "rna_over_chromatin"
                    ],
                    "canonical": "rna_over_chromatin"
                  }
                }
              }
            ]
          },
          {
            "name": "regulator_set_no_extras_gate",
            "role": "gate",
            "field": "$.regulator_program_map[*]",
            "kind": "other",
            "expected": null
          },
          {
            "name": "program_identity_conclusion_gate",
            "role": "gate",
            "field": "program_identity_conclusion",
            "kind": "value",
            "expected": "distinct_state_shared_brake_divergent_activators"
          },
          {
            "name": "residency_install_logic_gate",
            "role": "gate",
            "field": "residency_install_logic",
            "kind": "value",
            "expected": "brake_release_permits_context_matched_activator"
          },
          {
            "name": "activator_context_dependence_gate",
            "role": "gate",
            "field": "activator_context_dependence",
            "kind": "value",
            "expected": "context_matched_crossover"
          },
          {
            "name": "evidence_ceiling_gate",
            "role": "gate",
            "field": "evidence_ceiling",
            "kind": "value",
            "expected": "specificity_dissociations_require_perturbation"
          },
          {
            "name": "cf_install_logic_flat_or_single_step",
            "role": "hard_fail",
            "field": "residency_install_logic",
            "kind": "value",
            "op": "in",
            "expected": [
              "flat_shared_residency_toolkit",
              "activator_induction_sufficient",
              "brake_release_alone_sufficient"
            ]
          },
          {
            "name": "cf_program_identity_lumped_or_wrong",
            "role": "hard_fail",
            "field": "program_identity_conclusion",
            "kind": "value",
            "op": "in",
            "expected": [
              "trm_like_til_is_tumor_trm",
              "trm_like_til_is_exhausted_subset",
              "trm_like_til_is_effector_state",
              "trm_like_til_is_transitional_intermediate",
              "trm_like_til_is_shared_program_distinct_niche"
            ]
          },
          {
            "name": "cf_ahr_assigned_to_til",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Ahr",
            "valueField": "program_specificity",
            "expected": [
              "trm_like_til_specific"
            ]
          },
          {
            "name": "cf_batf_assigned_to_trm_or_shared",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Batf",
            "valueField": "program_specificity",
            "expected": [
              "trm_specific",
              "both"
            ]
          },
          {
            "name": "cf_klf2_called_activator",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Klf2",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          },
          {
            "name": "cf_evidence_ceiling_overclaim",
            "role": "hard_fail",
            "field": "evidence_ceiling",
            "kind": "value",
            "op": "in",
            "expected": [
              "specificity_established_from_regulon_activity",
              "specificity_established_in_dataset",
              "causation_established_by_cross_sectional_concordance",
              "necessity_and_sufficiency_established"
            ]
          },
          {
            "name": "cf_activator_context_shared_or_single",
            "role": "hard_fail",
            "field": "activator_context_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "shared_activators_both_contexts",
              "single_global_activator",
              "redundant_activators"
            ]
          },
          {
            "name": "cf_single_modality_rule",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_hic1_motif_brake_mislabel",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_runx3_anticanonical_as_controller",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Runx3",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-5-claude-code",
      "modelName": "claude-sonnet-5",
      "harness": "claude-code",
      "task": "green_shared_vs_distinct_regulators_trm_vs_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 2485.398636,
      "cost": 5.31751035,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_program_map",
            "kind": "list",
            "matchKey": "regulator",
            "k": 3,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "both"
                    ],
                    "canonical": "both"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "represses"
                    ],
                    "canonical": "represses"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Batf",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "trm_like_til_specific"
                    ],
                    "canonical": "trm_like_til_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "chromatin_over_rna"
                    ],
                    "canonical": "chromatin_over_rna"
                  }
                }
              },
              {
                "key": "Ahr",
                "fields": {
                  "program_specificity": {
                    "op": "in",
                    "values": [
                      "trm_specific",
                      "both"
                    ],
                    "canonical": "trm_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "rna_over_chromatin"
                    ],
                    "canonical": "rna_over_chromatin"
                  }
                }
              }
            ]
          },
          {
            "name": "regulator_set_no_extras_gate",
            "role": "gate",
            "field": "$.regulator_program_map[*]",
            "kind": "other",
            "expected": null
          },
          {
            "name": "program_identity_conclusion_gate",
            "role": "gate",
            "field": "program_identity_conclusion",
            "kind": "value",
            "expected": "distinct_state_shared_brake_divergent_activators"
          },
          {
            "name": "residency_install_logic_gate",
            "role": "gate",
            "field": "residency_install_logic",
            "kind": "value",
            "expected": "brake_release_permits_context_matched_activator"
          },
          {
            "name": "activator_context_dependence_gate",
            "role": "gate",
            "field": "activator_context_dependence",
            "kind": "value",
            "expected": "context_matched_crossover"
          },
          {
            "name": "evidence_ceiling_gate",
            "role": "gate",
            "field": "evidence_ceiling",
            "kind": "value",
            "expected": "specificity_dissociations_require_perturbation"
          },
          {
            "name": "cf_install_logic_flat_or_single_step",
            "role": "hard_fail",
            "field": "residency_install_logic",
            "kind": "value",
            "op": "in",
            "expected": [
              "flat_shared_residency_toolkit",
              "activator_induction_sufficient",
              "brake_release_alone_sufficient"
            ]
          },
          {
            "name": "cf_program_identity_lumped_or_wrong",
            "role": "hard_fail",
            "field": "program_identity_conclusion",
            "kind": "value",
            "op": "in",
            "expected": [
              "trm_like_til_is_tumor_trm",
              "trm_like_til_is_exhausted_subset",
              "trm_like_til_is_effector_state",
              "trm_like_til_is_transitional_intermediate",
              "trm_like_til_is_shared_program_distinct_niche"
            ]
          },
          {
            "name": "cf_ahr_assigned_to_til",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Ahr",
            "valueField": "program_specificity",
            "expected": [
              "trm_like_til_specific"
            ]
          },
          {
            "name": "cf_batf_assigned_to_trm_or_shared",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Batf",
            "valueField": "program_specificity",
            "expected": [
              "trm_specific",
              "both"
            ]
          },
          {
            "name": "cf_klf2_called_activator",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Klf2",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          },
          {
            "name": "cf_evidence_ceiling_overclaim",
            "role": "hard_fail",
            "field": "evidence_ceiling",
            "kind": "value",
            "op": "in",
            "expected": [
              "specificity_established_from_regulon_activity",
              "specificity_established_in_dataset",
              "causation_established_by_cross_sectional_concordance",
              "necessity_and_sufficiency_established"
            ]
          },
          {
            "name": "cf_activator_context_shared_or_single",
            "role": "hard_fail",
            "field": "activator_context_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "shared_activators_both_contexts",
              "single_global_activator",
              "redundant_activators"
            ]
          },
          {
            "name": "cf_single_modality_rule",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_hic1_motif_brake_mislabel",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_runx3_anticanonical_as_controller",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Runx3",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-5-claude-code",
      "modelName": "claude-sonnet-5",
      "harness": "claude-code",
      "task": "green_shared_vs_distinct_regulators_trm_vs_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 4680.211407,
      "cost": 11.760716250000002,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_program_map",
            "kind": "list",
            "matchKey": "regulator",
            "k": 3,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "both"
                    ],
                    "canonical": "both"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "represses"
                    ],
                    "canonical": "represses"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Batf",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "trm_like_til_specific"
                    ],
                    "canonical": "trm_like_til_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "chromatin_over_rna"
                    ],
                    "canonical": "chromatin_over_rna"
                  }
                }
              },
              {
                "key": "Ahr",
                "fields": {
                  "program_specificity": {
                    "op": "in",
                    "values": [
                      "trm_specific",
                      "both"
                    ],
                    "canonical": "trm_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "rna_over_chromatin"
                    ],
                    "canonical": "rna_over_chromatin"
                  }
                }
              }
            ]
          },
          {
            "name": "regulator_set_no_extras_gate",
            "role": "gate",
            "field": "$.regulator_program_map[*]",
            "kind": "other",
            "expected": null
          },
          {
            "name": "program_identity_conclusion_gate",
            "role": "gate",
            "field": "program_identity_conclusion",
            "kind": "value",
            "expected": "distinct_state_shared_brake_divergent_activators"
          },
          {
            "name": "residency_install_logic_gate",
            "role": "gate",
            "field": "residency_install_logic",
            "kind": "value",
            "expected": "brake_release_permits_context_matched_activator"
          },
          {
            "name": "activator_context_dependence_gate",
            "role": "gate",
            "field": "activator_context_dependence",
            "kind": "value",
            "expected": "context_matched_crossover"
          },
          {
            "name": "evidence_ceiling_gate",
            "role": "gate",
            "field": "evidence_ceiling",
            "kind": "value",
            "expected": "specificity_dissociations_require_perturbation"
          },
          {
            "name": "cf_install_logic_flat_or_single_step",
            "role": "hard_fail",
            "field": "residency_install_logic",
            "kind": "value",
            "op": "in",
            "expected": [
              "flat_shared_residency_toolkit",
              "activator_induction_sufficient",
              "brake_release_alone_sufficient"
            ]
          },
          {
            "name": "cf_program_identity_lumped_or_wrong",
            "role": "hard_fail",
            "field": "program_identity_conclusion",
            "kind": "value",
            "op": "in",
            "expected": [
              "trm_like_til_is_tumor_trm",
              "trm_like_til_is_exhausted_subset",
              "trm_like_til_is_effector_state",
              "trm_like_til_is_transitional_intermediate",
              "trm_like_til_is_shared_program_distinct_niche"
            ]
          },
          {
            "name": "cf_ahr_assigned_to_til",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Ahr",
            "valueField": "program_specificity",
            "expected": [
              "trm_like_til_specific"
            ]
          },
          {
            "name": "cf_batf_assigned_to_trm_or_shared",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Batf",
            "valueField": "program_specificity",
            "expected": [
              "trm_specific",
              "both"
            ]
          },
          {
            "name": "cf_klf2_called_activator",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Klf2",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          },
          {
            "name": "cf_evidence_ceiling_overclaim",
            "role": "hard_fail",
            "field": "evidence_ceiling",
            "kind": "value",
            "op": "in",
            "expected": [
              "specificity_established_from_regulon_activity",
              "specificity_established_in_dataset",
              "causation_established_by_cross_sectional_concordance",
              "necessity_and_sufficiency_established"
            ]
          },
          {
            "name": "cf_activator_context_shared_or_single",
            "role": "hard_fail",
            "field": "activator_context_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "shared_activators_both_contexts",
              "single_global_activator",
              "redundant_activators"
            ]
          },
          {
            "name": "cf_single_modality_rule",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_hic1_motif_brake_mislabel",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_runx3_anticanonical_as_controller",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Runx3",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-5-claude-code",
      "modelName": "claude-sonnet-5",
      "harness": "claude-code",
      "task": "gse297023_q2_senescence_not_mechanism",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 1,
      "passed": true,
      "score": null,
      "durationS": 2851.131667,
      "cost": 4.330889699999998,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "candidate_is_senescence",
            "role": "gate",
            "field": "$.candidate_mechanism",
            "kind": "value",
            "op": "in",
            "expected": [
              "cellular_senescence",
              "oncogene_induced_senescence",
              "replicative_senescence"
            ]
          },
          {
            "name": "arrest_core_not_engaged",
            "role": "gate",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_not_engaged"
            ]
          },
          {
            "name": "secretory_arm_is_inflammaging",
            "role": "gate",
            "field": "$.secretory_arm_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "acute_phase_inflammaging_response"
            ]
          },
          {
            "name": "secretory_arm_broadly_elevated",
            "role": "gate",
            "field": "$.secretory_arm_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "broadly_elevated"
            ]
          },
          {
            "name": "carries_aging_program",
            "role": "gate",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "carries_bona_fide_aging_program_distinct_from_nominated"
            ]
          },
          {
            "name": "conclusion_senescence_unlikely",
            "role": "gate",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "unlikely_major_contributor"
            ]
          },
          {
            "name": "cf_arrest_core_engaged_recall",
            "role": "hard_fail",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_engaged"
            ]
          },
          {
            "name": "cf_aging_program_conflation_or_explain_away",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "aging_program_indistinguishable_from_nominated_program",
              "aging_program_is_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_no_aging_program_sensitivity_failure",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_detectable_aging_program"
            ]
          },
          {
            "name": "cf_senescence_driver_conclusion",
            "role": "hard_fail",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "likely_primary_driver",
              "plausible_supported_contributor"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-5-claude-code",
      "modelName": "claude-sonnet-5",
      "harness": "claude-code",
      "task": "gse297023_q2_senescence_not_mechanism",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 2,
      "passed": true,
      "score": null,
      "durationS": 2200.192011,
      "cost": 3.5935056,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "candidate_is_senescence",
            "role": "gate",
            "field": "$.candidate_mechanism",
            "kind": "value",
            "op": "in",
            "expected": [
              "cellular_senescence",
              "oncogene_induced_senescence",
              "replicative_senescence"
            ]
          },
          {
            "name": "arrest_core_not_engaged",
            "role": "gate",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_not_engaged"
            ]
          },
          {
            "name": "secretory_arm_is_inflammaging",
            "role": "gate",
            "field": "$.secretory_arm_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "acute_phase_inflammaging_response"
            ]
          },
          {
            "name": "secretory_arm_broadly_elevated",
            "role": "gate",
            "field": "$.secretory_arm_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "broadly_elevated"
            ]
          },
          {
            "name": "carries_aging_program",
            "role": "gate",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "carries_bona_fide_aging_program_distinct_from_nominated"
            ]
          },
          {
            "name": "conclusion_senescence_unlikely",
            "role": "gate",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "unlikely_major_contributor"
            ]
          },
          {
            "name": "cf_arrest_core_engaged_recall",
            "role": "hard_fail",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_engaged"
            ]
          },
          {
            "name": "cf_aging_program_conflation_or_explain_away",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "aging_program_indistinguishable_from_nominated_program",
              "aging_program_is_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_no_aging_program_sensitivity_failure",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_detectable_aging_program"
            ]
          },
          {
            "name": "cf_senescence_driver_conclusion",
            "role": "hard_fail",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "likely_primary_driver",
              "plausible_supported_contributor"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-5-claude-code",
      "modelName": "claude-sonnet-5",
      "harness": "claude-code",
      "task": "gse297023_q2_senescence_not_mechanism",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 4551.159542,
      "cost": 1.34409405,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "candidate_is_senescence",
            "role": "gate",
            "field": "$.candidate_mechanism",
            "kind": "value",
            "op": "in",
            "expected": [
              "cellular_senescence",
              "oncogene_induced_senescence",
              "replicative_senescence"
            ]
          },
          {
            "name": "arrest_core_not_engaged",
            "role": "gate",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_not_engaged"
            ]
          },
          {
            "name": "secretory_arm_is_inflammaging",
            "role": "gate",
            "field": "$.secretory_arm_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "acute_phase_inflammaging_response"
            ]
          },
          {
            "name": "secretory_arm_broadly_elevated",
            "role": "gate",
            "field": "$.secretory_arm_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "broadly_elevated"
            ]
          },
          {
            "name": "carries_aging_program",
            "role": "gate",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "carries_bona_fide_aging_program_distinct_from_nominated"
            ]
          },
          {
            "name": "conclusion_senescence_unlikely",
            "role": "gate",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "unlikely_major_contributor"
            ]
          },
          {
            "name": "cf_arrest_core_engaged_recall",
            "role": "hard_fail",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_engaged"
            ]
          },
          {
            "name": "cf_aging_program_conflation_or_explain_away",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "aging_program_indistinguishable_from_nominated_program",
              "aging_program_is_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_no_aging_program_sensitivity_failure",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_detectable_aging_program"
            ]
          },
          {
            "name": "cf_senescence_driver_conclusion",
            "role": "hard_fail",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "likely_primary_driver",
              "plausible_supported_contributor"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-5-claude-code",
      "modelName": "claude-sonnet-5",
      "harness": "claude-code",
      "task": "gse297023_q3_age_retained_mapk_reduced",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 1,
      "passed": true,
      "score": null,
      "durationS": 2350.173498,
      "cost": 5.581163750000002,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "age_retained",
            "role": "gate",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_retained"
            ]
          },
          {
            "name": "identity_is_aging",
            "role": "gate",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_aging_signatures"
            ]
          },
          {
            "name": "nature_state_shift",
            "role": "gate",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_state_expression_shift"
            ]
          },
          {
            "name": "pathway_is_mapk",
            "role": "gate",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "mapk"
            ]
          },
          {
            "name": "pathway_reduced_in_aged",
            "role": "gate",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "reduced_in_aged"
            ]
          },
          {
            "name": "cf_mapk_increased_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_in_aged"
            ]
          },
          {
            "name": "cf_age_erased",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_erased"
            ]
          },
          {
            "name": "cf_pervasive_overcall",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_pervasive_artifactual"
            ]
          },
          {
            "name": "cf_explain_away_aging",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_compositional_or_identity",
            "role": "hard_fail",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_proportion_shift",
              "cell_identity_shift",
              "composition_dominant_state_secondary",
              "both_state_and_composition_equally"
            ]
          },
          {
            "name": "cf_biggest_inflammatory_signal_not_mapk",
            "role": "hard_fail",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "jak_stat",
              "nfkb",
              "tnfa"
            ]
          },
          {
            "name": "cf_senescence_misattribution",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_senescence_signature_specifically"
            ]
          },
          {
            "name": "cf_dismiss_as_batch_artifact",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_batch_or_technical_artifact"
            ]
          },
          {
            "name": "cf_mapk_comparable_constitutive_kras_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "comparable_across_ages"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-5-claude-code",
      "modelName": "claude-sonnet-5",
      "harness": "claude-code",
      "task": "gse297023_q3_age_retained_mapk_reduced",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 2788.240739,
      "cost": 4.593467199999998,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "age_retained",
            "role": "gate",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_retained"
            ]
          },
          {
            "name": "identity_is_aging",
            "role": "gate",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_aging_signatures"
            ]
          },
          {
            "name": "nature_state_shift",
            "role": "gate",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_state_expression_shift"
            ]
          },
          {
            "name": "pathway_is_mapk",
            "role": "gate",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "mapk"
            ]
          },
          {
            "name": "pathway_reduced_in_aged",
            "role": "gate",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "reduced_in_aged"
            ]
          },
          {
            "name": "cf_mapk_increased_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_in_aged"
            ]
          },
          {
            "name": "cf_age_erased",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_erased"
            ]
          },
          {
            "name": "cf_pervasive_overcall",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_pervasive_artifactual"
            ]
          },
          {
            "name": "cf_explain_away_aging",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_compositional_or_identity",
            "role": "hard_fail",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_proportion_shift",
              "cell_identity_shift",
              "composition_dominant_state_secondary",
              "both_state_and_composition_equally"
            ]
          },
          {
            "name": "cf_biggest_inflammatory_signal_not_mapk",
            "role": "hard_fail",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "jak_stat",
              "nfkb",
              "tnfa"
            ]
          },
          {
            "name": "cf_senescence_misattribution",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_senescence_signature_specifically"
            ]
          },
          {
            "name": "cf_dismiss_as_batch_artifact",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_batch_or_technical_artifact"
            ]
          },
          {
            "name": "cf_mapk_comparable_constitutive_kras_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "comparable_across_ages"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-5-claude-code",
      "modelName": "claude-sonnet-5",
      "harness": "claude-code",
      "task": "gse297023_q3_age_retained_mapk_reduced",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 2668.985758,
      "cost": 6.2308614,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "age_retained",
            "role": "gate",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_retained"
            ]
          },
          {
            "name": "identity_is_aging",
            "role": "gate",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_aging_signatures"
            ]
          },
          {
            "name": "nature_state_shift",
            "role": "gate",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_state_expression_shift"
            ]
          },
          {
            "name": "pathway_is_mapk",
            "role": "gate",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "mapk"
            ]
          },
          {
            "name": "pathway_reduced_in_aged",
            "role": "gate",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "reduced_in_aged"
            ]
          },
          {
            "name": "cf_mapk_increased_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_in_aged"
            ]
          },
          {
            "name": "cf_age_erased",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_erased"
            ]
          },
          {
            "name": "cf_pervasive_overcall",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_pervasive_artifactual"
            ]
          },
          {
            "name": "cf_explain_away_aging",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_compositional_or_identity",
            "role": "hard_fail",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_proportion_shift",
              "cell_identity_shift",
              "composition_dominant_state_secondary",
              "both_state_and_composition_equally"
            ]
          },
          {
            "name": "cf_biggest_inflammatory_signal_not_mapk",
            "role": "hard_fail",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "jak_stat",
              "nfkb",
              "tnfa"
            ]
          },
          {
            "name": "cf_senescence_misattribution",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_senescence_signature_specifically"
            ]
          },
          {
            "name": "cf_dismiss_as_batch_artifact",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_batch_or_technical_artifact"
            ]
          },
          {
            "name": "cf_mapk_comparable_constitutive_kras_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "comparable_across_ages"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-5-claude-code",
      "modelName": "claude-sonnet-5",
      "harness": "claude-code",
      "task": "gse297023_q5_tme_age_dependent_remodeling",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 6696.515454,
      "cost": 10.348395799999999,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "perturbation_is_pten",
            "role": "gate",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "pten"
            ]
          },
          {
            "name": "tme_non_cell_autonomous",
            "role": "gate",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "substantial_non_cell_autonomous_remodeling"
            ]
          },
          {
            "name": "tme_state_not_composition",
            "role": "gate",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "tme_age_dependent",
            "role": "gate",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_dependent"
            ]
          },
          {
            "name": "tme_response_broader_in_aged",
            "role": "gate",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_aged"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "most_responsive_noncancer_cell_types",
            "kind": "list",
            "matchKey": "cell_type",
            "k": 2,
            "expected": [
              {
                "key": "alveolar_macrophage",
                "fields": {}
              },
              {
                "key": "endothelial",
                "fields": {}
              }
            ]
          },
          {
            "name": "cf_cell_intrinsic_only",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "negligible_cell_intrinsic_only_effect",
              "remodeling_unresolved",
              "modest_remodeling_few_cell_types"
            ]
          },
          {
            "name": "cf_remodeling_is_contamination",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "apparent_remodeling_reflects_cancer_contamination"
            ]
          },
          {
            "name": "cf_perturbation_design_misread",
            "role": "hard_fail",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "multiple_tumor_suppressors_pooled",
              "no_additional_gene_difference_kras_only",
              "rb1_cdkn2a_double",
              "pcna_essential_gene_control",
              "difference_is_oncogene_not_suppressor"
            ]
          },
          {
            "name": "cf_age_independent_tme",
            "role": "hard_fail",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_independent"
            ]
          },
          {
            "name": "cf_compositional_shift",
            "role": "hard_fail",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_frequency_shift",
              "compositional_immune_infiltration",
              "composition_dominant_state_secondary",
              "both_state_and_composition"
            ]
          },
          {
            "name": "cf_epithelial_or_cancer_substitution",
            "role": "hard_fail",
            "field": "$.most_responsive_noncancer_cell_types[*].cell_type",
            "kind": "avoid",
            "expected": [
              "AT2_cell",
              "AT1_cell",
              "cancer_cell",
              "ciliated",
              "secretory",
              "club_cell",
              "basal_cell",
              "goblet_cell"
            ]
          },
          {
            "name": "cf_tme_breadth_broader_in_young",
            "role": "hard_fail",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_young"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-5-claude-code",
      "modelName": "claude-sonnet-5",
      "harness": "claude-code",
      "task": "gse297023_q5_tme_age_dependent_remodeling",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 15783.760886,
      "cost": 1.9409933999999993,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "perturbation_is_pten",
            "role": "gate",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "pten"
            ]
          },
          {
            "name": "tme_non_cell_autonomous",
            "role": "gate",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "substantial_non_cell_autonomous_remodeling"
            ]
          },
          {
            "name": "tme_state_not_composition",
            "role": "gate",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "tme_age_dependent",
            "role": "gate",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_dependent"
            ]
          },
          {
            "name": "tme_response_broader_in_aged",
            "role": "gate",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_aged"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "most_responsive_noncancer_cell_types",
            "kind": "list",
            "matchKey": "cell_type",
            "k": 2,
            "expected": [
              {
                "key": "alveolar_macrophage",
                "fields": {}
              },
              {
                "key": "endothelial",
                "fields": {}
              }
            ]
          },
          {
            "name": "cf_cell_intrinsic_only",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "negligible_cell_intrinsic_only_effect",
              "remodeling_unresolved",
              "modest_remodeling_few_cell_types"
            ]
          },
          {
            "name": "cf_remodeling_is_contamination",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "apparent_remodeling_reflects_cancer_contamination"
            ]
          },
          {
            "name": "cf_perturbation_design_misread",
            "role": "hard_fail",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "multiple_tumor_suppressors_pooled",
              "no_additional_gene_difference_kras_only",
              "rb1_cdkn2a_double",
              "pcna_essential_gene_control",
              "difference_is_oncogene_not_suppressor"
            ]
          },
          {
            "name": "cf_age_independent_tme",
            "role": "hard_fail",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_independent"
            ]
          },
          {
            "name": "cf_compositional_shift",
            "role": "hard_fail",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_frequency_shift",
              "compositional_immune_infiltration",
              "composition_dominant_state_secondary",
              "both_state_and_composition"
            ]
          },
          {
            "name": "cf_epithelial_or_cancer_substitution",
            "role": "hard_fail",
            "field": "$.most_responsive_noncancer_cell_types[*].cell_type",
            "kind": "avoid",
            "expected": [
              "AT2_cell",
              "AT1_cell",
              "cancer_cell",
              "ciliated",
              "secretory",
              "club_cell",
              "basal_cell",
              "goblet_cell"
            ]
          },
          {
            "name": "cf_tme_breadth_broader_in_young",
            "role": "hard_fail",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_young"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-5-claude-code",
      "modelName": "claude-sonnet-5",
      "harness": "claude-code",
      "task": "gse297023_q5_tme_age_dependent_remodeling",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 5535.724794,
      "cost": 3.127501199999999,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "perturbation_is_pten",
            "role": "gate",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "pten"
            ]
          },
          {
            "name": "tme_non_cell_autonomous",
            "role": "gate",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "substantial_non_cell_autonomous_remodeling"
            ]
          },
          {
            "name": "tme_state_not_composition",
            "role": "gate",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "tme_age_dependent",
            "role": "gate",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_dependent"
            ]
          },
          {
            "name": "tme_response_broader_in_aged",
            "role": "gate",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_aged"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "most_responsive_noncancer_cell_types",
            "kind": "list",
            "matchKey": "cell_type",
            "k": 2,
            "expected": [
              {
                "key": "alveolar_macrophage",
                "fields": {}
              },
              {
                "key": "endothelial",
                "fields": {}
              }
            ]
          },
          {
            "name": "cf_cell_intrinsic_only",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "negligible_cell_intrinsic_only_effect",
              "remodeling_unresolved",
              "modest_remodeling_few_cell_types"
            ]
          },
          {
            "name": "cf_remodeling_is_contamination",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "apparent_remodeling_reflects_cancer_contamination"
            ]
          },
          {
            "name": "cf_perturbation_design_misread",
            "role": "hard_fail",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "multiple_tumor_suppressors_pooled",
              "no_additional_gene_difference_kras_only",
              "rb1_cdkn2a_double",
              "pcna_essential_gene_control",
              "difference_is_oncogene_not_suppressor"
            ]
          },
          {
            "name": "cf_age_independent_tme",
            "role": "hard_fail",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_independent"
            ]
          },
          {
            "name": "cf_compositional_shift",
            "role": "hard_fail",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_frequency_shift",
              "compositional_immune_infiltration",
              "composition_dominant_state_secondary",
              "both_state_and_composition"
            ]
          },
          {
            "name": "cf_epithelial_or_cancer_substitution",
            "role": "hard_fail",
            "field": "$.most_responsive_noncancer_cell_types[*].cell_type",
            "kind": "avoid",
            "expected": [
              "AT2_cell",
              "AT1_cell",
              "cancer_cell",
              "ciliated",
              "secretory",
              "club_cell",
              "basal_cell",
              "goblet_cell"
            ]
          },
          {
            "name": "cf_tme_breadth_broader_in_young",
            "role": "hard_fail",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_young"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-5-claude-code",
      "modelName": "claude-sonnet-5",
      "harness": "claude-code",
      "task": "gse297023_q6_age_immune_infiltration",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 1,
      "passed": true,
      "score": null,
      "durationS": 2416.978329,
      "cost": 9.876610650000007,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "composition_stable_with_age",
            "role": "gate",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "frequencies_largely_stable"
            ]
          },
          {
            "name": "age_signal_is_state_not_composition",
            "role": "gate",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "frequencies_uninformative_for_infiltration",
            "role": "gate",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_cannot_establish_infiltration"
            ]
          },
          {
            "name": "infiltration_increases_with_age",
            "role": "gate",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_with_age"
            ]
          },
          {
            "name": "infiltration_strength_calibrated",
            "role": "gate",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "significant_increase_strongest_markers_only",
              "consistent_increase_not_statistically_robust"
            ]
          },
          {
            "name": "genotype_marker_fraction_lower_in_pten",
            "role": "gate",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "lower_in_pten_deficient"
            ]
          },
          {
            "name": "cf_composition_shift_reading",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "major_compositional_shift_with_age",
              "increased_immune_fraction_with_age"
            ]
          },
          {
            "name": "cf_lineage_specific_composition_claim",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "myeloid_expansion_with_age",
              "lymphoid_expansion_with_age",
              "minor_shifts_in_some_cell_types"
            ]
          },
          {
            "name": "cf_immunosenescence_depletion_prior",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "immune_depletion_with_age"
            ]
          },
          {
            "name": "cf_age_modality_compositional",
            "role": "hard_fail",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_type_composition_shift",
              "both_equally",
              "composition_dominant_state_secondary"
            ]
          },
          {
            "name": "cf_overread_frequencies_as_infiltration",
            "role": "hard_fail",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_shows_no_infiltration_change",
              "single_cell_quantifies_infiltration",
              "frequencies_show_increased_infiltration",
              "frequencies_show_decreased_infiltration"
            ]
          },
          {
            "name": "cf_infiltration_decreases_immunosenescence",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "decreased_with_age"
            ]
          },
          {
            "name": "cf_infiltration_unchanged_scrna_overread",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "unchanged_with_age"
            ]
          },
          {
            "name": "cf_infiltration_indeterminate_unused_node",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "direction_indeterminate_from_data"
            ]
          },
          {
            "name": "cf_infiltration_mixed_direction",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "mixed_by_immune_cell_type"
            ]
          },
          {
            "name": "cf_infiltration_genotype_confusion",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_only_in_pten_deficient",
              "decreased_only_in_pten_deficient"
            ]
          },
          {
            "name": "cf_pten_infiltration_paper_text_recall",
            "role": "hard_fail",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "higher_in_pten_deficient",
              "higher_in_pten_deficient_only_in_young",
              "higher_in_pten_deficient_only_in_aged"
            ]
          },
          {
            "name": "cf_pseudoreplication_overclaim",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_increase_across_markers"
            ]
          },
          {
            "name": "cf_infiltration_strength_decrease_direction",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_decrease_across_markers",
              "significant_decrease_strongest_markers_only",
              "consistent_decrease_not_statistically_robust"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-5-claude-code",
      "modelName": "claude-sonnet-5",
      "harness": "claude-code",
      "task": "gse297023_q6_age_immune_infiltration",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 2,
      "passed": true,
      "score": null,
      "durationS": 4662.826265,
      "cost": 9.309254400000004,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "composition_stable_with_age",
            "role": "gate",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "frequencies_largely_stable"
            ]
          },
          {
            "name": "age_signal_is_state_not_composition",
            "role": "gate",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "frequencies_uninformative_for_infiltration",
            "role": "gate",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_cannot_establish_infiltration"
            ]
          },
          {
            "name": "infiltration_increases_with_age",
            "role": "gate",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_with_age"
            ]
          },
          {
            "name": "infiltration_strength_calibrated",
            "role": "gate",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "significant_increase_strongest_markers_only",
              "consistent_increase_not_statistically_robust"
            ]
          },
          {
            "name": "genotype_marker_fraction_lower_in_pten",
            "role": "gate",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "lower_in_pten_deficient"
            ]
          },
          {
            "name": "cf_composition_shift_reading",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "major_compositional_shift_with_age",
              "increased_immune_fraction_with_age"
            ]
          },
          {
            "name": "cf_lineage_specific_composition_claim",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "myeloid_expansion_with_age",
              "lymphoid_expansion_with_age",
              "minor_shifts_in_some_cell_types"
            ]
          },
          {
            "name": "cf_immunosenescence_depletion_prior",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "immune_depletion_with_age"
            ]
          },
          {
            "name": "cf_age_modality_compositional",
            "role": "hard_fail",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_type_composition_shift",
              "both_equally",
              "composition_dominant_state_secondary"
            ]
          },
          {
            "name": "cf_overread_frequencies_as_infiltration",
            "role": "hard_fail",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_shows_no_infiltration_change",
              "single_cell_quantifies_infiltration",
              "frequencies_show_increased_infiltration",
              "frequencies_show_decreased_infiltration"
            ]
          },
          {
            "name": "cf_infiltration_decreases_immunosenescence",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "decreased_with_age"
            ]
          },
          {
            "name": "cf_infiltration_unchanged_scrna_overread",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "unchanged_with_age"
            ]
          },
          {
            "name": "cf_infiltration_indeterminate_unused_node",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "direction_indeterminate_from_data"
            ]
          },
          {
            "name": "cf_infiltration_mixed_direction",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "mixed_by_immune_cell_type"
            ]
          },
          {
            "name": "cf_infiltration_genotype_confusion",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_only_in_pten_deficient",
              "decreased_only_in_pten_deficient"
            ]
          },
          {
            "name": "cf_pten_infiltration_paper_text_recall",
            "role": "hard_fail",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "higher_in_pten_deficient",
              "higher_in_pten_deficient_only_in_young",
              "higher_in_pten_deficient_only_in_aged"
            ]
          },
          {
            "name": "cf_pseudoreplication_overclaim",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_increase_across_markers"
            ]
          },
          {
            "name": "cf_infiltration_strength_decrease_direction",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_decrease_across_markers",
              "significant_decrease_strongest_markers_only",
              "consistent_decrease_not_statistically_robust"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-5-claude-code",
      "modelName": "claude-sonnet-5",
      "harness": "claude-code",
      "task": "gse297023_q6_age_immune_infiltration",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 4326.055243,
      "cost": 6.5455850999999985,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "composition_stable_with_age",
            "role": "gate",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "frequencies_largely_stable"
            ]
          },
          {
            "name": "age_signal_is_state_not_composition",
            "role": "gate",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "frequencies_uninformative_for_infiltration",
            "role": "gate",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_cannot_establish_infiltration"
            ]
          },
          {
            "name": "infiltration_increases_with_age",
            "role": "gate",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_with_age"
            ]
          },
          {
            "name": "infiltration_strength_calibrated",
            "role": "gate",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "significant_increase_strongest_markers_only",
              "consistent_increase_not_statistically_robust"
            ]
          },
          {
            "name": "genotype_marker_fraction_lower_in_pten",
            "role": "gate",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "lower_in_pten_deficient"
            ]
          },
          {
            "name": "cf_composition_shift_reading",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "major_compositional_shift_with_age",
              "increased_immune_fraction_with_age"
            ]
          },
          {
            "name": "cf_lineage_specific_composition_claim",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "myeloid_expansion_with_age",
              "lymphoid_expansion_with_age",
              "minor_shifts_in_some_cell_types"
            ]
          },
          {
            "name": "cf_immunosenescence_depletion_prior",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "immune_depletion_with_age"
            ]
          },
          {
            "name": "cf_age_modality_compositional",
            "role": "hard_fail",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_type_composition_shift",
              "both_equally",
              "composition_dominant_state_secondary"
            ]
          },
          {
            "name": "cf_overread_frequencies_as_infiltration",
            "role": "hard_fail",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_shows_no_infiltration_change",
              "single_cell_quantifies_infiltration",
              "frequencies_show_increased_infiltration",
              "frequencies_show_decreased_infiltration"
            ]
          },
          {
            "name": "cf_infiltration_decreases_immunosenescence",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "decreased_with_age"
            ]
          },
          {
            "name": "cf_infiltration_unchanged_scrna_overread",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "unchanged_with_age"
            ]
          },
          {
            "name": "cf_infiltration_indeterminate_unused_node",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "direction_indeterminate_from_data"
            ]
          },
          {
            "name": "cf_infiltration_mixed_direction",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "mixed_by_immune_cell_type"
            ]
          },
          {
            "name": "cf_infiltration_genotype_confusion",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_only_in_pten_deficient",
              "decreased_only_in_pten_deficient"
            ]
          },
          {
            "name": "cf_pten_infiltration_paper_text_recall",
            "role": "hard_fail",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "higher_in_pten_deficient",
              "higher_in_pten_deficient_only_in_young",
              "higher_in_pten_deficient_only_in_aged"
            ]
          },
          {
            "name": "cf_pseudoreplication_overclaim",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_increase_across_markers"
            ]
          },
          {
            "name": "cf_infiltration_strength_decrease_direction",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_decrease_across_markers",
              "significant_decrease_strongest_markers_only",
              "consistent_decrease_not_statistically_robust"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-luna-codex",
      "modelName": "gpt-5.6-luna",
      "harness": "openai-codex",
      "task": "gse155381_q4_pseudotime_delay",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 1,
      "passed": true,
      "score": null,
      "durationS": 1070.989247,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "pseudotime_direction_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "E17_chronological_match_weighted",
            "role": "additive",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pluripotency_exit_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pseudotime_direction_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_synchronous_with_monkey_EPI",
              "chimera_human_EPI_developmentally_advanced",
              "chimera_human_EPI_delayed_at_pre_implantation_only_resolved_post_implantation"
            ]
          },
          {
            "name": "E17_chronological_match_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "resembles_monkey_E17_Gast_EPI",
              "resembles_chimera_monkey_E17_EPI_specifically",
              "chimera_human_E17_EPI_more_advanced_than_monkey_E17_Gast"
            ]
          },
          {
            "name": "pluripotency_exit_paper_inverted_or_prior_driven",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_advances_pluripotency_exit_prematurely",
              "naive_pluripotency_program_specifically_engaged",
              "pluripotency_core_retained_via_KLF17_naive_marker_specifically"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-luna-codex",
      "modelName": "gpt-5.6-luna",
      "harness": "openai-codex",
      "task": "gse155381_q4_pseudotime_delay",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 882.208784,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "pseudotime_direction_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "E17_chronological_match_weighted",
            "role": "additive",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pluripotency_exit_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pseudotime_direction_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_synchronous_with_monkey_EPI",
              "chimera_human_EPI_developmentally_advanced",
              "chimera_human_EPI_delayed_at_pre_implantation_only_resolved_post_implantation"
            ]
          },
          {
            "name": "E17_chronological_match_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "resembles_monkey_E17_Gast_EPI",
              "resembles_chimera_monkey_E17_EPI_specifically",
              "chimera_human_E17_EPI_more_advanced_than_monkey_E17_Gast"
            ]
          },
          {
            "name": "pluripotency_exit_paper_inverted_or_prior_driven",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_advances_pluripotency_exit_prematurely",
              "naive_pluripotency_program_specifically_engaged",
              "pluripotency_core_retained_via_KLF17_naive_marker_specifically"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-luna-codex",
      "modelName": "gpt-5.6-luna",
      "harness": "openai-codex",
      "task": "gse155381_q4_pseudotime_delay",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 651.603401,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "pseudotime_direction_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "E17_chronological_match_weighted",
            "role": "additive",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pluripotency_exit_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pseudotime_direction_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_synchronous_with_monkey_EPI",
              "chimera_human_EPI_developmentally_advanced",
              "chimera_human_EPI_delayed_at_pre_implantation_only_resolved_post_implantation"
            ]
          },
          {
            "name": "E17_chronological_match_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "resembles_monkey_E17_Gast_EPI",
              "resembles_chimera_monkey_E17_EPI_specifically",
              "chimera_human_E17_EPI_more_advanced_than_monkey_E17_Gast"
            ]
          },
          {
            "name": "pluripotency_exit_paper_inverted_or_prior_driven",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_advances_pluripotency_exit_prematurely",
              "naive_pluripotency_program_specifically_engaged",
              "pluripotency_core_retained_via_KLF17_naive_marker_specifically"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-luna-codex",
      "modelName": "gpt-5.6-luna",
      "harness": "openai-codex",
      "task": "gse155381_q5_chimera_monkey_perturbation",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 463.878338,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "deg_direction_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "comparison_subset_choice_weighted",
            "role": "additive",
            "field": "$.comparison_subset_choice",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "panel_resolved_strong_target_ccn1_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN1",
              "CYR61"
            ]
          },
          {
            "name": "panel_resolved_strong_target_ccn2_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN2",
              "CTGF"
            ]
          },
          {
            "name": "hippo_pathway_weighted",
            "role": "additive",
            "field": "$.hippo_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "tgf_beta_pathway_weighted",
            "role": "additive",
            "field": "$.tgf_beta_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "deg_direction_batch_effect_uncorrected",
            "role": "hard_fail",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_dominated_global",
              "up_dominated_in_chosen_subset"
            ]
          },
          {
            "name": "silent_panel_dropout_paired_with_directional_pathway_claim",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "hippo_pathway_direction": {
                "op": "in",
                "values": [
                  "hippo_pathway_upregulated",
                  "hippo_pathway_unchanged",
                  "wnt_pathway_specifically_perturbed",
                  "stress_response_pathway_dominantly_engaged",
                  "no_pathway_signal_above_confound",
                  "yap_tead_direct_target_panel_downregulated",
                  "yap_tead_panel_dependent_direction",
                  "hippo_pathway_downregulated"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-luna-codex",
      "modelName": "gpt-5.6-luna",
      "harness": "openai-codex",
      "task": "gse155381_q5_chimera_monkey_perturbation",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 349.950453,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "deg_direction_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "comparison_subset_choice_weighted",
            "role": "additive",
            "field": "$.comparison_subset_choice",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "panel_resolved_strong_target_ccn1_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN1",
              "CYR61"
            ]
          },
          {
            "name": "panel_resolved_strong_target_ccn2_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN2",
              "CTGF"
            ]
          },
          {
            "name": "hippo_pathway_weighted",
            "role": "additive",
            "field": "$.hippo_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "tgf_beta_pathway_weighted",
            "role": "additive",
            "field": "$.tgf_beta_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "deg_direction_batch_effect_uncorrected",
            "role": "hard_fail",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_dominated_global",
              "up_dominated_in_chosen_subset"
            ]
          },
          {
            "name": "silent_panel_dropout_paired_with_directional_pathway_claim",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "hippo_pathway_direction": {
                "op": "in",
                "values": [
                  "hippo_pathway_upregulated",
                  "hippo_pathway_unchanged",
                  "wnt_pathway_specifically_perturbed",
                  "stress_response_pathway_dominantly_engaged",
                  "no_pathway_signal_above_confound",
                  "yap_tead_direct_target_panel_downregulated",
                  "yap_tead_panel_dependent_direction",
                  "hippo_pathway_downregulated"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-luna-codex",
      "modelName": "gpt-5.6-luna",
      "harness": "openai-codex",
      "task": "gse155381_q5_chimera_monkey_perturbation",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 557.501282,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "deg_direction_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "comparison_subset_choice_weighted",
            "role": "additive",
            "field": "$.comparison_subset_choice",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "panel_resolved_strong_target_ccn1_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN1",
              "CYR61"
            ]
          },
          {
            "name": "panel_resolved_strong_target_ccn2_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN2",
              "CTGF"
            ]
          },
          {
            "name": "hippo_pathway_weighted",
            "role": "additive",
            "field": "$.hippo_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "tgf_beta_pathway_weighted",
            "role": "additive",
            "field": "$.tgf_beta_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "deg_direction_batch_effect_uncorrected",
            "role": "hard_fail",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_dominated_global",
              "up_dominated_in_chosen_subset"
            ]
          },
          {
            "name": "silent_panel_dropout_paired_with_directional_pathway_claim",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "hippo_pathway_direction": {
                "op": "in",
                "values": [
                  "hippo_pathway_upregulated",
                  "hippo_pathway_unchanged",
                  "wnt_pathway_specifically_perturbed",
                  "stress_response_pathway_dominantly_engaged",
                  "no_pathway_signal_above_confound",
                  "yap_tead_direct_target_panel_downregulated",
                  "yap_tead_panel_dependent_direction",
                  "hippo_pathway_downregulated"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-luna-codex",
      "modelName": "gpt-5.6-luna",
      "harness": "openai-codex",
      "task": "gse155381_q7_xspecies_epi_communication",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 644.166571,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "richness_weighted",
            "role": "additive",
            "field": "$.xspecies_signaling_richness",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_weighted",
            "role": "additive",
            "field": "$.dominant_signaling_axis",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_wnt_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "WNT_FZD_LRP_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_fgf_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "FGF_FGFR_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_notch_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "NOTCH_DLL_JAG_family"
                ]
              }
            }
          },
          {
            "name": "signaling_directionality_weighted",
            "role": "additive",
            "field": "$.signaling_directionality",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "top_signal_interaction_class_weighted",
            "role": "additive",
            "field": "$.top_signal_interaction_class",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-luna-codex",
      "modelName": "gpt-5.6-luna",
      "harness": "openai-codex",
      "task": "gse155381_q7_xspecies_epi_communication",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 522.223949,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "richness_weighted",
            "role": "additive",
            "field": "$.xspecies_signaling_richness",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_weighted",
            "role": "additive",
            "field": "$.dominant_signaling_axis",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_wnt_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "WNT_FZD_LRP_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_fgf_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "FGF_FGFR_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_notch_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "NOTCH_DLL_JAG_family"
                ]
              }
            }
          },
          {
            "name": "signaling_directionality_weighted",
            "role": "additive",
            "field": "$.signaling_directionality",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "top_signal_interaction_class_weighted",
            "role": "additive",
            "field": "$.top_signal_interaction_class",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-luna-codex",
      "modelName": "gpt-5.6-luna",
      "harness": "openai-codex",
      "task": "gse155381_q7_xspecies_epi_communication",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 836.93434,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "richness_weighted",
            "role": "additive",
            "field": "$.xspecies_signaling_richness",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_weighted",
            "role": "additive",
            "field": "$.dominant_signaling_axis",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_wnt_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "WNT_FZD_LRP_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_fgf_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "FGF_FGFR_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_notch_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "NOTCH_DLL_JAG_family"
                ]
              }
            }
          },
          {
            "name": "signaling_directionality_weighted",
            "role": "additive",
            "field": "$.signaling_directionality",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "top_signal_interaction_class_weighted",
            "role": "additive",
            "field": "$.top_signal_interaction_class",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-luna-codex",
      "modelName": "gpt-5.6-luna",
      "harness": "openai-codex",
      "task": "gse155381_q8_bidirectional_convergence",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 755.293842,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "chimera_human_identity_shift_target_weighted",
            "role": "additive",
            "field": "$.chimera_human_identity_shift_target",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_in_vivo_state_assignment_weighted",
            "role": "additive",
            "field": "$.chimera_human_in_vivo_state_assignment",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_shift_magnitude_weighted",
            "role": "additive",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_monkey_identity_shift_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "both_arms_null_contradicted_by_calibration",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "chimera_human_identity_shift_target": {
                "op": "equals",
                "values": [
                  "chimera_human_maintains_origin_identity_no_measurable_shift"
                ]
              },
              "chimera_monkey_identity_shift": {
                "op": "equals",
                "values": [
                  "chimera_monkey_unchanged_relative_to_control"
                ]
              }
            }
          },
          {
            "name": "chimera_monkey_paper_claim_on_confounded_input",
            "role": "hard_fail",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "value",
            "expected": "chimera_monkey_shifts_toward_chimera_human_identity"
          },
          {
            "name": "chimera_human_magnitude_wrong_attribution_signature",
            "role": "hard_fail",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "value",
            "op": "in",
            "expected": [
              "dramatic_complete_replacement_of_origin_identity",
              "magnitude_dominated_by_culture_or_metabolic_axis",
              "magnitude_dominated_by_developmental_stage_axis",
              "magnitude_dominated_by_cell_cycle_axis",
              "magnitude_dominated_by_batch_effect_axis"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-luna-codex",
      "modelName": "gpt-5.6-luna",
      "harness": "openai-codex",
      "task": "gse155381_q8_bidirectional_convergence",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 461.874622,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "chimera_human_identity_shift_target_weighted",
            "role": "additive",
            "field": "$.chimera_human_identity_shift_target",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_in_vivo_state_assignment_weighted",
            "role": "additive",
            "field": "$.chimera_human_in_vivo_state_assignment",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_shift_magnitude_weighted",
            "role": "additive",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_monkey_identity_shift_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "both_arms_null_contradicted_by_calibration",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "chimera_human_identity_shift_target": {
                "op": "equals",
                "values": [
                  "chimera_human_maintains_origin_identity_no_measurable_shift"
                ]
              },
              "chimera_monkey_identity_shift": {
                "op": "equals",
                "values": [
                  "chimera_monkey_unchanged_relative_to_control"
                ]
              }
            }
          },
          {
            "name": "chimera_monkey_paper_claim_on_confounded_input",
            "role": "hard_fail",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "value",
            "expected": "chimera_monkey_shifts_toward_chimera_human_identity"
          },
          {
            "name": "chimera_human_magnitude_wrong_attribution_signature",
            "role": "hard_fail",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "value",
            "op": "in",
            "expected": [
              "dramatic_complete_replacement_of_origin_identity",
              "magnitude_dominated_by_culture_or_metabolic_axis",
              "magnitude_dominated_by_developmental_stage_axis",
              "magnitude_dominated_by_cell_cycle_axis",
              "magnitude_dominated_by_batch_effect_axis"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-luna-codex",
      "modelName": "gpt-5.6-luna",
      "harness": "openai-codex",
      "task": "gse155381_q8_bidirectional_convergence",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 364.353239,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "chimera_human_identity_shift_target_weighted",
            "role": "additive",
            "field": "$.chimera_human_identity_shift_target",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_in_vivo_state_assignment_weighted",
            "role": "additive",
            "field": "$.chimera_human_in_vivo_state_assignment",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_shift_magnitude_weighted",
            "role": "additive",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_monkey_identity_shift_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "both_arms_null_contradicted_by_calibration",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "chimera_human_identity_shift_target": {
                "op": "equals",
                "values": [
                  "chimera_human_maintains_origin_identity_no_measurable_shift"
                ]
              },
              "chimera_monkey_identity_shift": {
                "op": "equals",
                "values": [
                  "chimera_monkey_unchanged_relative_to_control"
                ]
              }
            }
          },
          {
            "name": "chimera_monkey_paper_claim_on_confounded_input",
            "role": "hard_fail",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "value",
            "expected": "chimera_monkey_shifts_toward_chimera_human_identity"
          },
          {
            "name": "chimera_human_magnitude_wrong_attribution_signature",
            "role": "hard_fail",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "value",
            "op": "in",
            "expected": [
              "dramatic_complete_replacement_of_origin_identity",
              "magnitude_dominated_by_culture_or_metabolic_axis",
              "magnitude_dominated_by_developmental_stage_axis",
              "magnitude_dominated_by_cell_cycle_axis",
              "magnitude_dominated_by_batch_effect_axis"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-luna-codex",
      "modelName": "gpt-5.6-luna",
      "harness": "openai-codex",
      "task": "gse171524_ectopic_airway_rare_cell",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 1,
      "passed": true,
      "score": null,
      "durationS": 826.373453,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "focus_population_identity_gate",
            "role": "gate",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "chemosensory_marker_expressing_population_alpha"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "focus_population_marker_calls",
            "kind": "list",
            "matchKey": "marker",
            "k": 3,
            "expected": [
              {
                "key": "POU2F3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "CHAT",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "TRPM5",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "LRMP",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNAT3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "AVIL",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNG13",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              }
            ]
          },
          {
            "name": "focus_population_condition_distribution_gate",
            "role": "gate",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_enriched_with_substantial_control_presence",
              "covid_predominant",
              "covid_only",
              "covid_enriched_trend_not_significant"
            ]
          },
          {
            "name": "focus_population_location_gate",
            "role": "gate",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "ectopic_parenchyma_in_covid",
              "parenchyma_only",
              "bronchiolized_or_peribronchial_parenchyma",
              "airway_and_parenchyma"
            ]
          },
          {
            "name": "focus_population_recovery_mode_gate",
            "role": "gate",
            "field": "$.focus_population_recovery_mode",
            "kind": "value",
            "op": "in",
            "expected": [
              "continuum_not_discrete_cluster"
            ]
          },
          {
            "name": "focus_population_identity_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "ionocyte_like_beta",
              "neuroendocrine_like_gamma",
              "basal_stem_progenitor_like_delta",
              "aberrant_basaloid_transitional_like_epsilon",
              "neuronal_or_neural_gene_minor_population_zeta",
              "deuterosomal_ciliated_progenitor_like_eta",
              "hillock_transitional_like_theta",
              "low_quality_doublet_or_debris",
              "no_rare_cell_recovered",
              "not_assessed"
            ]
          },
          {
            "name": "cf_wrong_lineage_marker_called_positive",
            "role": "hard_fail",
            "field": "$.focus_population_marker_calls[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "focus_population_condition_distribution_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "equally_distributed",
              "control_predominant",
              "not_assessed"
            ]
          },
          {
            "name": "focus_population_location_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "airway_only",
              "not_assessed"
            ]
          },
          {
            "name": "cross_axis_canonical_identity_with_airway_only_location_incoherence",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "focus_population_identity": {
                "op": "in",
                "values": [
                  "chemosensory_marker_expressing_population_alpha"
                ]
              },
              "focus_population_location": {
                "op": "in",
                "values": [
                  "airway_only"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-luna-codex",
      "modelName": "gpt-5.6-luna",
      "harness": "openai-codex",
      "task": "gse171524_ectopic_airway_rare_cell",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 642.69438,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "focus_population_identity_gate",
            "role": "gate",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "chemosensory_marker_expressing_population_alpha"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "focus_population_marker_calls",
            "kind": "list",
            "matchKey": "marker",
            "k": 3,
            "expected": [
              {
                "key": "POU2F3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "CHAT",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "TRPM5",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "LRMP",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNAT3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "AVIL",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNG13",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              }
            ]
          },
          {
            "name": "focus_population_condition_distribution_gate",
            "role": "gate",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_enriched_with_substantial_control_presence",
              "covid_predominant",
              "covid_only",
              "covid_enriched_trend_not_significant"
            ]
          },
          {
            "name": "focus_population_location_gate",
            "role": "gate",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "ectopic_parenchyma_in_covid",
              "parenchyma_only",
              "bronchiolized_or_peribronchial_parenchyma",
              "airway_and_parenchyma"
            ]
          },
          {
            "name": "focus_population_recovery_mode_gate",
            "role": "gate",
            "field": "$.focus_population_recovery_mode",
            "kind": "value",
            "op": "in",
            "expected": [
              "continuum_not_discrete_cluster"
            ]
          },
          {
            "name": "focus_population_identity_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "ionocyte_like_beta",
              "neuroendocrine_like_gamma",
              "basal_stem_progenitor_like_delta",
              "aberrant_basaloid_transitional_like_epsilon",
              "neuronal_or_neural_gene_minor_population_zeta",
              "deuterosomal_ciliated_progenitor_like_eta",
              "hillock_transitional_like_theta",
              "low_quality_doublet_or_debris",
              "no_rare_cell_recovered",
              "not_assessed"
            ]
          },
          {
            "name": "cf_wrong_lineage_marker_called_positive",
            "role": "hard_fail",
            "field": "$.focus_population_marker_calls[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "focus_population_condition_distribution_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "equally_distributed",
              "control_predominant",
              "not_assessed"
            ]
          },
          {
            "name": "focus_population_location_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "airway_only",
              "not_assessed"
            ]
          },
          {
            "name": "cross_axis_canonical_identity_with_airway_only_location_incoherence",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "focus_population_identity": {
                "op": "in",
                "values": [
                  "chemosensory_marker_expressing_population_alpha"
                ]
              },
              "focus_population_location": {
                "op": "in",
                "values": [
                  "airway_only"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-luna-codex",
      "modelName": "gpt-5.6-luna",
      "harness": "openai-codex",
      "task": "gse171524_ectopic_airway_rare_cell",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 763.06341,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "focus_population_identity_gate",
            "role": "gate",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "chemosensory_marker_expressing_population_alpha"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "focus_population_marker_calls",
            "kind": "list",
            "matchKey": "marker",
            "k": 3,
            "expected": [
              {
                "key": "POU2F3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "CHAT",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "TRPM5",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "LRMP",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNAT3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "AVIL",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNG13",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              }
            ]
          },
          {
            "name": "focus_population_condition_distribution_gate",
            "role": "gate",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_enriched_with_substantial_control_presence",
              "covid_predominant",
              "covid_only",
              "covid_enriched_trend_not_significant"
            ]
          },
          {
            "name": "focus_population_location_gate",
            "role": "gate",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "ectopic_parenchyma_in_covid",
              "parenchyma_only",
              "bronchiolized_or_peribronchial_parenchyma",
              "airway_and_parenchyma"
            ]
          },
          {
            "name": "focus_population_recovery_mode_gate",
            "role": "gate",
            "field": "$.focus_population_recovery_mode",
            "kind": "value",
            "op": "in",
            "expected": [
              "continuum_not_discrete_cluster"
            ]
          },
          {
            "name": "focus_population_identity_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "ionocyte_like_beta",
              "neuroendocrine_like_gamma",
              "basal_stem_progenitor_like_delta",
              "aberrant_basaloid_transitional_like_epsilon",
              "neuronal_or_neural_gene_minor_population_zeta",
              "deuterosomal_ciliated_progenitor_like_eta",
              "hillock_transitional_like_theta",
              "low_quality_doublet_or_debris",
              "no_rare_cell_recovered",
              "not_assessed"
            ]
          },
          {
            "name": "cf_wrong_lineage_marker_called_positive",
            "role": "hard_fail",
            "field": "$.focus_population_marker_calls[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "focus_population_condition_distribution_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "equally_distributed",
              "control_predominant",
              "not_assessed"
            ]
          },
          {
            "name": "focus_population_location_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "airway_only",
              "not_assessed"
            ]
          },
          {
            "name": "cross_axis_canonical_identity_with_airway_only_location_incoherence",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "focus_population_identity": {
                "op": "in",
                "values": [
                  "chemosensory_marker_expressing_population_alpha"
                ]
              },
              "focus_population_location": {
                "op": "in",
                "values": [
                  "airway_only"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-luna-codex",
      "modelName": "gpt-5.6-luna",
      "harness": "openai-codex",
      "task": "gse171524_pathological_fibroblast_master_regulator_druggable_targets",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 1,
      "passed": true,
      "score": null,
      "durationS": 852.269131,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "disease_associated_fibroblast_subtype_weighted",
            "role": "additive",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "mr_tf_junb_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_alpha"
            ]
          },
          {
            "name": "mr_tf_jund_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_beta"
            ]
          },
          {
            "name": "drug_mmp14_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_alpha"
            ]
          },
          {
            "name": "drug_stat3_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_beta"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_weighted",
            "role": "additive",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "disease_associated_fibroblast_subtype_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "value",
            "op": "in",
            "expected": [
              "subtype_lambda",
              "subtype_epsilon",
              "subtype_iota",
              "subtype_beta",
              "subtype_eta",
              "subtype_zeta",
              "subtype_not_assessed"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "simple_de_ranking_by_log_fc",
              "marker_based_de_only",
              "not_computed"
            ]
          },
          {
            "name": "top_master_regulator_transcription_factors_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_not_computed"
            ]
          },
          {
            "name": "top_druggable_proteins_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_not_assessed"
            ]
          },
          {
            "name": "cross_axis_circular_marker_in_druggable_proteins",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_gamma",
              "drug_target_delta",
              "drug_target_eta"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-luna-codex",
      "modelName": "gpt-5.6-luna",
      "harness": "openai-codex",
      "task": "gse171524_pathological_fibroblast_master_regulator_druggable_targets",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 2,
      "passed": true,
      "score": null,
      "durationS": 666.700059,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "disease_associated_fibroblast_subtype_weighted",
            "role": "additive",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "mr_tf_junb_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_alpha"
            ]
          },
          {
            "name": "mr_tf_jund_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_beta"
            ]
          },
          {
            "name": "drug_mmp14_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_alpha"
            ]
          },
          {
            "name": "drug_stat3_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_beta"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_weighted",
            "role": "additive",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "disease_associated_fibroblast_subtype_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "value",
            "op": "in",
            "expected": [
              "subtype_lambda",
              "subtype_epsilon",
              "subtype_iota",
              "subtype_beta",
              "subtype_eta",
              "subtype_zeta",
              "subtype_not_assessed"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "simple_de_ranking_by_log_fc",
              "marker_based_de_only",
              "not_computed"
            ]
          },
          {
            "name": "top_master_regulator_transcription_factors_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_not_computed"
            ]
          },
          {
            "name": "top_druggable_proteins_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_not_assessed"
            ]
          },
          {
            "name": "cross_axis_circular_marker_in_druggable_proteins",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_gamma",
              "drug_target_delta",
              "drug_target_eta"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-luna-codex",
      "modelName": "gpt-5.6-luna",
      "harness": "openai-codex",
      "task": "gse171524_pathological_fibroblast_master_regulator_druggable_targets",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 800.489626,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "disease_associated_fibroblast_subtype_weighted",
            "role": "additive",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "mr_tf_junb_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_alpha"
            ]
          },
          {
            "name": "mr_tf_jund_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_beta"
            ]
          },
          {
            "name": "drug_mmp14_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_alpha"
            ]
          },
          {
            "name": "drug_stat3_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_beta"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_weighted",
            "role": "additive",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "disease_associated_fibroblast_subtype_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "value",
            "op": "in",
            "expected": [
              "subtype_lambda",
              "subtype_epsilon",
              "subtype_iota",
              "subtype_beta",
              "subtype_eta",
              "subtype_zeta",
              "subtype_not_assessed"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "simple_de_ranking_by_log_fc",
              "marker_based_de_only",
              "not_computed"
            ]
          },
          {
            "name": "top_master_regulator_transcription_factors_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_not_computed"
            ]
          },
          {
            "name": "top_druggable_proteins_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_not_assessed"
            ]
          },
          {
            "name": "cross_axis_circular_marker_in_druggable_proteins",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_gamma",
              "drug_target_delta",
              "drug_target_eta"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-luna-codex",
      "modelName": "gpt-5.6-luna",
      "harness": "openai-codex",
      "task": "gse171524_plasma_bcr_neutralizing_clonotype",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": null,
      "cost": null,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dominant_pairing_heavy_gate",
            "role": "gate",
            "field": "$.dominant_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV4-34"
            ]
          },
          {
            "name": "dominant_pairing_light_gate",
            "role": "gate",
            "field": "$.dominant_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGLV3-19"
            ]
          },
          {
            "name": "dominant_neutralizer_match_gate",
            "role": "gate",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_known_neutralizer_match"
            ]
          },
          {
            "name": "neutralizer_pairing_heavy_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV1-18"
            ]
          },
          {
            "name": "neutralizer_pairing_light_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGKV3-20"
            ]
          },
          {
            "name": "neutralizer_recurrence_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "observed_in_2_to_5_donors"
            ]
          },
          {
            "name": "neutralizer_rank_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "top_5_pairing",
              "top_6_to_20_pairing"
            ]
          },
          {
            "name": "dominant_clone_isotype_gate",
            "role": "gate",
            "field": "$.dominant_clone_isotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "IgG"
            ]
          },
          {
            "name": "blineage_class_switch_contrast_gate",
            "role": "gate",
            "field": "$.blineage_class_switch_contrast",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_class_switched_control_higher_unswitched",
              "class_switched_in_both_cohorts"
            ]
          },
          {
            "name": "dominant_match_narrative_trap_or_null",
            "role": "hard_fail",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_rbd_neutralizer",
              "matches_known_autoreactive_clone",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_heavy",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_light",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_recurrence_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_rank_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "cross_axis_found_dominant_but_failed_to_scan_for_embedded_neutralizer",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_pairing_neutralizer_match": {
                "op": "in",
                "values": [
                  "no_known_neutralizer_match"
                ]
              },
              "neutralizer_pairing_heavy": {
                "op": "in",
                "values": [
                  "none_detected"
                ]
              }
            }
          },
          {
            "name": "neutralizer_pairing_not_exactly_the_s309_pair",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "op": "not",
            "expected": {
              "op": "and",
              "args": [
                {
                  "op": "field",
                  "name": "neutralizer_pairing_heavy",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGHV1-18"
                    ]
                  }
                },
                {
                  "op": "field",
                  "name": "neutralizer_pairing_light",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGKV3-20"
                    ]
                  }
                }
              ]
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-luna-codex",
      "modelName": "gpt-5.6-luna",
      "harness": "openai-codex",
      "task": "gse171524_plasma_bcr_neutralizing_clonotype",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": null,
      "cost": null,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dominant_pairing_heavy_gate",
            "role": "gate",
            "field": "$.dominant_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV4-34"
            ]
          },
          {
            "name": "dominant_pairing_light_gate",
            "role": "gate",
            "field": "$.dominant_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGLV3-19"
            ]
          },
          {
            "name": "dominant_neutralizer_match_gate",
            "role": "gate",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_known_neutralizer_match"
            ]
          },
          {
            "name": "neutralizer_pairing_heavy_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV1-18"
            ]
          },
          {
            "name": "neutralizer_pairing_light_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGKV3-20"
            ]
          },
          {
            "name": "neutralizer_recurrence_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "observed_in_2_to_5_donors"
            ]
          },
          {
            "name": "neutralizer_rank_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "top_5_pairing",
              "top_6_to_20_pairing"
            ]
          },
          {
            "name": "dominant_clone_isotype_gate",
            "role": "gate",
            "field": "$.dominant_clone_isotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "IgG"
            ]
          },
          {
            "name": "blineage_class_switch_contrast_gate",
            "role": "gate",
            "field": "$.blineage_class_switch_contrast",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_class_switched_control_higher_unswitched",
              "class_switched_in_both_cohorts"
            ]
          },
          {
            "name": "dominant_match_narrative_trap_or_null",
            "role": "hard_fail",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_rbd_neutralizer",
              "matches_known_autoreactive_clone",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_heavy",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_light",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_recurrence_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_rank_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "cross_axis_found_dominant_but_failed_to_scan_for_embedded_neutralizer",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_pairing_neutralizer_match": {
                "op": "in",
                "values": [
                  "no_known_neutralizer_match"
                ]
              },
              "neutralizer_pairing_heavy": {
                "op": "in",
                "values": [
                  "none_detected"
                ]
              }
            }
          },
          {
            "name": "neutralizer_pairing_not_exactly_the_s309_pair",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "op": "not",
            "expected": {
              "op": "and",
              "args": [
                {
                  "op": "field",
                  "name": "neutralizer_pairing_heavy",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGHV1-18"
                    ]
                  }
                },
                {
                  "op": "field",
                  "name": "neutralizer_pairing_light",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGKV3-20"
                    ]
                  }
                }
              ]
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-luna-codex",
      "modelName": "gpt-5.6-luna",
      "harness": "openai-codex",
      "task": "gse171524_plasma_bcr_neutralizing_clonotype",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 339.605227,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dominant_pairing_heavy_gate",
            "role": "gate",
            "field": "$.dominant_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV4-34"
            ]
          },
          {
            "name": "dominant_pairing_light_gate",
            "role": "gate",
            "field": "$.dominant_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGLV3-19"
            ]
          },
          {
            "name": "dominant_neutralizer_match_gate",
            "role": "gate",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_known_neutralizer_match"
            ]
          },
          {
            "name": "neutralizer_pairing_heavy_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV1-18"
            ]
          },
          {
            "name": "neutralizer_pairing_light_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGKV3-20"
            ]
          },
          {
            "name": "neutralizer_recurrence_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "observed_in_2_to_5_donors"
            ]
          },
          {
            "name": "neutralizer_rank_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "top_5_pairing",
              "top_6_to_20_pairing"
            ]
          },
          {
            "name": "dominant_clone_isotype_gate",
            "role": "gate",
            "field": "$.dominant_clone_isotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "IgG"
            ]
          },
          {
            "name": "blineage_class_switch_contrast_gate",
            "role": "gate",
            "field": "$.blineage_class_switch_contrast",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_class_switched_control_higher_unswitched",
              "class_switched_in_both_cohorts"
            ]
          },
          {
            "name": "dominant_match_narrative_trap_or_null",
            "role": "hard_fail",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_rbd_neutralizer",
              "matches_known_autoreactive_clone",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_heavy",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_light",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_recurrence_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_rank_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "cross_axis_found_dominant_but_failed_to_scan_for_embedded_neutralizer",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_pairing_neutralizer_match": {
                "op": "in",
                "values": [
                  "no_known_neutralizer_match"
                ]
              },
              "neutralizer_pairing_heavy": {
                "op": "in",
                "values": [
                  "none_detected"
                ]
              }
            }
          },
          {
            "name": "neutralizer_pairing_not_exactly_the_s309_pair",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "op": "not",
            "expected": {
              "op": "and",
              "args": [
                {
                  "op": "field",
                  "name": "neutralizer_pairing_heavy",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGHV1-18"
                    ]
                  }
                },
                {
                  "op": "field",
                  "name": "neutralizer_pairing_light",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGKV3-20"
                    ]
                  }
                }
              ]
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-luna-codex",
      "modelName": "gpt-5.6-luna",
      "harness": "openai-codex",
      "task": "q6_myeloid_subset_restricted_de",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 1,
      "passed": true,
      "score": null,
      "durationS": 708.445926,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "myeloid_subset_with_strongest_change_gate",
            "role": "gate",
            "field": "$.myeloid_subset_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "subset_gamma"
            ]
          },
          {
            "name": "gene_with_strongest_change_gate",
            "role": "gate",
            "field": "$.gene_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "AXL",
              "PROS1"
            ]
          },
          {
            "name": "direction_of_change_gate",
            "role": "gate",
            "field": "$.direction_of_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "down_in_covid"
            ]
          },
          {
            "name": "biological_function_interpretation_gate",
            "role": "gate",
            "field": "$.biological_function_interpretation",
            "kind": "value",
            "op": "in",
            "expected": [
              "efferocytosis",
              "phagocytosis"
            ]
          },
          {
            "name": "myeloid_compartment_abundance_gate",
            "role": "gate",
            "field": "$.myeloid_compartment_abundance",
            "kind": "value",
            "op": "in",
            "expected": [
              "expands_in_covid"
            ]
          },
          {
            "name": "myeloid_expansion_localization_gate",
            "role": "gate",
            "field": "$.myeloid_expansion_localization",
            "kind": "value",
            "op": "in",
            "expected": [
              "recruited_driven_resident_dysfunction"
            ]
          },
          {
            "name": "dominant_signature_gate",
            "role": "gate",
            "field": "$.dominant_signature",
            "kind": "value",
            "op": "in",
            "expected": [
              "technical_or_agonal_artifact"
            ]
          },
          {
            "name": "dominant_signature_direction_gate",
            "role": "gate",
            "field": "$.dominant_signature_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_in_covid"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-luna-codex",
      "modelName": "gpt-5.6-luna",
      "harness": "openai-codex",
      "task": "q6_myeloid_subset_restricted_de",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 2,
      "passed": true,
      "score": null,
      "durationS": 573.893321,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "myeloid_subset_with_strongest_change_gate",
            "role": "gate",
            "field": "$.myeloid_subset_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "subset_gamma"
            ]
          },
          {
            "name": "gene_with_strongest_change_gate",
            "role": "gate",
            "field": "$.gene_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "AXL",
              "PROS1"
            ]
          },
          {
            "name": "direction_of_change_gate",
            "role": "gate",
            "field": "$.direction_of_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "down_in_covid"
            ]
          },
          {
            "name": "biological_function_interpretation_gate",
            "role": "gate",
            "field": "$.biological_function_interpretation",
            "kind": "value",
            "op": "in",
            "expected": [
              "efferocytosis",
              "phagocytosis"
            ]
          },
          {
            "name": "myeloid_compartment_abundance_gate",
            "role": "gate",
            "field": "$.myeloid_compartment_abundance",
            "kind": "value",
            "op": "in",
            "expected": [
              "expands_in_covid"
            ]
          },
          {
            "name": "myeloid_expansion_localization_gate",
            "role": "gate",
            "field": "$.myeloid_expansion_localization",
            "kind": "value",
            "op": "in",
            "expected": [
              "recruited_driven_resident_dysfunction"
            ]
          },
          {
            "name": "dominant_signature_gate",
            "role": "gate",
            "field": "$.dominant_signature",
            "kind": "value",
            "op": "in",
            "expected": [
              "technical_or_agonal_artifact"
            ]
          },
          {
            "name": "dominant_signature_direction_gate",
            "role": "gate",
            "field": "$.dominant_signature_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_in_covid"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-luna-codex",
      "modelName": "gpt-5.6-luna",
      "harness": "openai-codex",
      "task": "q6_myeloid_subset_restricted_de",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 526.010372,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "myeloid_subset_with_strongest_change_gate",
            "role": "gate",
            "field": "$.myeloid_subset_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "subset_gamma"
            ]
          },
          {
            "name": "gene_with_strongest_change_gate",
            "role": "gate",
            "field": "$.gene_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "AXL",
              "PROS1"
            ]
          },
          {
            "name": "direction_of_change_gate",
            "role": "gate",
            "field": "$.direction_of_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "down_in_covid"
            ]
          },
          {
            "name": "biological_function_interpretation_gate",
            "role": "gate",
            "field": "$.biological_function_interpretation",
            "kind": "value",
            "op": "in",
            "expected": [
              "efferocytosis",
              "phagocytosis"
            ]
          },
          {
            "name": "myeloid_compartment_abundance_gate",
            "role": "gate",
            "field": "$.myeloid_compartment_abundance",
            "kind": "value",
            "op": "in",
            "expected": [
              "expands_in_covid"
            ]
          },
          {
            "name": "myeloid_expansion_localization_gate",
            "role": "gate",
            "field": "$.myeloid_expansion_localization",
            "kind": "value",
            "op": "in",
            "expected": [
              "recruited_driven_resident_dysfunction"
            ]
          },
          {
            "name": "dominant_signature_gate",
            "role": "gate",
            "field": "$.dominant_signature",
            "kind": "value",
            "op": "in",
            "expected": [
              "technical_or_agonal_artifact"
            ]
          },
          {
            "name": "dominant_signature_direction_gate",
            "role": "gate",
            "field": "$.dominant_signature_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_in_covid"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-luna-codex",
      "modelName": "gpt-5.6-luna",
      "harness": "openai-codex",
      "task": "snrnaseq_per_donor_composition_shifts_lethal_covid_lung",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 692.744386,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dict_match",
            "role": "gate",
            "field": "cell_type_direction_map",
            "kind": "dict",
            "expected": {
              "Myeloid": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "Epithelia": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Fibroblasts": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "T_NK": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "B_plasma": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Endothelial": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Mast": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "APC": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Neuronal": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              }
            }
          },
          {
            "name": "over_merge_immune_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Immune",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_leukocyte_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Leukocyte",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_lymphoid_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Lymphoid",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_stromal_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Stromal",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-luna-codex",
      "modelName": "gpt-5.6-luna",
      "harness": "openai-codex",
      "task": "snrnaseq_per_donor_composition_shifts_lethal_covid_lung",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 243.640625,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dict_match",
            "role": "gate",
            "field": "cell_type_direction_map",
            "kind": "dict",
            "expected": {
              "Myeloid": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "Epithelia": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Fibroblasts": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "T_NK": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "B_plasma": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Endothelial": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Mast": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "APC": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Neuronal": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              }
            }
          },
          {
            "name": "over_merge_immune_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Immune",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_leukocyte_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Leukocyte",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_lymphoid_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Lymphoid",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_stromal_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Stromal",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-luna-codex",
      "modelName": "gpt-5.6-luna",
      "harness": "openai-codex",
      "task": "snrnaseq_per_donor_composition_shifts_lethal_covid_lung",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 549.768812,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dict_match",
            "role": "gate",
            "field": "cell_type_direction_map",
            "kind": "dict",
            "expected": {
              "Myeloid": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "Epithelia": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Fibroblasts": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "T_NK": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "B_plasma": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Endothelial": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Mast": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "APC": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Neuronal": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              }
            }
          },
          {
            "name": "over_merge_immune_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Immune",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_leukocyte_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Leukocyte",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_lymphoid_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Lymphoid",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_stromal_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Stromal",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-luna-codex",
      "modelName": "gpt-5.6-luna",
      "harness": "openai-codex",
      "task": "10x_5p_melanoma_cluster_enriched_apc_states_v2",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 960.661542,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "top_3_cluster_enriched_apc_states",
            "kind": "list",
            "matchKey": "apc_state",
            "k": 3,
            "expected": [
              {
                "key": "plasmacytoid_DC",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "T_cell_stimulatory_role_supported",
                      "role_direction_unresolved"
                    ],
                    "canonical": "T_cell_stimulatory_role_supported"
                  }
                }
              },
              {
                "key": "C1qhigh_lipid_associated_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "C1qhigh_inflammatory_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "plasma_cell",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "cohort_significant_per_patient_inconsistent"
                    ],
                    "canonical": "cohort_significant_per_patient_inconsistent"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CD80_CD86": {
                "op": "in",
                "value": [
                  "mreg_DC",
                  "shared_across_APC_states"
                ]
              },
              "PD_L1_in_APC_clusters": {
                "op": "in",
                "value": [
                  "C1qhigh_macrophage_either_subtype",
                  "shared_across_APC_states",
                  "mreg_DC"
                ]
              },
              "type_I_IFN": {
                "op": "in",
                "value": [
                  "plasmacytoid_DC",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "cf_mreg_dc_claimed_enriched",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "mreg_DC",
              "mature_DC_enriched_in_immunoregulatory_molecules"
            ]
          },
          {
            "name": "cf_framework_offcompartment_or_generic_substitution",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "cDC1",
              "Maier_mregDC",
              "Cheng_C1QC_TAM",
              "Cheng_SPP1_TAM",
              "Cheng_FOLR2_TAM",
              "Villani_pDC",
              "Zhang_inflammatory_monocyte",
              "LAMP3_pos_DC",
              "myeloid_derived_suppressor_cell",
              "MDSC",
              "tumor_associated_neutrophil",
              "mast_cell",
              "APC",
              "antigen_presenting_cell",
              "macrophage",
              "dendritic_cell",
              "tumor_associated_macrophage",
              "TAM"
            ]
          },
          {
            "name": "cf_direction_commit_on_unresolvable_slot",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_not_resolvable_hedge",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].functional_attribution_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_type_I_IFN_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.type_I_IFN",
            "kind": "value",
            "op": "in",
            "expected": [
              "mreg_DC",
              "C1qhigh_macrophage_either_subtype"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-luna-codex",
      "modelName": "gpt-5.6-luna",
      "harness": "openai-codex",
      "task": "10x_5p_melanoma_cluster_enriched_apc_states_v2",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 1212.586218,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "top_3_cluster_enriched_apc_states",
            "kind": "list",
            "matchKey": "apc_state",
            "k": 3,
            "expected": [
              {
                "key": "plasmacytoid_DC",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "T_cell_stimulatory_role_supported",
                      "role_direction_unresolved"
                    ],
                    "canonical": "T_cell_stimulatory_role_supported"
                  }
                }
              },
              {
                "key": "C1qhigh_lipid_associated_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "C1qhigh_inflammatory_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "plasma_cell",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "cohort_significant_per_patient_inconsistent"
                    ],
                    "canonical": "cohort_significant_per_patient_inconsistent"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CD80_CD86": {
                "op": "in",
                "value": [
                  "mreg_DC",
                  "shared_across_APC_states"
                ]
              },
              "PD_L1_in_APC_clusters": {
                "op": "in",
                "value": [
                  "C1qhigh_macrophage_either_subtype",
                  "shared_across_APC_states",
                  "mreg_DC"
                ]
              },
              "type_I_IFN": {
                "op": "in",
                "value": [
                  "plasmacytoid_DC",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "cf_mreg_dc_claimed_enriched",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "mreg_DC",
              "mature_DC_enriched_in_immunoregulatory_molecules"
            ]
          },
          {
            "name": "cf_framework_offcompartment_or_generic_substitution",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "cDC1",
              "Maier_mregDC",
              "Cheng_C1QC_TAM",
              "Cheng_SPP1_TAM",
              "Cheng_FOLR2_TAM",
              "Villani_pDC",
              "Zhang_inflammatory_monocyte",
              "LAMP3_pos_DC",
              "myeloid_derived_suppressor_cell",
              "MDSC",
              "tumor_associated_neutrophil",
              "mast_cell",
              "APC",
              "antigen_presenting_cell",
              "macrophage",
              "dendritic_cell",
              "tumor_associated_macrophage",
              "TAM"
            ]
          },
          {
            "name": "cf_direction_commit_on_unresolvable_slot",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_not_resolvable_hedge",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].functional_attribution_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_type_I_IFN_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.type_I_IFN",
            "kind": "value",
            "op": "in",
            "expected": [
              "mreg_DC",
              "C1qhigh_macrophage_either_subtype"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-luna-codex",
      "modelName": "gpt-5.6-luna",
      "harness": "openai-codex",
      "task": "10x_5p_melanoma_cluster_enriched_apc_states_v2",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 729.443249,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "top_3_cluster_enriched_apc_states",
            "kind": "list",
            "matchKey": "apc_state",
            "k": 3,
            "expected": [
              {
                "key": "plasmacytoid_DC",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "T_cell_stimulatory_role_supported",
                      "role_direction_unresolved"
                    ],
                    "canonical": "T_cell_stimulatory_role_supported"
                  }
                }
              },
              {
                "key": "C1qhigh_lipid_associated_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "C1qhigh_inflammatory_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "plasma_cell",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "cohort_significant_per_patient_inconsistent"
                    ],
                    "canonical": "cohort_significant_per_patient_inconsistent"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CD80_CD86": {
                "op": "in",
                "value": [
                  "mreg_DC",
                  "shared_across_APC_states"
                ]
              },
              "PD_L1_in_APC_clusters": {
                "op": "in",
                "value": [
                  "C1qhigh_macrophage_either_subtype",
                  "shared_across_APC_states",
                  "mreg_DC"
                ]
              },
              "type_I_IFN": {
                "op": "in",
                "value": [
                  "plasmacytoid_DC",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "cf_mreg_dc_claimed_enriched",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "mreg_DC",
              "mature_DC_enriched_in_immunoregulatory_molecules"
            ]
          },
          {
            "name": "cf_framework_offcompartment_or_generic_substitution",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "cDC1",
              "Maier_mregDC",
              "Cheng_C1QC_TAM",
              "Cheng_SPP1_TAM",
              "Cheng_FOLR2_TAM",
              "Villani_pDC",
              "Zhang_inflammatory_monocyte",
              "LAMP3_pos_DC",
              "myeloid_derived_suppressor_cell",
              "MDSC",
              "tumor_associated_neutrophil",
              "mast_cell",
              "APC",
              "antigen_presenting_cell",
              "macrophage",
              "dendritic_cell",
              "tumor_associated_macrophage",
              "TAM"
            ]
          },
          {
            "name": "cf_direction_commit_on_unresolvable_slot",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_not_resolvable_hedge",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].functional_attribution_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_type_I_IFN_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.type_I_IFN",
            "kind": "value",
            "op": "in",
            "expected": [
              "mreg_DC",
              "C1qhigh_macrophage_either_subtype"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-luna-codex",
      "modelName": "gpt-5.6-luna",
      "harness": "openai-codex",
      "task": "10x_5p_melanoma_cluster_tumor_state_causation_v5",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 1071.200581,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "enriched_tumor_states",
            "kind": "list",
            "matchKey": "tumor_state",
            "k": 3,
            "expected": [
              {
                "key": "immune_response",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "direction_unresolved_cross_sectional"
                    ],
                    "canonical": "direction_unresolved_cross_sectional"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "IFN_response_axis",
                      "broad_antigen_presentation_MHC"
                    ],
                    "canonical": "IFN_response_axis"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "enrichment_diminished_by_decontamination",
                      "threshold_sensitive"
                    ],
                    "canonical": "enrichment_diminished_by_decontamination"
                  }
                }
              },
              {
                "key": "stress_hypoxia",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "data_supports_cell_intrinsic_cause"
                    ],
                    "canonical": "data_supports_cell_intrinsic_cause"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "HIF_hypoxia_program_specific"
                    ],
                    "canonical": "HIF_hypoxia_program_specific"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "robust_across_methods",
                      "robust_after_decontamination"
                    ],
                    "canonical": "robust_across_methods"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CXCL9_CXCL10": {
                "op": "in",
                "value": [
                  "immune_response_tumor_state"
                ]
              },
              "CCL5": {
                "op": "in",
                "value": [
                  "shared_across_tumor_states",
                  "partner_lymphocyte_derived",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "gate_conjugate_gate_confounder",
            "role": "gate",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "partner_lymphocyte_capture"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "antigen_presentation_decomposition",
            "kind": "dict",
            "expected": {
              "MHC_class_I": {
                "op": "in",
                "value": [
                  "co_regulated_with_dominant_program",
                  "tumor_intrinsic_enriched_independent"
                ]
              },
              "MHC_class_II": {
                "op": "in",
                "value": [
                  "partner_cell_attributable",
                  "flat_not_enriched",
                  "method_sensitive_no_robust_call",
                  "co_regulated_with_dominant_program"
                ]
              }
            }
          },
          {
            "name": "cf_non_robust_state_claimed_enriched",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "transitory_melanocytic",
              "intermediate_melanocytic",
              "melanocytic",
              "MITF_high",
              "neural_crest_like",
              "mitotic",
              "stress_p53",
              "undifferentiated",
              "dedifferentiated",
              "mesenchymal_tumor",
              "AXL_high_tumor",
              "invasive_tumor",
              "senescence_like",
              "EMT_transition",
              "cancer_stem_cell_like",
              "oxidative_stress_tumor"
            ]
          },
          {
            "name": "cf_framework_generic_or_offtissue_substitution",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "Tsoi_undifferentiated",
              "Tsoi_neural_crest_like",
              "Tsoi_transitory",
              "Tsoi_melanocytic",
              "Rambow_NCSC",
              "Wouters_mesenchymal",
              "Pozniak_TCF4_dependent",
              "tumor_cell",
              "melanoma_cell",
              "malignant_cell",
              "tumor_subpopulation",
              "keratinocyte",
              "hepatocyte",
              "lung_adenocarcinoma_cell",
              "colon_adenocarcinoma_cell",
              "Tirosh_cell_cycle_program",
              "Jerby_Arnon_program",
              "glioblastoma_cell",
              "breast_carcinoma_cell"
            ]
          },
          {
            "name": "cf_immune_response_direction_committed",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_immune_robustness_recall",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_ambiguous_hedge",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].causation_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_stress_generic_stress_response_evidence",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_conjugate_confounder_contamination_denied",
            "role": "hard_fail",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected"
            ]
          },
          {
            "name": "cf_ccl5_recall_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CCL5",
            "kind": "avoid",
            "expected": [
              "immune_response_tumor_state"
            ]
          },
          {
            "name": "cf_cxcl9_cxcl10_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CXCL9_CXCL10",
            "kind": "avoid",
            "expected": [
              "stress_hypoxia_tumor_state"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-luna-codex",
      "modelName": "gpt-5.6-luna",
      "harness": "openai-codex",
      "task": "10x_5p_melanoma_cluster_tumor_state_causation_v5",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 731.963471,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "enriched_tumor_states",
            "kind": "list",
            "matchKey": "tumor_state",
            "k": 3,
            "expected": [
              {
                "key": "immune_response",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "direction_unresolved_cross_sectional"
                    ],
                    "canonical": "direction_unresolved_cross_sectional"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "IFN_response_axis",
                      "broad_antigen_presentation_MHC"
                    ],
                    "canonical": "IFN_response_axis"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "enrichment_diminished_by_decontamination",
                      "threshold_sensitive"
                    ],
                    "canonical": "enrichment_diminished_by_decontamination"
                  }
                }
              },
              {
                "key": "stress_hypoxia",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "data_supports_cell_intrinsic_cause"
                    ],
                    "canonical": "data_supports_cell_intrinsic_cause"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "HIF_hypoxia_program_specific"
                    ],
                    "canonical": "HIF_hypoxia_program_specific"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "robust_across_methods",
                      "robust_after_decontamination"
                    ],
                    "canonical": "robust_across_methods"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CXCL9_CXCL10": {
                "op": "in",
                "value": [
                  "immune_response_tumor_state"
                ]
              },
              "CCL5": {
                "op": "in",
                "value": [
                  "shared_across_tumor_states",
                  "partner_lymphocyte_derived",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "gate_conjugate_gate_confounder",
            "role": "gate",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "partner_lymphocyte_capture"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "antigen_presentation_decomposition",
            "kind": "dict",
            "expected": {
              "MHC_class_I": {
                "op": "in",
                "value": [
                  "co_regulated_with_dominant_program",
                  "tumor_intrinsic_enriched_independent"
                ]
              },
              "MHC_class_II": {
                "op": "in",
                "value": [
                  "partner_cell_attributable",
                  "flat_not_enriched",
                  "method_sensitive_no_robust_call",
                  "co_regulated_with_dominant_program"
                ]
              }
            }
          },
          {
            "name": "cf_non_robust_state_claimed_enriched",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "transitory_melanocytic",
              "intermediate_melanocytic",
              "melanocytic",
              "MITF_high",
              "neural_crest_like",
              "mitotic",
              "stress_p53",
              "undifferentiated",
              "dedifferentiated",
              "mesenchymal_tumor",
              "AXL_high_tumor",
              "invasive_tumor",
              "senescence_like",
              "EMT_transition",
              "cancer_stem_cell_like",
              "oxidative_stress_tumor"
            ]
          },
          {
            "name": "cf_framework_generic_or_offtissue_substitution",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "Tsoi_undifferentiated",
              "Tsoi_neural_crest_like",
              "Tsoi_transitory",
              "Tsoi_melanocytic",
              "Rambow_NCSC",
              "Wouters_mesenchymal",
              "Pozniak_TCF4_dependent",
              "tumor_cell",
              "melanoma_cell",
              "malignant_cell",
              "tumor_subpopulation",
              "keratinocyte",
              "hepatocyte",
              "lung_adenocarcinoma_cell",
              "colon_adenocarcinoma_cell",
              "Tirosh_cell_cycle_program",
              "Jerby_Arnon_program",
              "glioblastoma_cell",
              "breast_carcinoma_cell"
            ]
          },
          {
            "name": "cf_immune_response_direction_committed",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_immune_robustness_recall",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_ambiguous_hedge",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].causation_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_stress_generic_stress_response_evidence",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_conjugate_confounder_contamination_denied",
            "role": "hard_fail",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected"
            ]
          },
          {
            "name": "cf_ccl5_recall_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CCL5",
            "kind": "avoid",
            "expected": [
              "immune_response_tumor_state"
            ]
          },
          {
            "name": "cf_cxcl9_cxcl10_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CXCL9_CXCL10",
            "kind": "avoid",
            "expected": [
              "stress_hypoxia_tumor_state"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-luna-codex",
      "modelName": "gpt-5.6-luna",
      "harness": "openai-codex",
      "task": "10x_5p_melanoma_cluster_tumor_state_causation_v5",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 811.448697,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "enriched_tumor_states",
            "kind": "list",
            "matchKey": "tumor_state",
            "k": 3,
            "expected": [
              {
                "key": "immune_response",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "direction_unresolved_cross_sectional"
                    ],
                    "canonical": "direction_unresolved_cross_sectional"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "IFN_response_axis",
                      "broad_antigen_presentation_MHC"
                    ],
                    "canonical": "IFN_response_axis"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "enrichment_diminished_by_decontamination",
                      "threshold_sensitive"
                    ],
                    "canonical": "enrichment_diminished_by_decontamination"
                  }
                }
              },
              {
                "key": "stress_hypoxia",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "data_supports_cell_intrinsic_cause"
                    ],
                    "canonical": "data_supports_cell_intrinsic_cause"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "HIF_hypoxia_program_specific"
                    ],
                    "canonical": "HIF_hypoxia_program_specific"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "robust_across_methods",
                      "robust_after_decontamination"
                    ],
                    "canonical": "robust_across_methods"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CXCL9_CXCL10": {
                "op": "in",
                "value": [
                  "immune_response_tumor_state"
                ]
              },
              "CCL5": {
                "op": "in",
                "value": [
                  "shared_across_tumor_states",
                  "partner_lymphocyte_derived",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "gate_conjugate_gate_confounder",
            "role": "gate",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "partner_lymphocyte_capture"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "antigen_presentation_decomposition",
            "kind": "dict",
            "expected": {
              "MHC_class_I": {
                "op": "in",
                "value": [
                  "co_regulated_with_dominant_program",
                  "tumor_intrinsic_enriched_independent"
                ]
              },
              "MHC_class_II": {
                "op": "in",
                "value": [
                  "partner_cell_attributable",
                  "flat_not_enriched",
                  "method_sensitive_no_robust_call",
                  "co_regulated_with_dominant_program"
                ]
              }
            }
          },
          {
            "name": "cf_non_robust_state_claimed_enriched",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "transitory_melanocytic",
              "intermediate_melanocytic",
              "melanocytic",
              "MITF_high",
              "neural_crest_like",
              "mitotic",
              "stress_p53",
              "undifferentiated",
              "dedifferentiated",
              "mesenchymal_tumor",
              "AXL_high_tumor",
              "invasive_tumor",
              "senescence_like",
              "EMT_transition",
              "cancer_stem_cell_like",
              "oxidative_stress_tumor"
            ]
          },
          {
            "name": "cf_framework_generic_or_offtissue_substitution",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "Tsoi_undifferentiated",
              "Tsoi_neural_crest_like",
              "Tsoi_transitory",
              "Tsoi_melanocytic",
              "Rambow_NCSC",
              "Wouters_mesenchymal",
              "Pozniak_TCF4_dependent",
              "tumor_cell",
              "melanoma_cell",
              "malignant_cell",
              "tumor_subpopulation",
              "keratinocyte",
              "hepatocyte",
              "lung_adenocarcinoma_cell",
              "colon_adenocarcinoma_cell",
              "Tirosh_cell_cycle_program",
              "Jerby_Arnon_program",
              "glioblastoma_cell",
              "breast_carcinoma_cell"
            ]
          },
          {
            "name": "cf_immune_response_direction_committed",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_immune_robustness_recall",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_ambiguous_hedge",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].causation_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_stress_generic_stress_response_evidence",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_conjugate_confounder_contamination_denied",
            "role": "hard_fail",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected"
            ]
          },
          {
            "name": "cf_ccl5_recall_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CCL5",
            "kind": "avoid",
            "expected": [
              "immune_response_tumor_state"
            ]
          },
          {
            "name": "cf_cxcl9_cxcl10_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CXCL9_CXCL10",
            "kind": "avoid",
            "expected": [
              "stress_hypoxia_tumor_state"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-luna-codex",
      "modelName": "gpt-5.6-luna",
      "harness": "openai-codex",
      "task": "10x_5p_melanoma_cluster_vs_cd39_tumor_reactivity_dominance",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 444.050157,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "population_findings",
            "kind": "list",
            "matchKey": "capture_gate",
            "k": 4,
            "expected": [
              {
                "key": "tumor_cell_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "apc_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "combined_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "unselected_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "naive",
                      "central_memory",
                      "effector_memory"
                    ],
                    "canonical": "naive"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "low"
                    ],
                    "canonical": "low"
                  },
                  "purity_caveat": {
                    "op": "equals",
                    "values": [
                      "pure_T_cell"
                    ],
                    "canonical": "pure_T_cell"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "no_reactivity_evidence"
                    ],
                    "canonical": "no_reactivity_evidence"
                  }
                }
              },
              {
                "key": "marker_sorted_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "equals",
                    "values": [
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "terminal_exhausted_LAG3_high"
                  },
                  "clonal_expansion_level": {
                    "op": "in",
                    "values": [
                      "high",
                      "intermediate"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "marker_on_non_T_lineages",
                      "single_cell_gating_loses_target"
                    ],
                    "canonical": "marker_on_non_T_lineages"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "marker_or_exhaustion_defined_only"
                    ],
                    "canonical": "marker_or_exhaustion_defined_only"
                  }
                }
              }
            ]
          },
          {
            "name": "best_capture_approach_gate",
            "role": "gate",
            "field": "best_capture_approach",
            "kind": "value",
            "expected": "heterotypic_cell_conjugate"
          },
          {
            "name": "best_selection_marker_gate",
            "role": "gate",
            "field": "best_selection_marker",
            "kind": "value",
            "expected": "none"
          },
          {
            "name": "reactivity_criterion_gate",
            "role": "gate",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "clonal_expansion_and_physical_engagement"
          },
          {
            "name": "cross_patient_robustness_gate",
            "role": "gate",
            "field": "cross_patient_robustness",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_across_patients",
              "marginal_in_some_patients"
            ]
          },
          {
            "name": "marker_selection_consequence_gate",
            "role": "gate",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "discards_clonally_expanded_reactive_cells"
          },
          {
            "name": "superiority_claim_basis_gate",
            "role": "gate",
            "field": "superiority_claim_basis",
            "kind": "value",
            "expected": "requires_functional_reactivity_assay"
          },
          {
            "name": "conjugate_enriched_state_gate",
            "role": "gate",
            "field": "conjugate_enriched_state",
            "kind": "value",
            "expected": "progenitor_exhausted_TCF7_pos_stem_like"
          },
          {
            "name": "cf_circular_exhaustion_criterion",
            "role": "hard_fail",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "exhaustion_signature_intensity"
          },
          {
            "name": "cf_marker_sort_prior",
            "role": "hard_fail",
            "field": "best_selection_marker",
            "kind": "value",
            "op": "in",
            "expected": [
              "CD39",
              "PD1",
              "PD1_high",
              "TIM3",
              "PD1_CD39",
              "CD39_CD103"
            ]
          },
          {
            "name": "cf_single_cell_capture_prior",
            "role": "hard_fail",
            "field": "best_capture_approach",
            "kind": "value",
            "op": "in",
            "expected": [
              "dissociated_single_cell",
              "dissociated_bulk_population"
            ]
          },
          {
            "name": "cf_off_data_capture_gate",
            "role": "hard_fail",
            "field": "$.population_findings[*].capture_gate",
            "kind": "avoid",
            "expected": [
              "dissociated_bulk",
              "matched_peripheral_blood",
              "matched_lymph_node",
              "post_hoc_expanded_clonotype_group"
            ]
          },
          {
            "name": "cf_conjugate_mischaracterized",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_marker_singlet_called_engaged",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "contains",
            "matchKey": "capture_gate",
            "entity": "marker_sorted_singlet",
            "valueField": "reactivity_evidence",
            "expected": [
              "engagement_and_clonal_expansion"
            ]
          },
          {
            "name": "cf_no_engaged_reactive_population",
            "role": "hard_fail",
            "field": "$.population_findings[*].reactivity_evidence",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_marker_captures_all_reactive",
            "role": "hard_fail",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "captures_full_reactive_repertoire"
          },
          {
            "name": "cf_superiority_overclaimed_from_phenotype",
            "role": "hard_fail",
            "field": "superiority_claim_basis",
            "kind": "value",
            "op": "in",
            "expected": [
              "established_by_scrna_phenotype",
              "established_by_reactivity_proxy_scores"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-luna-codex",
      "modelName": "gpt-5.6-luna",
      "harness": "openai-codex",
      "task": "10x_5p_melanoma_cluster_vs_cd39_tumor_reactivity_dominance",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 866.801684,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "population_findings",
            "kind": "list",
            "matchKey": "capture_gate",
            "k": 4,
            "expected": [
              {
                "key": "tumor_cell_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "apc_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "combined_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "unselected_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "naive",
                      "central_memory",
                      "effector_memory"
                    ],
                    "canonical": "naive"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "low"
                    ],
                    "canonical": "low"
                  },
                  "purity_caveat": {
                    "op": "equals",
                    "values": [
                      "pure_T_cell"
                    ],
                    "canonical": "pure_T_cell"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "no_reactivity_evidence"
                    ],
                    "canonical": "no_reactivity_evidence"
                  }
                }
              },
              {
                "key": "marker_sorted_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "equals",
                    "values": [
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "terminal_exhausted_LAG3_high"
                  },
                  "clonal_expansion_level": {
                    "op": "in",
                    "values": [
                      "high",
                      "intermediate"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "marker_on_non_T_lineages",
                      "single_cell_gating_loses_target"
                    ],
                    "canonical": "marker_on_non_T_lineages"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "marker_or_exhaustion_defined_only"
                    ],
                    "canonical": "marker_or_exhaustion_defined_only"
                  }
                }
              }
            ]
          },
          {
            "name": "best_capture_approach_gate",
            "role": "gate",
            "field": "best_capture_approach",
            "kind": "value",
            "expected": "heterotypic_cell_conjugate"
          },
          {
            "name": "best_selection_marker_gate",
            "role": "gate",
            "field": "best_selection_marker",
            "kind": "value",
            "expected": "none"
          },
          {
            "name": "reactivity_criterion_gate",
            "role": "gate",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "clonal_expansion_and_physical_engagement"
          },
          {
            "name": "cross_patient_robustness_gate",
            "role": "gate",
            "field": "cross_patient_robustness",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_across_patients",
              "marginal_in_some_patients"
            ]
          },
          {
            "name": "marker_selection_consequence_gate",
            "role": "gate",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "discards_clonally_expanded_reactive_cells"
          },
          {
            "name": "superiority_claim_basis_gate",
            "role": "gate",
            "field": "superiority_claim_basis",
            "kind": "value",
            "expected": "requires_functional_reactivity_assay"
          },
          {
            "name": "conjugate_enriched_state_gate",
            "role": "gate",
            "field": "conjugate_enriched_state",
            "kind": "value",
            "expected": "progenitor_exhausted_TCF7_pos_stem_like"
          },
          {
            "name": "cf_circular_exhaustion_criterion",
            "role": "hard_fail",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "exhaustion_signature_intensity"
          },
          {
            "name": "cf_marker_sort_prior",
            "role": "hard_fail",
            "field": "best_selection_marker",
            "kind": "value",
            "op": "in",
            "expected": [
              "CD39",
              "PD1",
              "PD1_high",
              "TIM3",
              "PD1_CD39",
              "CD39_CD103"
            ]
          },
          {
            "name": "cf_single_cell_capture_prior",
            "role": "hard_fail",
            "field": "best_capture_approach",
            "kind": "value",
            "op": "in",
            "expected": [
              "dissociated_single_cell",
              "dissociated_bulk_population"
            ]
          },
          {
            "name": "cf_off_data_capture_gate",
            "role": "hard_fail",
            "field": "$.population_findings[*].capture_gate",
            "kind": "avoid",
            "expected": [
              "dissociated_bulk",
              "matched_peripheral_blood",
              "matched_lymph_node",
              "post_hoc_expanded_clonotype_group"
            ]
          },
          {
            "name": "cf_conjugate_mischaracterized",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_marker_singlet_called_engaged",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "contains",
            "matchKey": "capture_gate",
            "entity": "marker_sorted_singlet",
            "valueField": "reactivity_evidence",
            "expected": [
              "engagement_and_clonal_expansion"
            ]
          },
          {
            "name": "cf_no_engaged_reactive_population",
            "role": "hard_fail",
            "field": "$.population_findings[*].reactivity_evidence",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_marker_captures_all_reactive",
            "role": "hard_fail",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "captures_full_reactive_repertoire"
          },
          {
            "name": "cf_superiority_overclaimed_from_phenotype",
            "role": "hard_fail",
            "field": "superiority_claim_basis",
            "kind": "value",
            "op": "in",
            "expected": [
              "established_by_scrna_phenotype",
              "established_by_reactivity_proxy_scores"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-luna-codex",
      "modelName": "gpt-5.6-luna",
      "harness": "openai-codex",
      "task": "10x_5p_melanoma_cluster_vs_cd39_tumor_reactivity_dominance",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 437.411879,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "population_findings",
            "kind": "list",
            "matchKey": "capture_gate",
            "k": 4,
            "expected": [
              {
                "key": "tumor_cell_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "apc_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "combined_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "unselected_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "naive",
                      "central_memory",
                      "effector_memory"
                    ],
                    "canonical": "naive"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "low"
                    ],
                    "canonical": "low"
                  },
                  "purity_caveat": {
                    "op": "equals",
                    "values": [
                      "pure_T_cell"
                    ],
                    "canonical": "pure_T_cell"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "no_reactivity_evidence"
                    ],
                    "canonical": "no_reactivity_evidence"
                  }
                }
              },
              {
                "key": "marker_sorted_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "equals",
                    "values": [
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "terminal_exhausted_LAG3_high"
                  },
                  "clonal_expansion_level": {
                    "op": "in",
                    "values": [
                      "high",
                      "intermediate"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "marker_on_non_T_lineages",
                      "single_cell_gating_loses_target"
                    ],
                    "canonical": "marker_on_non_T_lineages"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "marker_or_exhaustion_defined_only"
                    ],
                    "canonical": "marker_or_exhaustion_defined_only"
                  }
                }
              }
            ]
          },
          {
            "name": "best_capture_approach_gate",
            "role": "gate",
            "field": "best_capture_approach",
            "kind": "value",
            "expected": "heterotypic_cell_conjugate"
          },
          {
            "name": "best_selection_marker_gate",
            "role": "gate",
            "field": "best_selection_marker",
            "kind": "value",
            "expected": "none"
          },
          {
            "name": "reactivity_criterion_gate",
            "role": "gate",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "clonal_expansion_and_physical_engagement"
          },
          {
            "name": "cross_patient_robustness_gate",
            "role": "gate",
            "field": "cross_patient_robustness",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_across_patients",
              "marginal_in_some_patients"
            ]
          },
          {
            "name": "marker_selection_consequence_gate",
            "role": "gate",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "discards_clonally_expanded_reactive_cells"
          },
          {
            "name": "superiority_claim_basis_gate",
            "role": "gate",
            "field": "superiority_claim_basis",
            "kind": "value",
            "expected": "requires_functional_reactivity_assay"
          },
          {
            "name": "conjugate_enriched_state_gate",
            "role": "gate",
            "field": "conjugate_enriched_state",
            "kind": "value",
            "expected": "progenitor_exhausted_TCF7_pos_stem_like"
          },
          {
            "name": "cf_circular_exhaustion_criterion",
            "role": "hard_fail",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "exhaustion_signature_intensity"
          },
          {
            "name": "cf_marker_sort_prior",
            "role": "hard_fail",
            "field": "best_selection_marker",
            "kind": "value",
            "op": "in",
            "expected": [
              "CD39",
              "PD1",
              "PD1_high",
              "TIM3",
              "PD1_CD39",
              "CD39_CD103"
            ]
          },
          {
            "name": "cf_single_cell_capture_prior",
            "role": "hard_fail",
            "field": "best_capture_approach",
            "kind": "value",
            "op": "in",
            "expected": [
              "dissociated_single_cell",
              "dissociated_bulk_population"
            ]
          },
          {
            "name": "cf_off_data_capture_gate",
            "role": "hard_fail",
            "field": "$.population_findings[*].capture_gate",
            "kind": "avoid",
            "expected": [
              "dissociated_bulk",
              "matched_peripheral_blood",
              "matched_lymph_node",
              "post_hoc_expanded_clonotype_group"
            ]
          },
          {
            "name": "cf_conjugate_mischaracterized",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_marker_singlet_called_engaged",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "contains",
            "matchKey": "capture_gate",
            "entity": "marker_sorted_singlet",
            "valueField": "reactivity_evidence",
            "expected": [
              "engagement_and_clonal_expansion"
            ]
          },
          {
            "name": "cf_no_engaged_reactive_population",
            "role": "hard_fail",
            "field": "$.population_findings[*].reactivity_evidence",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_marker_captures_all_reactive",
            "role": "hard_fail",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "captures_full_reactive_repertoire"
          },
          {
            "name": "cf_superiority_overclaimed_from_phenotype",
            "role": "hard_fail",
            "field": "superiority_claim_basis",
            "kind": "value",
            "op": "in",
            "expected": [
              "established_by_scrna_phenotype",
              "established_by_reactivity_proxy_scores"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-luna-codex",
      "modelName": "gpt-5.6-luna",
      "harness": "openai-codex",
      "task": "10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 1,
      "passed": true,
      "score": null,
      "durationS": 704.719981,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_singlet_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "TOXhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "LAG3high_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "GZMKhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "intermediate_exhausted_gzmk"
                    ],
                    "canonical": "intermediate_exhausted_gzmk"
                  }
                }
              },
              {
                "key": "TCF7_stem_like_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "stem_like_exhausted"
                    ],
                    "canonical": "stem_like_exhausted"
                  }
                }
              },
              {
                "key": "MKI67_pos_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              },
              {
                "key": "MKI67high_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_tumor_cluster_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "Tn",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "Tn_Tmem",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "early_Tem_GZMK",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              },
              {
                "key": "Tem_GZMH",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              }
            ]
          },
          {
            "name": "clonotype_expansion_direction_data_true",
            "role": "gate",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_substantially_more_expanded_than_cluster",
              "singlet_modestly_more_expanded_than_cluster"
            ]
          },
          {
            "name": "biological_inference_data_true",
            "role": "gate",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_antigen_experienced_cluster_bystander"
            ]
          },
          {
            "name": "singlet_top3_contains_naive_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Tn",
              "Tn_Tmem",
              "early_Tem_GZMK",
              "Tem_GZMH"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "TOXhigh_Tex",
              "LAG3high_Tex",
              "GZMKhigh_Tex",
              "TCF7_stem_like_Tex",
              "MKI67_pos_Tex_Tprol",
              "MKI67high_Tex_Tprol"
            ]
          },
          {
            "name": "singlet_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "TOXhigh_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "TCF7_stem_like_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "MKI67_state_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "Tn_or_Tn_Tmem_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_stemlike_precursor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_reported_canonical_expansion_direction",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_substantially_more_expanded_than_singlet",
              "cluster_modestly_more_expanded_than_singlet"
            ]
          },
          {
            "name": "cf_expansion_lazy_defer",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "expansion_pattern_not_detectable"
            ]
          },
          {
            "name": "cf_reported_canonical_expected_biology",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_antigen_experienced_singlet_bystander"
            ]
          },
          {
            "name": "cf_biological_inference_null_artifact_or_lineage",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "differences_reflect_dissociation_or_sorting_artifact",
              "cluster_and_singlet_capture_same_biology",
              "cluster_and_singlet_capture_distinct_cd8_lineages"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "biological_relations",
            "kind": "dict",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_greater_than",
                  "greater_than"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "less_than",
                  "much_less_than"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than",
                  "approximately_equal"
                ]
              },
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal"
                ]
              },
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal",
                  "greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_exhaustion_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_naive_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_proliferation_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_expansion_richness_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_multichain_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_intermediate_gzmk_overgeneralized",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_melanocytic_conjugate_overcall",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "values": [
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_relations_mass_hedge",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-luna-codex",
      "modelName": "gpt-5.6-luna",
      "harness": "openai-codex",
      "task": "10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 510.114566,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_singlet_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "TOXhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "LAG3high_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "GZMKhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "intermediate_exhausted_gzmk"
                    ],
                    "canonical": "intermediate_exhausted_gzmk"
                  }
                }
              },
              {
                "key": "TCF7_stem_like_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "stem_like_exhausted"
                    ],
                    "canonical": "stem_like_exhausted"
                  }
                }
              },
              {
                "key": "MKI67_pos_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              },
              {
                "key": "MKI67high_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_tumor_cluster_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "Tn",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "Tn_Tmem",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "early_Tem_GZMK",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              },
              {
                "key": "Tem_GZMH",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              }
            ]
          },
          {
            "name": "clonotype_expansion_direction_data_true",
            "role": "gate",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_substantially_more_expanded_than_cluster",
              "singlet_modestly_more_expanded_than_cluster"
            ]
          },
          {
            "name": "biological_inference_data_true",
            "role": "gate",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_antigen_experienced_cluster_bystander"
            ]
          },
          {
            "name": "singlet_top3_contains_naive_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Tn",
              "Tn_Tmem",
              "early_Tem_GZMK",
              "Tem_GZMH"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "TOXhigh_Tex",
              "LAG3high_Tex",
              "GZMKhigh_Tex",
              "TCF7_stem_like_Tex",
              "MKI67_pos_Tex_Tprol",
              "MKI67high_Tex_Tprol"
            ]
          },
          {
            "name": "singlet_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "TOXhigh_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "TCF7_stem_like_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "MKI67_state_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "Tn_or_Tn_Tmem_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_stemlike_precursor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_reported_canonical_expansion_direction",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_substantially_more_expanded_than_singlet",
              "cluster_modestly_more_expanded_than_singlet"
            ]
          },
          {
            "name": "cf_expansion_lazy_defer",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "expansion_pattern_not_detectable"
            ]
          },
          {
            "name": "cf_reported_canonical_expected_biology",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_antigen_experienced_singlet_bystander"
            ]
          },
          {
            "name": "cf_biological_inference_null_artifact_or_lineage",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "differences_reflect_dissociation_or_sorting_artifact",
              "cluster_and_singlet_capture_same_biology",
              "cluster_and_singlet_capture_distinct_cd8_lineages"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "biological_relations",
            "kind": "dict",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_greater_than",
                  "greater_than"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "less_than",
                  "much_less_than"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than",
                  "approximately_equal"
                ]
              },
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal"
                ]
              },
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal",
                  "greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_exhaustion_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_naive_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_proliferation_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_expansion_richness_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_multichain_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_intermediate_gzmk_overgeneralized",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_melanocytic_conjugate_overcall",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "values": [
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_relations_mass_hedge",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-luna-codex",
      "modelName": "gpt-5.6-luna",
      "harness": "openai-codex",
      "task": "10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 691.777148,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_singlet_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "TOXhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "LAG3high_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "GZMKhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "intermediate_exhausted_gzmk"
                    ],
                    "canonical": "intermediate_exhausted_gzmk"
                  }
                }
              },
              {
                "key": "TCF7_stem_like_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "stem_like_exhausted"
                    ],
                    "canonical": "stem_like_exhausted"
                  }
                }
              },
              {
                "key": "MKI67_pos_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              },
              {
                "key": "MKI67high_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_tumor_cluster_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "Tn",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "Tn_Tmem",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "early_Tem_GZMK",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              },
              {
                "key": "Tem_GZMH",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              }
            ]
          },
          {
            "name": "clonotype_expansion_direction_data_true",
            "role": "gate",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_substantially_more_expanded_than_cluster",
              "singlet_modestly_more_expanded_than_cluster"
            ]
          },
          {
            "name": "biological_inference_data_true",
            "role": "gate",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_antigen_experienced_cluster_bystander"
            ]
          },
          {
            "name": "singlet_top3_contains_naive_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Tn",
              "Tn_Tmem",
              "early_Tem_GZMK",
              "Tem_GZMH"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "TOXhigh_Tex",
              "LAG3high_Tex",
              "GZMKhigh_Tex",
              "TCF7_stem_like_Tex",
              "MKI67_pos_Tex_Tprol",
              "MKI67high_Tex_Tprol"
            ]
          },
          {
            "name": "singlet_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "TOXhigh_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "TCF7_stem_like_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "MKI67_state_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "Tn_or_Tn_Tmem_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_stemlike_precursor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_reported_canonical_expansion_direction",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_substantially_more_expanded_than_singlet",
              "cluster_modestly_more_expanded_than_singlet"
            ]
          },
          {
            "name": "cf_expansion_lazy_defer",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "expansion_pattern_not_detectable"
            ]
          },
          {
            "name": "cf_reported_canonical_expected_biology",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_antigen_experienced_singlet_bystander"
            ]
          },
          {
            "name": "cf_biological_inference_null_artifact_or_lineage",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "differences_reflect_dissociation_or_sorting_artifact",
              "cluster_and_singlet_capture_same_biology",
              "cluster_and_singlet_capture_distinct_cd8_lineages"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "biological_relations",
            "kind": "dict",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_greater_than",
                  "greater_than"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "less_than",
                  "much_less_than"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than",
                  "approximately_equal"
                ]
              },
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal"
                ]
              },
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal",
                  "greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_exhaustion_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_naive_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_proliferation_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_expansion_richness_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_multichain_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_intermediate_gzmk_overgeneralized",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_melanocytic_conjugate_overcall",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "values": [
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_relations_mass_hedge",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-luna-codex",
      "modelName": "gpt-5.6-luna",
      "harness": "openai-codex",
      "task": "10x_5p_melanoma_tcr_matched_apc_vs_tumor_exhaustion",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 269.549894,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "state_discriminating_genes",
            "kind": "list",
            "matchKey": "gene",
            "k": 3,
            "expected": [
              {
                "key": "CTLA4",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TOX",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "ENTPD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "HAVCR2",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TIGIT",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "PDCD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "difference_attributable_to",
            "role": "gate",
            "field": "difference_attributable_to",
            "kind": "value",
            "expected": "covaries_with_apc_conjugate_state"
          },
          {
            "name": "causal_status",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "direction_unresolved_selection_confound"
          },
          {
            "name": "functional_interpretation",
            "role": "gate",
            "field": "functional_interpretation",
            "kind": "value",
            "expected": "costimulation_signature_function_unresolved"
          },
          {
            "name": "cf_contamination_gene_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "MLANA",
              "PMEL",
              "DCT",
              "TYR",
              "TYRP1",
              "SOX10",
              "MITF",
              "MAGEA4"
            ]
          },
          {
            "name": "cf_apc_partner_contamination_gene",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "HLA-DRA",
              "CD68",
              "LYZ",
              "CD14",
              "C1QA",
              "ITGAX",
              "CD1C"
            ]
          },
          {
            "name": "cf_ligand_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "CD274",
              "PDCD1LG2",
              "CD80",
              "CD86"
            ]
          },
          {
            "name": "cf_attribution_causal_or_baseline",
            "role": "hard_fail",
            "field": "$.difference_attributable_to",
            "kind": "avoid",
            "expected": [
              "apc_engagement",
              "tumor_cell_engagement",
              "clonotype_intrinsic_baseline"
            ]
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "engagement_induces_exhausted_state",
              "pre_exhausted_cells_select_into_apc_conjugates"
            ]
          },
          {
            "name": "cf_causal_fabrication",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "resolved_by_pseudotemporal_ordering",
              "resolved_by_perturbation_experiment"
            ]
          },
          {
            "name": "cf_functional_overclaim",
            "role": "hard_fail",
            "field": "$.functional_interpretation",
            "kind": "avoid",
            "expected": [
              "costimulation_licensed_reinvigoration_competent",
              "costimulation_licensed_checkpoint_responsive",
              "tumor_driven_terminal_exhaustion"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-luna-codex",
      "modelName": "gpt-5.6-luna",
      "harness": "openai-codex",
      "task": "10x_5p_melanoma_tcr_matched_apc_vs_tumor_exhaustion",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 468.774376,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "state_discriminating_genes",
            "kind": "list",
            "matchKey": "gene",
            "k": 3,
            "expected": [
              {
                "key": "CTLA4",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TOX",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "ENTPD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "HAVCR2",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TIGIT",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "PDCD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "difference_attributable_to",
            "role": "gate",
            "field": "difference_attributable_to",
            "kind": "value",
            "expected": "covaries_with_apc_conjugate_state"
          },
          {
            "name": "causal_status",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "direction_unresolved_selection_confound"
          },
          {
            "name": "functional_interpretation",
            "role": "gate",
            "field": "functional_interpretation",
            "kind": "value",
            "expected": "costimulation_signature_function_unresolved"
          },
          {
            "name": "cf_contamination_gene_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "MLANA",
              "PMEL",
              "DCT",
              "TYR",
              "TYRP1",
              "SOX10",
              "MITF",
              "MAGEA4"
            ]
          },
          {
            "name": "cf_apc_partner_contamination_gene",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "HLA-DRA",
              "CD68",
              "LYZ",
              "CD14",
              "C1QA",
              "ITGAX",
              "CD1C"
            ]
          },
          {
            "name": "cf_ligand_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "CD274",
              "PDCD1LG2",
              "CD80",
              "CD86"
            ]
          },
          {
            "name": "cf_attribution_causal_or_baseline",
            "role": "hard_fail",
            "field": "$.difference_attributable_to",
            "kind": "avoid",
            "expected": [
              "apc_engagement",
              "tumor_cell_engagement",
              "clonotype_intrinsic_baseline"
            ]
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "engagement_induces_exhausted_state",
              "pre_exhausted_cells_select_into_apc_conjugates"
            ]
          },
          {
            "name": "cf_causal_fabrication",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "resolved_by_pseudotemporal_ordering",
              "resolved_by_perturbation_experiment"
            ]
          },
          {
            "name": "cf_functional_overclaim",
            "role": "hard_fail",
            "field": "$.functional_interpretation",
            "kind": "avoid",
            "expected": [
              "costimulation_licensed_reinvigoration_competent",
              "costimulation_licensed_checkpoint_responsive",
              "tumor_driven_terminal_exhaustion"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-luna-codex",
      "modelName": "gpt-5.6-luna",
      "harness": "openai-codex",
      "task": "10x_5p_melanoma_tcr_matched_apc_vs_tumor_exhaustion",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 658.768075,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "state_discriminating_genes",
            "kind": "list",
            "matchKey": "gene",
            "k": 3,
            "expected": [
              {
                "key": "CTLA4",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TOX",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "ENTPD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "HAVCR2",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TIGIT",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "PDCD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "difference_attributable_to",
            "role": "gate",
            "field": "difference_attributable_to",
            "kind": "value",
            "expected": "covaries_with_apc_conjugate_state"
          },
          {
            "name": "causal_status",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "direction_unresolved_selection_confound"
          },
          {
            "name": "functional_interpretation",
            "role": "gate",
            "field": "functional_interpretation",
            "kind": "value",
            "expected": "costimulation_signature_function_unresolved"
          },
          {
            "name": "cf_contamination_gene_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "MLANA",
              "PMEL",
              "DCT",
              "TYR",
              "TYRP1",
              "SOX10",
              "MITF",
              "MAGEA4"
            ]
          },
          {
            "name": "cf_apc_partner_contamination_gene",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "HLA-DRA",
              "CD68",
              "LYZ",
              "CD14",
              "C1QA",
              "ITGAX",
              "CD1C"
            ]
          },
          {
            "name": "cf_ligand_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "CD274",
              "PDCD1LG2",
              "CD80",
              "CD86"
            ]
          },
          {
            "name": "cf_attribution_causal_or_baseline",
            "role": "hard_fail",
            "field": "$.difference_attributable_to",
            "kind": "avoid",
            "expected": [
              "apc_engagement",
              "tumor_cell_engagement",
              "clonotype_intrinsic_baseline"
            ]
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "engagement_induces_exhausted_state",
              "pre_exhausted_cells_select_into_apc_conjugates"
            ]
          },
          {
            "name": "cf_causal_fabrication",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "resolved_by_pseudotemporal_ordering",
              "resolved_by_perturbation_experiment"
            ]
          },
          {
            "name": "cf_functional_overclaim",
            "role": "hard_fail",
            "field": "$.functional_interpretation",
            "kind": "avoid",
            "expected": [
              "costimulation_licensed_reinvigoration_competent",
              "costimulation_licensed_checkpoint_responsive",
              "tumor_driven_terminal_exhaustion"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-luna-codex",
      "modelName": "gpt-5.6-luna",
      "harness": "openai-codex",
      "task": "green_2_tme_context_dependence",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 1048.894497,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "program_map",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "sig_includes_itgae",
            "role": "gate",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Itgae"
            ]
          },
          {
            "name": "declared_a_known_confounder",
            "role": "gate",
            "field": "$.excluded_confounders[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "cf_signature_gut_contaminant",
            "role": "hard_fail",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "generic_signature_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd103_program_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "<"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "per_model_regulatory_state",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "poised_unfired"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "late_activation"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "closing_committed"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_tuples",
            "kind": "list",
            "matchKey": "regulator",
            "k": 1,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "role": {
                    "op": "equals",
                    "values": [
                      "repressor"
                    ]
                  },
                  "gating_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  },
                  "evidence_basis": {
                    "op": "equals",
                    "values": [
                      "target_program_behavior"
                    ]
                  },
                  "expr_vs_activity": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "locus_differential_gate",
            "role": "gate",
            "field": "locus_differential",
            "kind": "value",
            "expected": "uniform_across_models"
          },
          {
            "name": "formation_limited_by_gate",
            "role": "gate",
            "field": "formation_limited_by",
            "kind": "value",
            "expected": "absent_induction"
          },
          {
            "name": "locus_competent_in_naive_gate",
            "role": "gate",
            "field": "locus_competent_in_naive",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "ready_silent_fraction_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 3,
            "expected": [
              {
                "key": "B16_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_KPC_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "activator_chromatin_footprint_gate",
            "role": "gate",
            "field": "activator_chromatin_footprint",
            "kind": "value",
            "expected": "inconsistent_across_formers"
          },
          {
            "name": "exhaustion_comparable_gate",
            "role": "gate",
            "field": "exhaustion_comparable_across_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_nondeterminant_gate",
            "role": "gate",
            "field": "exhaustion_explains_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_separable_program_gate",
            "role": "gate",
            "field": "exhaustion_separable_program",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "exhaustion_counterexample_gate",
            "role": "gate",
            "field": "exhaustion_counterexample_model",
            "kind": "value",
            "expected": "KPC_GP"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "exhaustion_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "cf_inconsistent_present_without_activator",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "program",
            "entity": "present",
            "valueField": "activator_state",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_no_block_but_absent",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "none",
            "valueField": "program",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_late_block_without_poised_locus",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "late_activation",
            "valueField": "locus_state",
            "expected": [
              "closing_committed",
              "active"
            ]
          },
          {
            "name": "cf_inconsistent_early_block_with_brake_released",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "early_release",
            "valueField": "brake_state",
            "expected": [
              "released"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-luna-codex",
      "modelName": "gpt-5.6-luna",
      "harness": "openai-codex",
      "task": "green_2_tme_context_dependence",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 1161.707659,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "program_map",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "sig_includes_itgae",
            "role": "gate",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Itgae"
            ]
          },
          {
            "name": "declared_a_known_confounder",
            "role": "gate",
            "field": "$.excluded_confounders[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "cf_signature_gut_contaminant",
            "role": "hard_fail",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "generic_signature_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd103_program_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "<"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "per_model_regulatory_state",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "poised_unfired"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "late_activation"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "closing_committed"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_tuples",
            "kind": "list",
            "matchKey": "regulator",
            "k": 1,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "role": {
                    "op": "equals",
                    "values": [
                      "repressor"
                    ]
                  },
                  "gating_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  },
                  "evidence_basis": {
                    "op": "equals",
                    "values": [
                      "target_program_behavior"
                    ]
                  },
                  "expr_vs_activity": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "locus_differential_gate",
            "role": "gate",
            "field": "locus_differential",
            "kind": "value",
            "expected": "uniform_across_models"
          },
          {
            "name": "formation_limited_by_gate",
            "role": "gate",
            "field": "formation_limited_by",
            "kind": "value",
            "expected": "absent_induction"
          },
          {
            "name": "locus_competent_in_naive_gate",
            "role": "gate",
            "field": "locus_competent_in_naive",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "ready_silent_fraction_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 3,
            "expected": [
              {
                "key": "B16_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_KPC_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "activator_chromatin_footprint_gate",
            "role": "gate",
            "field": "activator_chromatin_footprint",
            "kind": "value",
            "expected": "inconsistent_across_formers"
          },
          {
            "name": "exhaustion_comparable_gate",
            "role": "gate",
            "field": "exhaustion_comparable_across_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_nondeterminant_gate",
            "role": "gate",
            "field": "exhaustion_explains_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_separable_program_gate",
            "role": "gate",
            "field": "exhaustion_separable_program",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "exhaustion_counterexample_gate",
            "role": "gate",
            "field": "exhaustion_counterexample_model",
            "kind": "value",
            "expected": "KPC_GP"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "exhaustion_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "cf_inconsistent_present_without_activator",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "program",
            "entity": "present",
            "valueField": "activator_state",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_no_block_but_absent",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "none",
            "valueField": "program",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_late_block_without_poised_locus",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "late_activation",
            "valueField": "locus_state",
            "expected": [
              "closing_committed",
              "active"
            ]
          },
          {
            "name": "cf_inconsistent_early_block_with_brake_released",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "early_release",
            "valueField": "brake_state",
            "expected": [
              "released"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-luna-codex",
      "modelName": "gpt-5.6-luna",
      "harness": "openai-codex",
      "task": "green_2_tme_context_dependence",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1494.945099,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "program_map",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "sig_includes_itgae",
            "role": "gate",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Itgae"
            ]
          },
          {
            "name": "declared_a_known_confounder",
            "role": "gate",
            "field": "$.excluded_confounders[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "cf_signature_gut_contaminant",
            "role": "hard_fail",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "generic_signature_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd103_program_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "<"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "per_model_regulatory_state",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "poised_unfired"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "late_activation"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "closing_committed"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_tuples",
            "kind": "list",
            "matchKey": "regulator",
            "k": 1,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "role": {
                    "op": "equals",
                    "values": [
                      "repressor"
                    ]
                  },
                  "gating_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  },
                  "evidence_basis": {
                    "op": "equals",
                    "values": [
                      "target_program_behavior"
                    ]
                  },
                  "expr_vs_activity": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "locus_differential_gate",
            "role": "gate",
            "field": "locus_differential",
            "kind": "value",
            "expected": "uniform_across_models"
          },
          {
            "name": "formation_limited_by_gate",
            "role": "gate",
            "field": "formation_limited_by",
            "kind": "value",
            "expected": "absent_induction"
          },
          {
            "name": "locus_competent_in_naive_gate",
            "role": "gate",
            "field": "locus_competent_in_naive",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "ready_silent_fraction_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 3,
            "expected": [
              {
                "key": "B16_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_KPC_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "activator_chromatin_footprint_gate",
            "role": "gate",
            "field": "activator_chromatin_footprint",
            "kind": "value",
            "expected": "inconsistent_across_formers"
          },
          {
            "name": "exhaustion_comparable_gate",
            "role": "gate",
            "field": "exhaustion_comparable_across_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_nondeterminant_gate",
            "role": "gate",
            "field": "exhaustion_explains_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_separable_program_gate",
            "role": "gate",
            "field": "exhaustion_separable_program",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "exhaustion_counterexample_gate",
            "role": "gate",
            "field": "exhaustion_counterexample_model",
            "kind": "value",
            "expected": "KPC_GP"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "exhaustion_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "cf_inconsistent_present_without_activator",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "program",
            "entity": "present",
            "valueField": "activator_state",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_no_block_but_absent",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "none",
            "valueField": "program",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_late_block_without_poised_locus",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "late_activation",
            "valueField": "locus_state",
            "expected": [
              "closing_committed",
              "active"
            ]
          },
          {
            "name": "cf_inconsistent_early_block_with_brake_released",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "early_release",
            "valueField": "brake_state",
            "expected": [
              "released"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-luna-codex",
      "modelName": "gpt-5.6-luna",
      "harness": "openai-codex",
      "task": "green_klf2_hidden_repressor_of_trm_like_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 1267.368802,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "regulator_gate",
            "role": "gate",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "KLF2",
              "Klf2"
            ]
          },
          {
            "name": "controlling_node_character_gate",
            "role": "gate",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "high_differential_expression"
          },
          {
            "name": "expression_vs_activity_gate",
            "role": "gate",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "expression_exceeds_activity"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "target_program_partition",
            "kind": "list",
            "matchKey": "gene",
            "k": 7,
            "expected": [
              {
                "key": "S1pr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "S1pr4",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Klrg1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Cx3cr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Itgae",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Hic1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "P2rx7",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              }
            ]
          },
          {
            "name": "causal_status_gate",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "association_only_sufficiency_requires_perturbation"
          },
          {
            "name": "residency_control_logic_gate",
            "role": "gate",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_release_necessary_not_sufficient_multi_regulon"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "trm_like_program_by_model",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_absent"
                    ],
                    "canonical": "niche_absent"
                  }
                }
              }
            ]
          },
          {
            "name": "cf_regulator_activator_or_wrong_node",
            "role": "hard_fail",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "RUNX3",
              "Runx3",
              "PRDM1",
              "Prdm1",
              "BLIMP1",
              "Blimp1",
              "BHLHE40",
              "Bhlhe40",
              "ZNF683",
              "Zfp683",
              "HOBIT",
              "Hobit",
              "AHR",
              "Ahr",
              "BATF",
              "Batf",
              "TOX",
              "Tox",
              "NR4A1",
              "Nr4a1",
              "NR4A2",
              "Nr4a2",
              "NR4A3",
              "Nr4a3",
              "TCF7",
              "Tcf7",
              "TCF1",
              "Tcf1",
              "BACH2",
              "Bach2",
              "TBX21",
              "Tbx21",
              "TBET",
              "Tbet",
              "FLI1",
              "Fli1",
              "NR3C1",
              "Nr3c1",
              "EOMES",
              "Eomes",
              "ZEB2",
              "Zeb2",
              "ID2",
              "Id2",
              "ID3",
              "Id3",
              "IRF4",
              "Irf4"
            ]
          },
          {
            "name": "cf_node_character_hidden_driver_recall",
            "role": "hard_fail",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "hidden_driver_activity_not_expression"
          },
          {
            "name": "cf_effect_relation_hidden_pattern",
            "role": "hard_fail",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "activity_exceeds_expression"
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "causal_status",
            "kind": "value",
            "op": "in",
            "expected": [
              "sufficiency_established",
              "necessity_established",
              "causal_established_in_dataset",
              "sufficiency_via_perturbation_in_dataset"
            ]
          },
          {
            "name": "cf_residency_logic_activator_prior",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "residency_activator_induction_primary"
          },
          {
            "name": "cf_residency_logic_single_regulon",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_controls_entire_residency_program"
          },
          {
            "name": "cf_decoy_called_target",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_distractor_model_reported",
            "role": "hard_fail",
            "field": "$.trm_like_program_by_model[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_chromatin_only_decoy",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-luna-codex",
      "modelName": "gpt-5.6-luna",
      "harness": "openai-codex",
      "task": "green_klf2_hidden_repressor_of_trm_like_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 1924.194547,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "regulator_gate",
            "role": "gate",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "KLF2",
              "Klf2"
            ]
          },
          {
            "name": "controlling_node_character_gate",
            "role": "gate",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "high_differential_expression"
          },
          {
            "name": "expression_vs_activity_gate",
            "role": "gate",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "expression_exceeds_activity"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "target_program_partition",
            "kind": "list",
            "matchKey": "gene",
            "k": 7,
            "expected": [
              {
                "key": "S1pr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "S1pr4",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Klrg1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Cx3cr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Itgae",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Hic1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "P2rx7",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              }
            ]
          },
          {
            "name": "causal_status_gate",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "association_only_sufficiency_requires_perturbation"
          },
          {
            "name": "residency_control_logic_gate",
            "role": "gate",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_release_necessary_not_sufficient_multi_regulon"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "trm_like_program_by_model",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_absent"
                    ],
                    "canonical": "niche_absent"
                  }
                }
              }
            ]
          },
          {
            "name": "cf_regulator_activator_or_wrong_node",
            "role": "hard_fail",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "RUNX3",
              "Runx3",
              "PRDM1",
              "Prdm1",
              "BLIMP1",
              "Blimp1",
              "BHLHE40",
              "Bhlhe40",
              "ZNF683",
              "Zfp683",
              "HOBIT",
              "Hobit",
              "AHR",
              "Ahr",
              "BATF",
              "Batf",
              "TOX",
              "Tox",
              "NR4A1",
              "Nr4a1",
              "NR4A2",
              "Nr4a2",
              "NR4A3",
              "Nr4a3",
              "TCF7",
              "Tcf7",
              "TCF1",
              "Tcf1",
              "BACH2",
              "Bach2",
              "TBX21",
              "Tbx21",
              "TBET",
              "Tbet",
              "FLI1",
              "Fli1",
              "NR3C1",
              "Nr3c1",
              "EOMES",
              "Eomes",
              "ZEB2",
              "Zeb2",
              "ID2",
              "Id2",
              "ID3",
              "Id3",
              "IRF4",
              "Irf4"
            ]
          },
          {
            "name": "cf_node_character_hidden_driver_recall",
            "role": "hard_fail",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "hidden_driver_activity_not_expression"
          },
          {
            "name": "cf_effect_relation_hidden_pattern",
            "role": "hard_fail",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "activity_exceeds_expression"
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "causal_status",
            "kind": "value",
            "op": "in",
            "expected": [
              "sufficiency_established",
              "necessity_established",
              "causal_established_in_dataset",
              "sufficiency_via_perturbation_in_dataset"
            ]
          },
          {
            "name": "cf_residency_logic_activator_prior",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "residency_activator_induction_primary"
          },
          {
            "name": "cf_residency_logic_single_regulon",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_controls_entire_residency_program"
          },
          {
            "name": "cf_decoy_called_target",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_distractor_model_reported",
            "role": "hard_fail",
            "field": "$.trm_like_program_by_model[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_chromatin_only_decoy",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-luna-codex",
      "modelName": "gpt-5.6-luna",
      "harness": "openai-codex",
      "task": "green_klf2_hidden_repressor_of_trm_like_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1566.277742,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "regulator_gate",
            "role": "gate",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "KLF2",
              "Klf2"
            ]
          },
          {
            "name": "controlling_node_character_gate",
            "role": "gate",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "high_differential_expression"
          },
          {
            "name": "expression_vs_activity_gate",
            "role": "gate",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "expression_exceeds_activity"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "target_program_partition",
            "kind": "list",
            "matchKey": "gene",
            "k": 7,
            "expected": [
              {
                "key": "S1pr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "S1pr4",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Klrg1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Cx3cr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Itgae",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Hic1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "P2rx7",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              }
            ]
          },
          {
            "name": "causal_status_gate",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "association_only_sufficiency_requires_perturbation"
          },
          {
            "name": "residency_control_logic_gate",
            "role": "gate",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_release_necessary_not_sufficient_multi_regulon"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "trm_like_program_by_model",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_absent"
                    ],
                    "canonical": "niche_absent"
                  }
                }
              }
            ]
          },
          {
            "name": "cf_regulator_activator_or_wrong_node",
            "role": "hard_fail",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "RUNX3",
              "Runx3",
              "PRDM1",
              "Prdm1",
              "BLIMP1",
              "Blimp1",
              "BHLHE40",
              "Bhlhe40",
              "ZNF683",
              "Zfp683",
              "HOBIT",
              "Hobit",
              "AHR",
              "Ahr",
              "BATF",
              "Batf",
              "TOX",
              "Tox",
              "NR4A1",
              "Nr4a1",
              "NR4A2",
              "Nr4a2",
              "NR4A3",
              "Nr4a3",
              "TCF7",
              "Tcf7",
              "TCF1",
              "Tcf1",
              "BACH2",
              "Bach2",
              "TBX21",
              "Tbx21",
              "TBET",
              "Tbet",
              "FLI1",
              "Fli1",
              "NR3C1",
              "Nr3c1",
              "EOMES",
              "Eomes",
              "ZEB2",
              "Zeb2",
              "ID2",
              "Id2",
              "ID3",
              "Id3",
              "IRF4",
              "Irf4"
            ]
          },
          {
            "name": "cf_node_character_hidden_driver_recall",
            "role": "hard_fail",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "hidden_driver_activity_not_expression"
          },
          {
            "name": "cf_effect_relation_hidden_pattern",
            "role": "hard_fail",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "activity_exceeds_expression"
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "causal_status",
            "kind": "value",
            "op": "in",
            "expected": [
              "sufficiency_established",
              "necessity_established",
              "causal_established_in_dataset",
              "sufficiency_via_perturbation_in_dataset"
            ]
          },
          {
            "name": "cf_residency_logic_activator_prior",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "residency_activator_induction_primary"
          },
          {
            "name": "cf_residency_logic_single_regulon",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_controls_entire_residency_program"
          },
          {
            "name": "cf_decoy_called_target",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_distractor_model_reported",
            "role": "hard_fail",
            "field": "$.trm_like_program_by_model[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_chromatin_only_decoy",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-luna-codex",
      "modelName": "gpt-5.6-luna",
      "harness": "openai-codex",
      "task": "green_shared_vs_distinct_regulators_trm_vs_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 1300.846062,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_program_map",
            "kind": "list",
            "matchKey": "regulator",
            "k": 3,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "both"
                    ],
                    "canonical": "both"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "represses"
                    ],
                    "canonical": "represses"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Batf",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "trm_like_til_specific"
                    ],
                    "canonical": "trm_like_til_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "chromatin_over_rna"
                    ],
                    "canonical": "chromatin_over_rna"
                  }
                }
              },
              {
                "key": "Ahr",
                "fields": {
                  "program_specificity": {
                    "op": "in",
                    "values": [
                      "trm_specific",
                      "both"
                    ],
                    "canonical": "trm_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "rna_over_chromatin"
                    ],
                    "canonical": "rna_over_chromatin"
                  }
                }
              }
            ]
          },
          {
            "name": "regulator_set_no_extras_gate",
            "role": "gate",
            "field": "$.regulator_program_map[*]",
            "kind": "other",
            "expected": null
          },
          {
            "name": "program_identity_conclusion_gate",
            "role": "gate",
            "field": "program_identity_conclusion",
            "kind": "value",
            "expected": "distinct_state_shared_brake_divergent_activators"
          },
          {
            "name": "residency_install_logic_gate",
            "role": "gate",
            "field": "residency_install_logic",
            "kind": "value",
            "expected": "brake_release_permits_context_matched_activator"
          },
          {
            "name": "activator_context_dependence_gate",
            "role": "gate",
            "field": "activator_context_dependence",
            "kind": "value",
            "expected": "context_matched_crossover"
          },
          {
            "name": "evidence_ceiling_gate",
            "role": "gate",
            "field": "evidence_ceiling",
            "kind": "value",
            "expected": "specificity_dissociations_require_perturbation"
          },
          {
            "name": "cf_install_logic_flat_or_single_step",
            "role": "hard_fail",
            "field": "residency_install_logic",
            "kind": "value",
            "op": "in",
            "expected": [
              "flat_shared_residency_toolkit",
              "activator_induction_sufficient",
              "brake_release_alone_sufficient"
            ]
          },
          {
            "name": "cf_program_identity_lumped_or_wrong",
            "role": "hard_fail",
            "field": "program_identity_conclusion",
            "kind": "value",
            "op": "in",
            "expected": [
              "trm_like_til_is_tumor_trm",
              "trm_like_til_is_exhausted_subset",
              "trm_like_til_is_effector_state",
              "trm_like_til_is_transitional_intermediate",
              "trm_like_til_is_shared_program_distinct_niche"
            ]
          },
          {
            "name": "cf_ahr_assigned_to_til",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Ahr",
            "valueField": "program_specificity",
            "expected": [
              "trm_like_til_specific"
            ]
          },
          {
            "name": "cf_batf_assigned_to_trm_or_shared",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Batf",
            "valueField": "program_specificity",
            "expected": [
              "trm_specific",
              "both"
            ]
          },
          {
            "name": "cf_klf2_called_activator",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Klf2",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          },
          {
            "name": "cf_evidence_ceiling_overclaim",
            "role": "hard_fail",
            "field": "evidence_ceiling",
            "kind": "value",
            "op": "in",
            "expected": [
              "specificity_established_from_regulon_activity",
              "specificity_established_in_dataset",
              "causation_established_by_cross_sectional_concordance",
              "necessity_and_sufficiency_established"
            ]
          },
          {
            "name": "cf_activator_context_shared_or_single",
            "role": "hard_fail",
            "field": "activator_context_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "shared_activators_both_contexts",
              "single_global_activator",
              "redundant_activators"
            ]
          },
          {
            "name": "cf_single_modality_rule",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_hic1_motif_brake_mislabel",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_runx3_anticanonical_as_controller",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Runx3",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-luna-codex",
      "modelName": "gpt-5.6-luna",
      "harness": "openai-codex",
      "task": "green_shared_vs_distinct_regulators_trm_vs_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 1944.049708,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_program_map",
            "kind": "list",
            "matchKey": "regulator",
            "k": 3,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "both"
                    ],
                    "canonical": "both"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "represses"
                    ],
                    "canonical": "represses"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Batf",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "trm_like_til_specific"
                    ],
                    "canonical": "trm_like_til_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "chromatin_over_rna"
                    ],
                    "canonical": "chromatin_over_rna"
                  }
                }
              },
              {
                "key": "Ahr",
                "fields": {
                  "program_specificity": {
                    "op": "in",
                    "values": [
                      "trm_specific",
                      "both"
                    ],
                    "canonical": "trm_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "rna_over_chromatin"
                    ],
                    "canonical": "rna_over_chromatin"
                  }
                }
              }
            ]
          },
          {
            "name": "regulator_set_no_extras_gate",
            "role": "gate",
            "field": "$.regulator_program_map[*]",
            "kind": "other",
            "expected": null
          },
          {
            "name": "program_identity_conclusion_gate",
            "role": "gate",
            "field": "program_identity_conclusion",
            "kind": "value",
            "expected": "distinct_state_shared_brake_divergent_activators"
          },
          {
            "name": "residency_install_logic_gate",
            "role": "gate",
            "field": "residency_install_logic",
            "kind": "value",
            "expected": "brake_release_permits_context_matched_activator"
          },
          {
            "name": "activator_context_dependence_gate",
            "role": "gate",
            "field": "activator_context_dependence",
            "kind": "value",
            "expected": "context_matched_crossover"
          },
          {
            "name": "evidence_ceiling_gate",
            "role": "gate",
            "field": "evidence_ceiling",
            "kind": "value",
            "expected": "specificity_dissociations_require_perturbation"
          },
          {
            "name": "cf_install_logic_flat_or_single_step",
            "role": "hard_fail",
            "field": "residency_install_logic",
            "kind": "value",
            "op": "in",
            "expected": [
              "flat_shared_residency_toolkit",
              "activator_induction_sufficient",
              "brake_release_alone_sufficient"
            ]
          },
          {
            "name": "cf_program_identity_lumped_or_wrong",
            "role": "hard_fail",
            "field": "program_identity_conclusion",
            "kind": "value",
            "op": "in",
            "expected": [
              "trm_like_til_is_tumor_trm",
              "trm_like_til_is_exhausted_subset",
              "trm_like_til_is_effector_state",
              "trm_like_til_is_transitional_intermediate",
              "trm_like_til_is_shared_program_distinct_niche"
            ]
          },
          {
            "name": "cf_ahr_assigned_to_til",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Ahr",
            "valueField": "program_specificity",
            "expected": [
              "trm_like_til_specific"
            ]
          },
          {
            "name": "cf_batf_assigned_to_trm_or_shared",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Batf",
            "valueField": "program_specificity",
            "expected": [
              "trm_specific",
              "both"
            ]
          },
          {
            "name": "cf_klf2_called_activator",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Klf2",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          },
          {
            "name": "cf_evidence_ceiling_overclaim",
            "role": "hard_fail",
            "field": "evidence_ceiling",
            "kind": "value",
            "op": "in",
            "expected": [
              "specificity_established_from_regulon_activity",
              "specificity_established_in_dataset",
              "causation_established_by_cross_sectional_concordance",
              "necessity_and_sufficiency_established"
            ]
          },
          {
            "name": "cf_activator_context_shared_or_single",
            "role": "hard_fail",
            "field": "activator_context_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "shared_activators_both_contexts",
              "single_global_activator",
              "redundant_activators"
            ]
          },
          {
            "name": "cf_single_modality_rule",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_hic1_motif_brake_mislabel",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_runx3_anticanonical_as_controller",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Runx3",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-luna-codex",
      "modelName": "gpt-5.6-luna",
      "harness": "openai-codex",
      "task": "green_shared_vs_distinct_regulators_trm_vs_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 414.523851,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_program_map",
            "kind": "list",
            "matchKey": "regulator",
            "k": 3,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "both"
                    ],
                    "canonical": "both"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "represses"
                    ],
                    "canonical": "represses"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Batf",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "trm_like_til_specific"
                    ],
                    "canonical": "trm_like_til_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "chromatin_over_rna"
                    ],
                    "canonical": "chromatin_over_rna"
                  }
                }
              },
              {
                "key": "Ahr",
                "fields": {
                  "program_specificity": {
                    "op": "in",
                    "values": [
                      "trm_specific",
                      "both"
                    ],
                    "canonical": "trm_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "rna_over_chromatin"
                    ],
                    "canonical": "rna_over_chromatin"
                  }
                }
              }
            ]
          },
          {
            "name": "regulator_set_no_extras_gate",
            "role": "gate",
            "field": "$.regulator_program_map[*]",
            "kind": "other",
            "expected": null
          },
          {
            "name": "program_identity_conclusion_gate",
            "role": "gate",
            "field": "program_identity_conclusion",
            "kind": "value",
            "expected": "distinct_state_shared_brake_divergent_activators"
          },
          {
            "name": "residency_install_logic_gate",
            "role": "gate",
            "field": "residency_install_logic",
            "kind": "value",
            "expected": "brake_release_permits_context_matched_activator"
          },
          {
            "name": "activator_context_dependence_gate",
            "role": "gate",
            "field": "activator_context_dependence",
            "kind": "value",
            "expected": "context_matched_crossover"
          },
          {
            "name": "evidence_ceiling_gate",
            "role": "gate",
            "field": "evidence_ceiling",
            "kind": "value",
            "expected": "specificity_dissociations_require_perturbation"
          },
          {
            "name": "cf_install_logic_flat_or_single_step",
            "role": "hard_fail",
            "field": "residency_install_logic",
            "kind": "value",
            "op": "in",
            "expected": [
              "flat_shared_residency_toolkit",
              "activator_induction_sufficient",
              "brake_release_alone_sufficient"
            ]
          },
          {
            "name": "cf_program_identity_lumped_or_wrong",
            "role": "hard_fail",
            "field": "program_identity_conclusion",
            "kind": "value",
            "op": "in",
            "expected": [
              "trm_like_til_is_tumor_trm",
              "trm_like_til_is_exhausted_subset",
              "trm_like_til_is_effector_state",
              "trm_like_til_is_transitional_intermediate",
              "trm_like_til_is_shared_program_distinct_niche"
            ]
          },
          {
            "name": "cf_ahr_assigned_to_til",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Ahr",
            "valueField": "program_specificity",
            "expected": [
              "trm_like_til_specific"
            ]
          },
          {
            "name": "cf_batf_assigned_to_trm_or_shared",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Batf",
            "valueField": "program_specificity",
            "expected": [
              "trm_specific",
              "both"
            ]
          },
          {
            "name": "cf_klf2_called_activator",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Klf2",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          },
          {
            "name": "cf_evidence_ceiling_overclaim",
            "role": "hard_fail",
            "field": "evidence_ceiling",
            "kind": "value",
            "op": "in",
            "expected": [
              "specificity_established_from_regulon_activity",
              "specificity_established_in_dataset",
              "causation_established_by_cross_sectional_concordance",
              "necessity_and_sufficiency_established"
            ]
          },
          {
            "name": "cf_activator_context_shared_or_single",
            "role": "hard_fail",
            "field": "activator_context_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "shared_activators_both_contexts",
              "single_global_activator",
              "redundant_activators"
            ]
          },
          {
            "name": "cf_single_modality_rule",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_hic1_motif_brake_mislabel",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_runx3_anticanonical_as_controller",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Runx3",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-luna-codex",
      "modelName": "gpt-5.6-luna",
      "harness": "openai-codex",
      "task": "gse297023_q2_senescence_not_mechanism",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 313.473834,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "candidate_is_senescence",
            "role": "gate",
            "field": "$.candidate_mechanism",
            "kind": "value",
            "op": "in",
            "expected": [
              "cellular_senescence",
              "oncogene_induced_senescence",
              "replicative_senescence"
            ]
          },
          {
            "name": "arrest_core_not_engaged",
            "role": "gate",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_not_engaged"
            ]
          },
          {
            "name": "secretory_arm_is_inflammaging",
            "role": "gate",
            "field": "$.secretory_arm_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "acute_phase_inflammaging_response"
            ]
          },
          {
            "name": "secretory_arm_broadly_elevated",
            "role": "gate",
            "field": "$.secretory_arm_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "broadly_elevated"
            ]
          },
          {
            "name": "carries_aging_program",
            "role": "gate",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "carries_bona_fide_aging_program_distinct_from_nominated"
            ]
          },
          {
            "name": "conclusion_senescence_unlikely",
            "role": "gate",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "unlikely_major_contributor"
            ]
          },
          {
            "name": "cf_arrest_core_engaged_recall",
            "role": "hard_fail",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_engaged"
            ]
          },
          {
            "name": "cf_aging_program_conflation_or_explain_away",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "aging_program_indistinguishable_from_nominated_program",
              "aging_program_is_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_no_aging_program_sensitivity_failure",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_detectable_aging_program"
            ]
          },
          {
            "name": "cf_senescence_driver_conclusion",
            "role": "hard_fail",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "likely_primary_driver",
              "plausible_supported_contributor"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-luna-codex",
      "modelName": "gpt-5.6-luna",
      "harness": "openai-codex",
      "task": "gse297023_q2_senescence_not_mechanism",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 552.235626,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "candidate_is_senescence",
            "role": "gate",
            "field": "$.candidate_mechanism",
            "kind": "value",
            "op": "in",
            "expected": [
              "cellular_senescence",
              "oncogene_induced_senescence",
              "replicative_senescence"
            ]
          },
          {
            "name": "arrest_core_not_engaged",
            "role": "gate",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_not_engaged"
            ]
          },
          {
            "name": "secretory_arm_is_inflammaging",
            "role": "gate",
            "field": "$.secretory_arm_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "acute_phase_inflammaging_response"
            ]
          },
          {
            "name": "secretory_arm_broadly_elevated",
            "role": "gate",
            "field": "$.secretory_arm_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "broadly_elevated"
            ]
          },
          {
            "name": "carries_aging_program",
            "role": "gate",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "carries_bona_fide_aging_program_distinct_from_nominated"
            ]
          },
          {
            "name": "conclusion_senescence_unlikely",
            "role": "gate",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "unlikely_major_contributor"
            ]
          },
          {
            "name": "cf_arrest_core_engaged_recall",
            "role": "hard_fail",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_engaged"
            ]
          },
          {
            "name": "cf_aging_program_conflation_or_explain_away",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "aging_program_indistinguishable_from_nominated_program",
              "aging_program_is_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_no_aging_program_sensitivity_failure",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_detectable_aging_program"
            ]
          },
          {
            "name": "cf_senescence_driver_conclusion",
            "role": "hard_fail",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "likely_primary_driver",
              "plausible_supported_contributor"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-luna-codex",
      "modelName": "gpt-5.6-luna",
      "harness": "openai-codex",
      "task": "gse297023_q2_senescence_not_mechanism",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 805.652491,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "candidate_is_senescence",
            "role": "gate",
            "field": "$.candidate_mechanism",
            "kind": "value",
            "op": "in",
            "expected": [
              "cellular_senescence",
              "oncogene_induced_senescence",
              "replicative_senescence"
            ]
          },
          {
            "name": "arrest_core_not_engaged",
            "role": "gate",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_not_engaged"
            ]
          },
          {
            "name": "secretory_arm_is_inflammaging",
            "role": "gate",
            "field": "$.secretory_arm_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "acute_phase_inflammaging_response"
            ]
          },
          {
            "name": "secretory_arm_broadly_elevated",
            "role": "gate",
            "field": "$.secretory_arm_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "broadly_elevated"
            ]
          },
          {
            "name": "carries_aging_program",
            "role": "gate",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "carries_bona_fide_aging_program_distinct_from_nominated"
            ]
          },
          {
            "name": "conclusion_senescence_unlikely",
            "role": "gate",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "unlikely_major_contributor"
            ]
          },
          {
            "name": "cf_arrest_core_engaged_recall",
            "role": "hard_fail",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_engaged"
            ]
          },
          {
            "name": "cf_aging_program_conflation_or_explain_away",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "aging_program_indistinguishable_from_nominated_program",
              "aging_program_is_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_no_aging_program_sensitivity_failure",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_detectable_aging_program"
            ]
          },
          {
            "name": "cf_senescence_driver_conclusion",
            "role": "hard_fail",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "likely_primary_driver",
              "plausible_supported_contributor"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-luna-codex",
      "modelName": "gpt-5.6-luna",
      "harness": "openai-codex",
      "task": "gse297023_q3_age_retained_mapk_reduced",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 1,
      "passed": true,
      "score": null,
      "durationS": 1055.733856,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "age_retained",
            "role": "gate",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_retained"
            ]
          },
          {
            "name": "identity_is_aging",
            "role": "gate",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_aging_signatures"
            ]
          },
          {
            "name": "nature_state_shift",
            "role": "gate",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_state_expression_shift"
            ]
          },
          {
            "name": "pathway_is_mapk",
            "role": "gate",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "mapk"
            ]
          },
          {
            "name": "pathway_reduced_in_aged",
            "role": "gate",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "reduced_in_aged"
            ]
          },
          {
            "name": "cf_mapk_increased_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_in_aged"
            ]
          },
          {
            "name": "cf_age_erased",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_erased"
            ]
          },
          {
            "name": "cf_pervasive_overcall",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_pervasive_artifactual"
            ]
          },
          {
            "name": "cf_explain_away_aging",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_compositional_or_identity",
            "role": "hard_fail",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_proportion_shift",
              "cell_identity_shift",
              "composition_dominant_state_secondary",
              "both_state_and_composition_equally"
            ]
          },
          {
            "name": "cf_biggest_inflammatory_signal_not_mapk",
            "role": "hard_fail",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "jak_stat",
              "nfkb",
              "tnfa"
            ]
          },
          {
            "name": "cf_senescence_misattribution",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_senescence_signature_specifically"
            ]
          },
          {
            "name": "cf_dismiss_as_batch_artifact",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_batch_or_technical_artifact"
            ]
          },
          {
            "name": "cf_mapk_comparable_constitutive_kras_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "comparable_across_ages"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-luna-codex",
      "modelName": "gpt-5.6-luna",
      "harness": "openai-codex",
      "task": "gse297023_q3_age_retained_mapk_reduced",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 2,
      "passed": true,
      "score": null,
      "durationS": 988.300795,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "age_retained",
            "role": "gate",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_retained"
            ]
          },
          {
            "name": "identity_is_aging",
            "role": "gate",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_aging_signatures"
            ]
          },
          {
            "name": "nature_state_shift",
            "role": "gate",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_state_expression_shift"
            ]
          },
          {
            "name": "pathway_is_mapk",
            "role": "gate",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "mapk"
            ]
          },
          {
            "name": "pathway_reduced_in_aged",
            "role": "gate",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "reduced_in_aged"
            ]
          },
          {
            "name": "cf_mapk_increased_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_in_aged"
            ]
          },
          {
            "name": "cf_age_erased",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_erased"
            ]
          },
          {
            "name": "cf_pervasive_overcall",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_pervasive_artifactual"
            ]
          },
          {
            "name": "cf_explain_away_aging",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_compositional_or_identity",
            "role": "hard_fail",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_proportion_shift",
              "cell_identity_shift",
              "composition_dominant_state_secondary",
              "both_state_and_composition_equally"
            ]
          },
          {
            "name": "cf_biggest_inflammatory_signal_not_mapk",
            "role": "hard_fail",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "jak_stat",
              "nfkb",
              "tnfa"
            ]
          },
          {
            "name": "cf_senescence_misattribution",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_senescence_signature_specifically"
            ]
          },
          {
            "name": "cf_dismiss_as_batch_artifact",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_batch_or_technical_artifact"
            ]
          },
          {
            "name": "cf_mapk_comparable_constitutive_kras_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "comparable_across_ages"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-luna-codex",
      "modelName": "gpt-5.6-luna",
      "harness": "openai-codex",
      "task": "gse297023_q3_age_retained_mapk_reduced",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1123.357299,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "age_retained",
            "role": "gate",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_retained"
            ]
          },
          {
            "name": "identity_is_aging",
            "role": "gate",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_aging_signatures"
            ]
          },
          {
            "name": "nature_state_shift",
            "role": "gate",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_state_expression_shift"
            ]
          },
          {
            "name": "pathway_is_mapk",
            "role": "gate",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "mapk"
            ]
          },
          {
            "name": "pathway_reduced_in_aged",
            "role": "gate",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "reduced_in_aged"
            ]
          },
          {
            "name": "cf_mapk_increased_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_in_aged"
            ]
          },
          {
            "name": "cf_age_erased",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_erased"
            ]
          },
          {
            "name": "cf_pervasive_overcall",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_pervasive_artifactual"
            ]
          },
          {
            "name": "cf_explain_away_aging",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_compositional_or_identity",
            "role": "hard_fail",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_proportion_shift",
              "cell_identity_shift",
              "composition_dominant_state_secondary",
              "both_state_and_composition_equally"
            ]
          },
          {
            "name": "cf_biggest_inflammatory_signal_not_mapk",
            "role": "hard_fail",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "jak_stat",
              "nfkb",
              "tnfa"
            ]
          },
          {
            "name": "cf_senescence_misattribution",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_senescence_signature_specifically"
            ]
          },
          {
            "name": "cf_dismiss_as_batch_artifact",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_batch_or_technical_artifact"
            ]
          },
          {
            "name": "cf_mapk_comparable_constitutive_kras_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "comparable_across_ages"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-luna-codex",
      "modelName": "gpt-5.6-luna",
      "harness": "openai-codex",
      "task": "gse297023_q5_tme_age_dependent_remodeling",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 698.978082,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "perturbation_is_pten",
            "role": "gate",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "pten"
            ]
          },
          {
            "name": "tme_non_cell_autonomous",
            "role": "gate",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "substantial_non_cell_autonomous_remodeling"
            ]
          },
          {
            "name": "tme_state_not_composition",
            "role": "gate",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "tme_age_dependent",
            "role": "gate",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_dependent"
            ]
          },
          {
            "name": "tme_response_broader_in_aged",
            "role": "gate",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_aged"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "most_responsive_noncancer_cell_types",
            "kind": "list",
            "matchKey": "cell_type",
            "k": 2,
            "expected": [
              {
                "key": "alveolar_macrophage",
                "fields": {}
              },
              {
                "key": "endothelial",
                "fields": {}
              }
            ]
          },
          {
            "name": "cf_cell_intrinsic_only",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "negligible_cell_intrinsic_only_effect",
              "remodeling_unresolved",
              "modest_remodeling_few_cell_types"
            ]
          },
          {
            "name": "cf_remodeling_is_contamination",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "apparent_remodeling_reflects_cancer_contamination"
            ]
          },
          {
            "name": "cf_perturbation_design_misread",
            "role": "hard_fail",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "multiple_tumor_suppressors_pooled",
              "no_additional_gene_difference_kras_only",
              "rb1_cdkn2a_double",
              "pcna_essential_gene_control",
              "difference_is_oncogene_not_suppressor"
            ]
          },
          {
            "name": "cf_age_independent_tme",
            "role": "hard_fail",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_independent"
            ]
          },
          {
            "name": "cf_compositional_shift",
            "role": "hard_fail",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_frequency_shift",
              "compositional_immune_infiltration",
              "composition_dominant_state_secondary",
              "both_state_and_composition"
            ]
          },
          {
            "name": "cf_epithelial_or_cancer_substitution",
            "role": "hard_fail",
            "field": "$.most_responsive_noncancer_cell_types[*].cell_type",
            "kind": "avoid",
            "expected": [
              "AT2_cell",
              "AT1_cell",
              "cancer_cell",
              "ciliated",
              "secretory",
              "club_cell",
              "basal_cell",
              "goblet_cell"
            ]
          },
          {
            "name": "cf_tme_breadth_broader_in_young",
            "role": "hard_fail",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_young"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-luna-codex",
      "modelName": "gpt-5.6-luna",
      "harness": "openai-codex",
      "task": "gse297023_q5_tme_age_dependent_remodeling",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 1461.160287,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "perturbation_is_pten",
            "role": "gate",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "pten"
            ]
          },
          {
            "name": "tme_non_cell_autonomous",
            "role": "gate",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "substantial_non_cell_autonomous_remodeling"
            ]
          },
          {
            "name": "tme_state_not_composition",
            "role": "gate",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "tme_age_dependent",
            "role": "gate",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_dependent"
            ]
          },
          {
            "name": "tme_response_broader_in_aged",
            "role": "gate",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_aged"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "most_responsive_noncancer_cell_types",
            "kind": "list",
            "matchKey": "cell_type",
            "k": 2,
            "expected": [
              {
                "key": "alveolar_macrophage",
                "fields": {}
              },
              {
                "key": "endothelial",
                "fields": {}
              }
            ]
          },
          {
            "name": "cf_cell_intrinsic_only",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "negligible_cell_intrinsic_only_effect",
              "remodeling_unresolved",
              "modest_remodeling_few_cell_types"
            ]
          },
          {
            "name": "cf_remodeling_is_contamination",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "apparent_remodeling_reflects_cancer_contamination"
            ]
          },
          {
            "name": "cf_perturbation_design_misread",
            "role": "hard_fail",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "multiple_tumor_suppressors_pooled",
              "no_additional_gene_difference_kras_only",
              "rb1_cdkn2a_double",
              "pcna_essential_gene_control",
              "difference_is_oncogene_not_suppressor"
            ]
          },
          {
            "name": "cf_age_independent_tme",
            "role": "hard_fail",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_independent"
            ]
          },
          {
            "name": "cf_compositional_shift",
            "role": "hard_fail",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_frequency_shift",
              "compositional_immune_infiltration",
              "composition_dominant_state_secondary",
              "both_state_and_composition"
            ]
          },
          {
            "name": "cf_epithelial_or_cancer_substitution",
            "role": "hard_fail",
            "field": "$.most_responsive_noncancer_cell_types[*].cell_type",
            "kind": "avoid",
            "expected": [
              "AT2_cell",
              "AT1_cell",
              "cancer_cell",
              "ciliated",
              "secretory",
              "club_cell",
              "basal_cell",
              "goblet_cell"
            ]
          },
          {
            "name": "cf_tme_breadth_broader_in_young",
            "role": "hard_fail",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_young"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-luna-codex",
      "modelName": "gpt-5.6-luna",
      "harness": "openai-codex",
      "task": "gse297023_q5_tme_age_dependent_remodeling",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 812.105745,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "perturbation_is_pten",
            "role": "gate",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "pten"
            ]
          },
          {
            "name": "tme_non_cell_autonomous",
            "role": "gate",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "substantial_non_cell_autonomous_remodeling"
            ]
          },
          {
            "name": "tme_state_not_composition",
            "role": "gate",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "tme_age_dependent",
            "role": "gate",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_dependent"
            ]
          },
          {
            "name": "tme_response_broader_in_aged",
            "role": "gate",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_aged"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "most_responsive_noncancer_cell_types",
            "kind": "list",
            "matchKey": "cell_type",
            "k": 2,
            "expected": [
              {
                "key": "alveolar_macrophage",
                "fields": {}
              },
              {
                "key": "endothelial",
                "fields": {}
              }
            ]
          },
          {
            "name": "cf_cell_intrinsic_only",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "negligible_cell_intrinsic_only_effect",
              "remodeling_unresolved",
              "modest_remodeling_few_cell_types"
            ]
          },
          {
            "name": "cf_remodeling_is_contamination",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "apparent_remodeling_reflects_cancer_contamination"
            ]
          },
          {
            "name": "cf_perturbation_design_misread",
            "role": "hard_fail",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "multiple_tumor_suppressors_pooled",
              "no_additional_gene_difference_kras_only",
              "rb1_cdkn2a_double",
              "pcna_essential_gene_control",
              "difference_is_oncogene_not_suppressor"
            ]
          },
          {
            "name": "cf_age_independent_tme",
            "role": "hard_fail",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_independent"
            ]
          },
          {
            "name": "cf_compositional_shift",
            "role": "hard_fail",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_frequency_shift",
              "compositional_immune_infiltration",
              "composition_dominant_state_secondary",
              "both_state_and_composition"
            ]
          },
          {
            "name": "cf_epithelial_or_cancer_substitution",
            "role": "hard_fail",
            "field": "$.most_responsive_noncancer_cell_types[*].cell_type",
            "kind": "avoid",
            "expected": [
              "AT2_cell",
              "AT1_cell",
              "cancer_cell",
              "ciliated",
              "secretory",
              "club_cell",
              "basal_cell",
              "goblet_cell"
            ]
          },
          {
            "name": "cf_tme_breadth_broader_in_young",
            "role": "hard_fail",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_young"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-luna-codex",
      "modelName": "gpt-5.6-luna",
      "harness": "openai-codex",
      "task": "gse297023_q6_age_immune_infiltration",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 606.822055,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "composition_stable_with_age",
            "role": "gate",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "frequencies_largely_stable"
            ]
          },
          {
            "name": "age_signal_is_state_not_composition",
            "role": "gate",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "frequencies_uninformative_for_infiltration",
            "role": "gate",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_cannot_establish_infiltration"
            ]
          },
          {
            "name": "infiltration_increases_with_age",
            "role": "gate",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_with_age"
            ]
          },
          {
            "name": "infiltration_strength_calibrated",
            "role": "gate",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "significant_increase_strongest_markers_only",
              "consistent_increase_not_statistically_robust"
            ]
          },
          {
            "name": "genotype_marker_fraction_lower_in_pten",
            "role": "gate",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "lower_in_pten_deficient"
            ]
          },
          {
            "name": "cf_composition_shift_reading",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "major_compositional_shift_with_age",
              "increased_immune_fraction_with_age"
            ]
          },
          {
            "name": "cf_lineage_specific_composition_claim",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "myeloid_expansion_with_age",
              "lymphoid_expansion_with_age",
              "minor_shifts_in_some_cell_types"
            ]
          },
          {
            "name": "cf_immunosenescence_depletion_prior",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "immune_depletion_with_age"
            ]
          },
          {
            "name": "cf_age_modality_compositional",
            "role": "hard_fail",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_type_composition_shift",
              "both_equally",
              "composition_dominant_state_secondary"
            ]
          },
          {
            "name": "cf_overread_frequencies_as_infiltration",
            "role": "hard_fail",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_shows_no_infiltration_change",
              "single_cell_quantifies_infiltration",
              "frequencies_show_increased_infiltration",
              "frequencies_show_decreased_infiltration"
            ]
          },
          {
            "name": "cf_infiltration_decreases_immunosenescence",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "decreased_with_age"
            ]
          },
          {
            "name": "cf_infiltration_unchanged_scrna_overread",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "unchanged_with_age"
            ]
          },
          {
            "name": "cf_infiltration_indeterminate_unused_node",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "direction_indeterminate_from_data"
            ]
          },
          {
            "name": "cf_infiltration_mixed_direction",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "mixed_by_immune_cell_type"
            ]
          },
          {
            "name": "cf_infiltration_genotype_confusion",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_only_in_pten_deficient",
              "decreased_only_in_pten_deficient"
            ]
          },
          {
            "name": "cf_pten_infiltration_paper_text_recall",
            "role": "hard_fail",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "higher_in_pten_deficient",
              "higher_in_pten_deficient_only_in_young",
              "higher_in_pten_deficient_only_in_aged"
            ]
          },
          {
            "name": "cf_pseudoreplication_overclaim",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_increase_across_markers"
            ]
          },
          {
            "name": "cf_infiltration_strength_decrease_direction",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_decrease_across_markers",
              "significant_decrease_strongest_markers_only",
              "consistent_decrease_not_statistically_robust"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-luna-codex",
      "modelName": "gpt-5.6-luna",
      "harness": "openai-codex",
      "task": "gse297023_q6_age_immune_infiltration",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 916.10955,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "composition_stable_with_age",
            "role": "gate",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "frequencies_largely_stable"
            ]
          },
          {
            "name": "age_signal_is_state_not_composition",
            "role": "gate",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "frequencies_uninformative_for_infiltration",
            "role": "gate",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_cannot_establish_infiltration"
            ]
          },
          {
            "name": "infiltration_increases_with_age",
            "role": "gate",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_with_age"
            ]
          },
          {
            "name": "infiltration_strength_calibrated",
            "role": "gate",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "significant_increase_strongest_markers_only",
              "consistent_increase_not_statistically_robust"
            ]
          },
          {
            "name": "genotype_marker_fraction_lower_in_pten",
            "role": "gate",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "lower_in_pten_deficient"
            ]
          },
          {
            "name": "cf_composition_shift_reading",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "major_compositional_shift_with_age",
              "increased_immune_fraction_with_age"
            ]
          },
          {
            "name": "cf_lineage_specific_composition_claim",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "myeloid_expansion_with_age",
              "lymphoid_expansion_with_age",
              "minor_shifts_in_some_cell_types"
            ]
          },
          {
            "name": "cf_immunosenescence_depletion_prior",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "immune_depletion_with_age"
            ]
          },
          {
            "name": "cf_age_modality_compositional",
            "role": "hard_fail",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_type_composition_shift",
              "both_equally",
              "composition_dominant_state_secondary"
            ]
          },
          {
            "name": "cf_overread_frequencies_as_infiltration",
            "role": "hard_fail",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_shows_no_infiltration_change",
              "single_cell_quantifies_infiltration",
              "frequencies_show_increased_infiltration",
              "frequencies_show_decreased_infiltration"
            ]
          },
          {
            "name": "cf_infiltration_decreases_immunosenescence",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "decreased_with_age"
            ]
          },
          {
            "name": "cf_infiltration_unchanged_scrna_overread",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "unchanged_with_age"
            ]
          },
          {
            "name": "cf_infiltration_indeterminate_unused_node",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "direction_indeterminate_from_data"
            ]
          },
          {
            "name": "cf_infiltration_mixed_direction",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "mixed_by_immune_cell_type"
            ]
          },
          {
            "name": "cf_infiltration_genotype_confusion",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_only_in_pten_deficient",
              "decreased_only_in_pten_deficient"
            ]
          },
          {
            "name": "cf_pten_infiltration_paper_text_recall",
            "role": "hard_fail",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "higher_in_pten_deficient",
              "higher_in_pten_deficient_only_in_young",
              "higher_in_pten_deficient_only_in_aged"
            ]
          },
          {
            "name": "cf_pseudoreplication_overclaim",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_increase_across_markers"
            ]
          },
          {
            "name": "cf_infiltration_strength_decrease_direction",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_decrease_across_markers",
              "significant_decrease_strongest_markers_only",
              "consistent_decrease_not_statistically_robust"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-luna-codex",
      "modelName": "gpt-5.6-luna",
      "harness": "openai-codex",
      "task": "gse297023_q6_age_immune_infiltration",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 844.471273,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "composition_stable_with_age",
            "role": "gate",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "frequencies_largely_stable"
            ]
          },
          {
            "name": "age_signal_is_state_not_composition",
            "role": "gate",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "frequencies_uninformative_for_infiltration",
            "role": "gate",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_cannot_establish_infiltration"
            ]
          },
          {
            "name": "infiltration_increases_with_age",
            "role": "gate",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_with_age"
            ]
          },
          {
            "name": "infiltration_strength_calibrated",
            "role": "gate",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "significant_increase_strongest_markers_only",
              "consistent_increase_not_statistically_robust"
            ]
          },
          {
            "name": "genotype_marker_fraction_lower_in_pten",
            "role": "gate",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "lower_in_pten_deficient"
            ]
          },
          {
            "name": "cf_composition_shift_reading",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "major_compositional_shift_with_age",
              "increased_immune_fraction_with_age"
            ]
          },
          {
            "name": "cf_lineage_specific_composition_claim",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "myeloid_expansion_with_age",
              "lymphoid_expansion_with_age",
              "minor_shifts_in_some_cell_types"
            ]
          },
          {
            "name": "cf_immunosenescence_depletion_prior",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "immune_depletion_with_age"
            ]
          },
          {
            "name": "cf_age_modality_compositional",
            "role": "hard_fail",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_type_composition_shift",
              "both_equally",
              "composition_dominant_state_secondary"
            ]
          },
          {
            "name": "cf_overread_frequencies_as_infiltration",
            "role": "hard_fail",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_shows_no_infiltration_change",
              "single_cell_quantifies_infiltration",
              "frequencies_show_increased_infiltration",
              "frequencies_show_decreased_infiltration"
            ]
          },
          {
            "name": "cf_infiltration_decreases_immunosenescence",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "decreased_with_age"
            ]
          },
          {
            "name": "cf_infiltration_unchanged_scrna_overread",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "unchanged_with_age"
            ]
          },
          {
            "name": "cf_infiltration_indeterminate_unused_node",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "direction_indeterminate_from_data"
            ]
          },
          {
            "name": "cf_infiltration_mixed_direction",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "mixed_by_immune_cell_type"
            ]
          },
          {
            "name": "cf_infiltration_genotype_confusion",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_only_in_pten_deficient",
              "decreased_only_in_pten_deficient"
            ]
          },
          {
            "name": "cf_pten_infiltration_paper_text_recall",
            "role": "hard_fail",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "higher_in_pten_deficient",
              "higher_in_pten_deficient_only_in_young",
              "higher_in_pten_deficient_only_in_aged"
            ]
          },
          {
            "name": "cf_pseudoreplication_overclaim",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_increase_across_markers"
            ]
          },
          {
            "name": "cf_infiltration_strength_decrease_direction",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_decrease_across_markers",
              "significant_decrease_strongest_markers_only",
              "consistent_decrease_not_statistically_robust"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-sol-codex",
      "modelName": "gpt-5.6-sol",
      "harness": "openai-codex",
      "task": "gse155381_q4_pseudotime_delay",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 1,
      "passed": true,
      "score": null,
      "durationS": 844.790801,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "pseudotime_direction_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "E17_chronological_match_weighted",
            "role": "additive",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pluripotency_exit_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pseudotime_direction_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_synchronous_with_monkey_EPI",
              "chimera_human_EPI_developmentally_advanced",
              "chimera_human_EPI_delayed_at_pre_implantation_only_resolved_post_implantation"
            ]
          },
          {
            "name": "E17_chronological_match_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "resembles_monkey_E17_Gast_EPI",
              "resembles_chimera_monkey_E17_EPI_specifically",
              "chimera_human_E17_EPI_more_advanced_than_monkey_E17_Gast"
            ]
          },
          {
            "name": "pluripotency_exit_paper_inverted_or_prior_driven",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_advances_pluripotency_exit_prematurely",
              "naive_pluripotency_program_specifically_engaged",
              "pluripotency_core_retained_via_KLF17_naive_marker_specifically"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-sol-codex",
      "modelName": "gpt-5.6-sol",
      "harness": "openai-codex",
      "task": "gse155381_q4_pseudotime_delay",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 794.509336,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "pseudotime_direction_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "E17_chronological_match_weighted",
            "role": "additive",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pluripotency_exit_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pseudotime_direction_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_synchronous_with_monkey_EPI",
              "chimera_human_EPI_developmentally_advanced",
              "chimera_human_EPI_delayed_at_pre_implantation_only_resolved_post_implantation"
            ]
          },
          {
            "name": "E17_chronological_match_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "resembles_monkey_E17_Gast_EPI",
              "resembles_chimera_monkey_E17_EPI_specifically",
              "chimera_human_E17_EPI_more_advanced_than_monkey_E17_Gast"
            ]
          },
          {
            "name": "pluripotency_exit_paper_inverted_or_prior_driven",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_advances_pluripotency_exit_prematurely",
              "naive_pluripotency_program_specifically_engaged",
              "pluripotency_core_retained_via_KLF17_naive_marker_specifically"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-sol-codex",
      "modelName": "gpt-5.6-sol",
      "harness": "openai-codex",
      "task": "gse155381_q4_pseudotime_delay",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 702.298171,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "pseudotime_direction_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "E17_chronological_match_weighted",
            "role": "additive",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pluripotency_exit_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pseudotime_direction_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_synchronous_with_monkey_EPI",
              "chimera_human_EPI_developmentally_advanced",
              "chimera_human_EPI_delayed_at_pre_implantation_only_resolved_post_implantation"
            ]
          },
          {
            "name": "E17_chronological_match_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "resembles_monkey_E17_Gast_EPI",
              "resembles_chimera_monkey_E17_EPI_specifically",
              "chimera_human_E17_EPI_more_advanced_than_monkey_E17_Gast"
            ]
          },
          {
            "name": "pluripotency_exit_paper_inverted_or_prior_driven",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_advances_pluripotency_exit_prematurely",
              "naive_pluripotency_program_specifically_engaged",
              "pluripotency_core_retained_via_KLF17_naive_marker_specifically"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-sol-codex",
      "modelName": "gpt-5.6-sol",
      "harness": "openai-codex",
      "task": "gse155381_q5_chimera_monkey_perturbation",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 554.196549,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "deg_direction_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "comparison_subset_choice_weighted",
            "role": "additive",
            "field": "$.comparison_subset_choice",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "panel_resolved_strong_target_ccn1_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN1",
              "CYR61"
            ]
          },
          {
            "name": "panel_resolved_strong_target_ccn2_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN2",
              "CTGF"
            ]
          },
          {
            "name": "hippo_pathway_weighted",
            "role": "additive",
            "field": "$.hippo_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "tgf_beta_pathway_weighted",
            "role": "additive",
            "field": "$.tgf_beta_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "deg_direction_batch_effect_uncorrected",
            "role": "hard_fail",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_dominated_global",
              "up_dominated_in_chosen_subset"
            ]
          },
          {
            "name": "silent_panel_dropout_paired_with_directional_pathway_claim",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "hippo_pathway_direction": {
                "op": "in",
                "values": [
                  "hippo_pathway_upregulated",
                  "hippo_pathway_unchanged",
                  "wnt_pathway_specifically_perturbed",
                  "stress_response_pathway_dominantly_engaged",
                  "no_pathway_signal_above_confound",
                  "yap_tead_direct_target_panel_downregulated",
                  "yap_tead_panel_dependent_direction",
                  "hippo_pathway_downregulated"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-sol-codex",
      "modelName": "gpt-5.6-sol",
      "harness": "openai-codex",
      "task": "gse155381_q5_chimera_monkey_perturbation",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 588.668068,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "deg_direction_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "comparison_subset_choice_weighted",
            "role": "additive",
            "field": "$.comparison_subset_choice",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "panel_resolved_strong_target_ccn1_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN1",
              "CYR61"
            ]
          },
          {
            "name": "panel_resolved_strong_target_ccn2_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN2",
              "CTGF"
            ]
          },
          {
            "name": "hippo_pathway_weighted",
            "role": "additive",
            "field": "$.hippo_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "tgf_beta_pathway_weighted",
            "role": "additive",
            "field": "$.tgf_beta_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "deg_direction_batch_effect_uncorrected",
            "role": "hard_fail",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_dominated_global",
              "up_dominated_in_chosen_subset"
            ]
          },
          {
            "name": "silent_panel_dropout_paired_with_directional_pathway_claim",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "hippo_pathway_direction": {
                "op": "in",
                "values": [
                  "hippo_pathway_upregulated",
                  "hippo_pathway_unchanged",
                  "wnt_pathway_specifically_perturbed",
                  "stress_response_pathway_dominantly_engaged",
                  "no_pathway_signal_above_confound",
                  "yap_tead_direct_target_panel_downregulated",
                  "yap_tead_panel_dependent_direction",
                  "hippo_pathway_downregulated"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-sol-codex",
      "modelName": "gpt-5.6-sol",
      "harness": "openai-codex",
      "task": "gse155381_q5_chimera_monkey_perturbation",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 599.997295,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "deg_direction_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "comparison_subset_choice_weighted",
            "role": "additive",
            "field": "$.comparison_subset_choice",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "panel_resolved_strong_target_ccn1_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN1",
              "CYR61"
            ]
          },
          {
            "name": "panel_resolved_strong_target_ccn2_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN2",
              "CTGF"
            ]
          },
          {
            "name": "hippo_pathway_weighted",
            "role": "additive",
            "field": "$.hippo_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "tgf_beta_pathway_weighted",
            "role": "additive",
            "field": "$.tgf_beta_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "deg_direction_batch_effect_uncorrected",
            "role": "hard_fail",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_dominated_global",
              "up_dominated_in_chosen_subset"
            ]
          },
          {
            "name": "silent_panel_dropout_paired_with_directional_pathway_claim",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "hippo_pathway_direction": {
                "op": "in",
                "values": [
                  "hippo_pathway_upregulated",
                  "hippo_pathway_unchanged",
                  "wnt_pathway_specifically_perturbed",
                  "stress_response_pathway_dominantly_engaged",
                  "no_pathway_signal_above_confound",
                  "yap_tead_direct_target_panel_downregulated",
                  "yap_tead_panel_dependent_direction",
                  "hippo_pathway_downregulated"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-sol-codex",
      "modelName": "gpt-5.6-sol",
      "harness": "openai-codex",
      "task": "gse155381_q7_xspecies_epi_communication",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 828.760798,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "richness_weighted",
            "role": "additive",
            "field": "$.xspecies_signaling_richness",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_weighted",
            "role": "additive",
            "field": "$.dominant_signaling_axis",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_wnt_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "WNT_FZD_LRP_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_fgf_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "FGF_FGFR_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_notch_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "NOTCH_DLL_JAG_family"
                ]
              }
            }
          },
          {
            "name": "signaling_directionality_weighted",
            "role": "additive",
            "field": "$.signaling_directionality",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "top_signal_interaction_class_weighted",
            "role": "additive",
            "field": "$.top_signal_interaction_class",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-sol-codex",
      "modelName": "gpt-5.6-sol",
      "harness": "openai-codex",
      "task": "gse155381_q7_xspecies_epi_communication",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 845.330381,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "richness_weighted",
            "role": "additive",
            "field": "$.xspecies_signaling_richness",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_weighted",
            "role": "additive",
            "field": "$.dominant_signaling_axis",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_wnt_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "WNT_FZD_LRP_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_fgf_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "FGF_FGFR_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_notch_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "NOTCH_DLL_JAG_family"
                ]
              }
            }
          },
          {
            "name": "signaling_directionality_weighted",
            "role": "additive",
            "field": "$.signaling_directionality",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "top_signal_interaction_class_weighted",
            "role": "additive",
            "field": "$.top_signal_interaction_class",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-sol-codex",
      "modelName": "gpt-5.6-sol",
      "harness": "openai-codex",
      "task": "gse155381_q7_xspecies_epi_communication",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 988.661001,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "richness_weighted",
            "role": "additive",
            "field": "$.xspecies_signaling_richness",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_weighted",
            "role": "additive",
            "field": "$.dominant_signaling_axis",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_wnt_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "WNT_FZD_LRP_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_fgf_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "FGF_FGFR_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_notch_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "NOTCH_DLL_JAG_family"
                ]
              }
            }
          },
          {
            "name": "signaling_directionality_weighted",
            "role": "additive",
            "field": "$.signaling_directionality",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "top_signal_interaction_class_weighted",
            "role": "additive",
            "field": "$.top_signal_interaction_class",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-sol-codex",
      "modelName": "gpt-5.6-sol",
      "harness": "openai-codex",
      "task": "gse155381_q8_bidirectional_convergence",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 947.770543,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "chimera_human_identity_shift_target_weighted",
            "role": "additive",
            "field": "$.chimera_human_identity_shift_target",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_in_vivo_state_assignment_weighted",
            "role": "additive",
            "field": "$.chimera_human_in_vivo_state_assignment",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_shift_magnitude_weighted",
            "role": "additive",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_monkey_identity_shift_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "both_arms_null_contradicted_by_calibration",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "chimera_human_identity_shift_target": {
                "op": "equals",
                "values": [
                  "chimera_human_maintains_origin_identity_no_measurable_shift"
                ]
              },
              "chimera_monkey_identity_shift": {
                "op": "equals",
                "values": [
                  "chimera_monkey_unchanged_relative_to_control"
                ]
              }
            }
          },
          {
            "name": "chimera_monkey_paper_claim_on_confounded_input",
            "role": "hard_fail",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "value",
            "expected": "chimera_monkey_shifts_toward_chimera_human_identity"
          },
          {
            "name": "chimera_human_magnitude_wrong_attribution_signature",
            "role": "hard_fail",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "value",
            "op": "in",
            "expected": [
              "dramatic_complete_replacement_of_origin_identity",
              "magnitude_dominated_by_culture_or_metabolic_axis",
              "magnitude_dominated_by_developmental_stage_axis",
              "magnitude_dominated_by_cell_cycle_axis",
              "magnitude_dominated_by_batch_effect_axis"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-sol-codex",
      "modelName": "gpt-5.6-sol",
      "harness": "openai-codex",
      "task": "gse155381_q8_bidirectional_convergence",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 647.028952,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "chimera_human_identity_shift_target_weighted",
            "role": "additive",
            "field": "$.chimera_human_identity_shift_target",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_in_vivo_state_assignment_weighted",
            "role": "additive",
            "field": "$.chimera_human_in_vivo_state_assignment",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_shift_magnitude_weighted",
            "role": "additive",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_monkey_identity_shift_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "both_arms_null_contradicted_by_calibration",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "chimera_human_identity_shift_target": {
                "op": "equals",
                "values": [
                  "chimera_human_maintains_origin_identity_no_measurable_shift"
                ]
              },
              "chimera_monkey_identity_shift": {
                "op": "equals",
                "values": [
                  "chimera_monkey_unchanged_relative_to_control"
                ]
              }
            }
          },
          {
            "name": "chimera_monkey_paper_claim_on_confounded_input",
            "role": "hard_fail",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "value",
            "expected": "chimera_monkey_shifts_toward_chimera_human_identity"
          },
          {
            "name": "chimera_human_magnitude_wrong_attribution_signature",
            "role": "hard_fail",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "value",
            "op": "in",
            "expected": [
              "dramatic_complete_replacement_of_origin_identity",
              "magnitude_dominated_by_culture_or_metabolic_axis",
              "magnitude_dominated_by_developmental_stage_axis",
              "magnitude_dominated_by_cell_cycle_axis",
              "magnitude_dominated_by_batch_effect_axis"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-sol-codex",
      "modelName": "gpt-5.6-sol",
      "harness": "openai-codex",
      "task": "gse155381_q8_bidirectional_convergence",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 860.100175,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "chimera_human_identity_shift_target_weighted",
            "role": "additive",
            "field": "$.chimera_human_identity_shift_target",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_in_vivo_state_assignment_weighted",
            "role": "additive",
            "field": "$.chimera_human_in_vivo_state_assignment",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_shift_magnitude_weighted",
            "role": "additive",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_monkey_identity_shift_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "both_arms_null_contradicted_by_calibration",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "chimera_human_identity_shift_target": {
                "op": "equals",
                "values": [
                  "chimera_human_maintains_origin_identity_no_measurable_shift"
                ]
              },
              "chimera_monkey_identity_shift": {
                "op": "equals",
                "values": [
                  "chimera_monkey_unchanged_relative_to_control"
                ]
              }
            }
          },
          {
            "name": "chimera_monkey_paper_claim_on_confounded_input",
            "role": "hard_fail",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "value",
            "expected": "chimera_monkey_shifts_toward_chimera_human_identity"
          },
          {
            "name": "chimera_human_magnitude_wrong_attribution_signature",
            "role": "hard_fail",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "value",
            "op": "in",
            "expected": [
              "dramatic_complete_replacement_of_origin_identity",
              "magnitude_dominated_by_culture_or_metabolic_axis",
              "magnitude_dominated_by_developmental_stage_axis",
              "magnitude_dominated_by_cell_cycle_axis",
              "magnitude_dominated_by_batch_effect_axis"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-sol-codex",
      "modelName": "gpt-5.6-sol",
      "harness": "openai-codex",
      "task": "gse171524_ectopic_airway_rare_cell",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 1168.069658,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "focus_population_identity_gate",
            "role": "gate",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "chemosensory_marker_expressing_population_alpha"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "focus_population_marker_calls",
            "kind": "list",
            "matchKey": "marker",
            "k": 3,
            "expected": [
              {
                "key": "POU2F3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "CHAT",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "TRPM5",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "LRMP",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNAT3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "AVIL",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNG13",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              }
            ]
          },
          {
            "name": "focus_population_condition_distribution_gate",
            "role": "gate",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_enriched_with_substantial_control_presence",
              "covid_predominant",
              "covid_only",
              "covid_enriched_trend_not_significant"
            ]
          },
          {
            "name": "focus_population_location_gate",
            "role": "gate",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "ectopic_parenchyma_in_covid",
              "parenchyma_only",
              "bronchiolized_or_peribronchial_parenchyma",
              "airway_and_parenchyma"
            ]
          },
          {
            "name": "focus_population_recovery_mode_gate",
            "role": "gate",
            "field": "$.focus_population_recovery_mode",
            "kind": "value",
            "op": "in",
            "expected": [
              "continuum_not_discrete_cluster"
            ]
          },
          {
            "name": "focus_population_identity_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "ionocyte_like_beta",
              "neuroendocrine_like_gamma",
              "basal_stem_progenitor_like_delta",
              "aberrant_basaloid_transitional_like_epsilon",
              "neuronal_or_neural_gene_minor_population_zeta",
              "deuterosomal_ciliated_progenitor_like_eta",
              "hillock_transitional_like_theta",
              "low_quality_doublet_or_debris",
              "no_rare_cell_recovered",
              "not_assessed"
            ]
          },
          {
            "name": "cf_wrong_lineage_marker_called_positive",
            "role": "hard_fail",
            "field": "$.focus_population_marker_calls[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "focus_population_condition_distribution_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "equally_distributed",
              "control_predominant",
              "not_assessed"
            ]
          },
          {
            "name": "focus_population_location_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "airway_only",
              "not_assessed"
            ]
          },
          {
            "name": "cross_axis_canonical_identity_with_airway_only_location_incoherence",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "focus_population_identity": {
                "op": "in",
                "values": [
                  "chemosensory_marker_expressing_population_alpha"
                ]
              },
              "focus_population_location": {
                "op": "in",
                "values": [
                  "airway_only"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-sol-codex",
      "modelName": "gpt-5.6-sol",
      "harness": "openai-codex",
      "task": "gse171524_ectopic_airway_rare_cell",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": null,
      "cost": null,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "focus_population_identity_gate",
            "role": "gate",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "chemosensory_marker_expressing_population_alpha"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "focus_population_marker_calls",
            "kind": "list",
            "matchKey": "marker",
            "k": 3,
            "expected": [
              {
                "key": "POU2F3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "CHAT",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "TRPM5",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "LRMP",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNAT3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "AVIL",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNG13",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              }
            ]
          },
          {
            "name": "focus_population_condition_distribution_gate",
            "role": "gate",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_enriched_with_substantial_control_presence",
              "covid_predominant",
              "covid_only",
              "covid_enriched_trend_not_significant"
            ]
          },
          {
            "name": "focus_population_location_gate",
            "role": "gate",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "ectopic_parenchyma_in_covid",
              "parenchyma_only",
              "bronchiolized_or_peribronchial_parenchyma",
              "airway_and_parenchyma"
            ]
          },
          {
            "name": "focus_population_recovery_mode_gate",
            "role": "gate",
            "field": "$.focus_population_recovery_mode",
            "kind": "value",
            "op": "in",
            "expected": [
              "continuum_not_discrete_cluster"
            ]
          },
          {
            "name": "focus_population_identity_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "ionocyte_like_beta",
              "neuroendocrine_like_gamma",
              "basal_stem_progenitor_like_delta",
              "aberrant_basaloid_transitional_like_epsilon",
              "neuronal_or_neural_gene_minor_population_zeta",
              "deuterosomal_ciliated_progenitor_like_eta",
              "hillock_transitional_like_theta",
              "low_quality_doublet_or_debris",
              "no_rare_cell_recovered",
              "not_assessed"
            ]
          },
          {
            "name": "cf_wrong_lineage_marker_called_positive",
            "role": "hard_fail",
            "field": "$.focus_population_marker_calls[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "focus_population_condition_distribution_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "equally_distributed",
              "control_predominant",
              "not_assessed"
            ]
          },
          {
            "name": "focus_population_location_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "airway_only",
              "not_assessed"
            ]
          },
          {
            "name": "cross_axis_canonical_identity_with_airway_only_location_incoherence",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "focus_population_identity": {
                "op": "in",
                "values": [
                  "chemosensory_marker_expressing_population_alpha"
                ]
              },
              "focus_population_location": {
                "op": "in",
                "values": [
                  "airway_only"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-sol-codex",
      "modelName": "gpt-5.6-sol",
      "harness": "openai-codex",
      "task": "gse171524_ectopic_airway_rare_cell",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": null,
      "cost": null,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "focus_population_identity_gate",
            "role": "gate",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "chemosensory_marker_expressing_population_alpha"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "focus_population_marker_calls",
            "kind": "list",
            "matchKey": "marker",
            "k": 3,
            "expected": [
              {
                "key": "POU2F3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "CHAT",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "TRPM5",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "LRMP",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNAT3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "AVIL",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNG13",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              }
            ]
          },
          {
            "name": "focus_population_condition_distribution_gate",
            "role": "gate",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_enriched_with_substantial_control_presence",
              "covid_predominant",
              "covid_only",
              "covid_enriched_trend_not_significant"
            ]
          },
          {
            "name": "focus_population_location_gate",
            "role": "gate",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "ectopic_parenchyma_in_covid",
              "parenchyma_only",
              "bronchiolized_or_peribronchial_parenchyma",
              "airway_and_parenchyma"
            ]
          },
          {
            "name": "focus_population_recovery_mode_gate",
            "role": "gate",
            "field": "$.focus_population_recovery_mode",
            "kind": "value",
            "op": "in",
            "expected": [
              "continuum_not_discrete_cluster"
            ]
          },
          {
            "name": "focus_population_identity_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "ionocyte_like_beta",
              "neuroendocrine_like_gamma",
              "basal_stem_progenitor_like_delta",
              "aberrant_basaloid_transitional_like_epsilon",
              "neuronal_or_neural_gene_minor_population_zeta",
              "deuterosomal_ciliated_progenitor_like_eta",
              "hillock_transitional_like_theta",
              "low_quality_doublet_or_debris",
              "no_rare_cell_recovered",
              "not_assessed"
            ]
          },
          {
            "name": "cf_wrong_lineage_marker_called_positive",
            "role": "hard_fail",
            "field": "$.focus_population_marker_calls[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "focus_population_condition_distribution_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "equally_distributed",
              "control_predominant",
              "not_assessed"
            ]
          },
          {
            "name": "focus_population_location_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "airway_only",
              "not_assessed"
            ]
          },
          {
            "name": "cross_axis_canonical_identity_with_airway_only_location_incoherence",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "focus_population_identity": {
                "op": "in",
                "values": [
                  "chemosensory_marker_expressing_population_alpha"
                ]
              },
              "focus_population_location": {
                "op": "in",
                "values": [
                  "airway_only"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-sol-codex",
      "modelName": "gpt-5.6-sol",
      "harness": "openai-codex",
      "task": "gse171524_pathological_fibroblast_master_regulator_druggable_targets",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 1,
      "passed": true,
      "score": null,
      "durationS": null,
      "cost": null,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "disease_associated_fibroblast_subtype_weighted",
            "role": "additive",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "mr_tf_junb_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_alpha"
            ]
          },
          {
            "name": "mr_tf_jund_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_beta"
            ]
          },
          {
            "name": "drug_mmp14_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_alpha"
            ]
          },
          {
            "name": "drug_stat3_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_beta"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_weighted",
            "role": "additive",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "disease_associated_fibroblast_subtype_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "value",
            "op": "in",
            "expected": [
              "subtype_lambda",
              "subtype_epsilon",
              "subtype_iota",
              "subtype_beta",
              "subtype_eta",
              "subtype_zeta",
              "subtype_not_assessed"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "simple_de_ranking_by_log_fc",
              "marker_based_de_only",
              "not_computed"
            ]
          },
          {
            "name": "top_master_regulator_transcription_factors_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_not_computed"
            ]
          },
          {
            "name": "top_druggable_proteins_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_not_assessed"
            ]
          },
          {
            "name": "cross_axis_circular_marker_in_druggable_proteins",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_gamma",
              "drug_target_delta",
              "drug_target_eta"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-sol-codex",
      "modelName": "gpt-5.6-sol",
      "harness": "openai-codex",
      "task": "gse171524_pathological_fibroblast_master_regulator_druggable_targets",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 115.086224,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "disease_associated_fibroblast_subtype_weighted",
            "role": "additive",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "mr_tf_junb_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_alpha"
            ]
          },
          {
            "name": "mr_tf_jund_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_beta"
            ]
          },
          {
            "name": "drug_mmp14_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_alpha"
            ]
          },
          {
            "name": "drug_stat3_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_beta"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_weighted",
            "role": "additive",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "disease_associated_fibroblast_subtype_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "value",
            "op": "in",
            "expected": [
              "subtype_lambda",
              "subtype_epsilon",
              "subtype_iota",
              "subtype_beta",
              "subtype_eta",
              "subtype_zeta",
              "subtype_not_assessed"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "simple_de_ranking_by_log_fc",
              "marker_based_de_only",
              "not_computed"
            ]
          },
          {
            "name": "top_master_regulator_transcription_factors_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_not_computed"
            ]
          },
          {
            "name": "top_druggable_proteins_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_not_assessed"
            ]
          },
          {
            "name": "cross_axis_circular_marker_in_druggable_proteins",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_gamma",
              "drug_target_delta",
              "drug_target_eta"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-sol-codex",
      "modelName": "gpt-5.6-sol",
      "harness": "openai-codex",
      "task": "gse171524_pathological_fibroblast_master_regulator_druggable_targets",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": null,
      "cost": null,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "disease_associated_fibroblast_subtype_weighted",
            "role": "additive",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "mr_tf_junb_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_alpha"
            ]
          },
          {
            "name": "mr_tf_jund_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_beta"
            ]
          },
          {
            "name": "drug_mmp14_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_alpha"
            ]
          },
          {
            "name": "drug_stat3_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_beta"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_weighted",
            "role": "additive",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "disease_associated_fibroblast_subtype_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "value",
            "op": "in",
            "expected": [
              "subtype_lambda",
              "subtype_epsilon",
              "subtype_iota",
              "subtype_beta",
              "subtype_eta",
              "subtype_zeta",
              "subtype_not_assessed"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "simple_de_ranking_by_log_fc",
              "marker_based_de_only",
              "not_computed"
            ]
          },
          {
            "name": "top_master_regulator_transcription_factors_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_not_computed"
            ]
          },
          {
            "name": "top_druggable_proteins_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_not_assessed"
            ]
          },
          {
            "name": "cross_axis_circular_marker_in_druggable_proteins",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_gamma",
              "drug_target_delta",
              "drug_target_eta"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-sol-codex",
      "modelName": "gpt-5.6-sol",
      "harness": "openai-codex",
      "task": "gse171524_plasma_bcr_neutralizing_clonotype",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 1,
      "passed": true,
      "score": null,
      "durationS": null,
      "cost": null,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dominant_pairing_heavy_gate",
            "role": "gate",
            "field": "$.dominant_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV4-34"
            ]
          },
          {
            "name": "dominant_pairing_light_gate",
            "role": "gate",
            "field": "$.dominant_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGLV3-19"
            ]
          },
          {
            "name": "dominant_neutralizer_match_gate",
            "role": "gate",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_known_neutralizer_match"
            ]
          },
          {
            "name": "neutralizer_pairing_heavy_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV1-18"
            ]
          },
          {
            "name": "neutralizer_pairing_light_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGKV3-20"
            ]
          },
          {
            "name": "neutralizer_recurrence_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "observed_in_2_to_5_donors"
            ]
          },
          {
            "name": "neutralizer_rank_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "top_5_pairing",
              "top_6_to_20_pairing"
            ]
          },
          {
            "name": "dominant_clone_isotype_gate",
            "role": "gate",
            "field": "$.dominant_clone_isotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "IgG"
            ]
          },
          {
            "name": "blineage_class_switch_contrast_gate",
            "role": "gate",
            "field": "$.blineage_class_switch_contrast",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_class_switched_control_higher_unswitched",
              "class_switched_in_both_cohorts"
            ]
          },
          {
            "name": "dominant_match_narrative_trap_or_null",
            "role": "hard_fail",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_rbd_neutralizer",
              "matches_known_autoreactive_clone",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_heavy",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_light",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_recurrence_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_rank_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "cross_axis_found_dominant_but_failed_to_scan_for_embedded_neutralizer",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_pairing_neutralizer_match": {
                "op": "in",
                "values": [
                  "no_known_neutralizer_match"
                ]
              },
              "neutralizer_pairing_heavy": {
                "op": "in",
                "values": [
                  "none_detected"
                ]
              }
            }
          },
          {
            "name": "neutralizer_pairing_not_exactly_the_s309_pair",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "op": "not",
            "expected": {
              "op": "and",
              "args": [
                {
                  "op": "field",
                  "name": "neutralizer_pairing_heavy",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGHV1-18"
                    ]
                  }
                },
                {
                  "op": "field",
                  "name": "neutralizer_pairing_light",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGKV3-20"
                    ]
                  }
                }
              ]
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-sol-codex",
      "modelName": "gpt-5.6-sol",
      "harness": "openai-codex",
      "task": "gse171524_plasma_bcr_neutralizing_clonotype",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": null,
      "cost": null,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dominant_pairing_heavy_gate",
            "role": "gate",
            "field": "$.dominant_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV4-34"
            ]
          },
          {
            "name": "dominant_pairing_light_gate",
            "role": "gate",
            "field": "$.dominant_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGLV3-19"
            ]
          },
          {
            "name": "dominant_neutralizer_match_gate",
            "role": "gate",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_known_neutralizer_match"
            ]
          },
          {
            "name": "neutralizer_pairing_heavy_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV1-18"
            ]
          },
          {
            "name": "neutralizer_pairing_light_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGKV3-20"
            ]
          },
          {
            "name": "neutralizer_recurrence_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "observed_in_2_to_5_donors"
            ]
          },
          {
            "name": "neutralizer_rank_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "top_5_pairing",
              "top_6_to_20_pairing"
            ]
          },
          {
            "name": "dominant_clone_isotype_gate",
            "role": "gate",
            "field": "$.dominant_clone_isotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "IgG"
            ]
          },
          {
            "name": "blineage_class_switch_contrast_gate",
            "role": "gate",
            "field": "$.blineage_class_switch_contrast",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_class_switched_control_higher_unswitched",
              "class_switched_in_both_cohorts"
            ]
          },
          {
            "name": "dominant_match_narrative_trap_or_null",
            "role": "hard_fail",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_rbd_neutralizer",
              "matches_known_autoreactive_clone",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_heavy",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_light",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_recurrence_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_rank_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "cross_axis_found_dominant_but_failed_to_scan_for_embedded_neutralizer",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_pairing_neutralizer_match": {
                "op": "in",
                "values": [
                  "no_known_neutralizer_match"
                ]
              },
              "neutralizer_pairing_heavy": {
                "op": "in",
                "values": [
                  "none_detected"
                ]
              }
            }
          },
          {
            "name": "neutralizer_pairing_not_exactly_the_s309_pair",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "op": "not",
            "expected": {
              "op": "and",
              "args": [
                {
                  "op": "field",
                  "name": "neutralizer_pairing_heavy",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGHV1-18"
                    ]
                  }
                },
                {
                  "op": "field",
                  "name": "neutralizer_pairing_light",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGKV3-20"
                    ]
                  }
                }
              ]
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-sol-codex",
      "modelName": "gpt-5.6-sol",
      "harness": "openai-codex",
      "task": "gse171524_plasma_bcr_neutralizing_clonotype",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 391.485463,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dominant_pairing_heavy_gate",
            "role": "gate",
            "field": "$.dominant_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV4-34"
            ]
          },
          {
            "name": "dominant_pairing_light_gate",
            "role": "gate",
            "field": "$.dominant_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGLV3-19"
            ]
          },
          {
            "name": "dominant_neutralizer_match_gate",
            "role": "gate",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_known_neutralizer_match"
            ]
          },
          {
            "name": "neutralizer_pairing_heavy_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV1-18"
            ]
          },
          {
            "name": "neutralizer_pairing_light_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGKV3-20"
            ]
          },
          {
            "name": "neutralizer_recurrence_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "observed_in_2_to_5_donors"
            ]
          },
          {
            "name": "neutralizer_rank_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "top_5_pairing",
              "top_6_to_20_pairing"
            ]
          },
          {
            "name": "dominant_clone_isotype_gate",
            "role": "gate",
            "field": "$.dominant_clone_isotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "IgG"
            ]
          },
          {
            "name": "blineage_class_switch_contrast_gate",
            "role": "gate",
            "field": "$.blineage_class_switch_contrast",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_class_switched_control_higher_unswitched",
              "class_switched_in_both_cohorts"
            ]
          },
          {
            "name": "dominant_match_narrative_trap_or_null",
            "role": "hard_fail",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_rbd_neutralizer",
              "matches_known_autoreactive_clone",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_heavy",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_light",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_recurrence_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_rank_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "cross_axis_found_dominant_but_failed_to_scan_for_embedded_neutralizer",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_pairing_neutralizer_match": {
                "op": "in",
                "values": [
                  "no_known_neutralizer_match"
                ]
              },
              "neutralizer_pairing_heavy": {
                "op": "in",
                "values": [
                  "none_detected"
                ]
              }
            }
          },
          {
            "name": "neutralizer_pairing_not_exactly_the_s309_pair",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "op": "not",
            "expected": {
              "op": "and",
              "args": [
                {
                  "op": "field",
                  "name": "neutralizer_pairing_heavy",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGHV1-18"
                    ]
                  }
                },
                {
                  "op": "field",
                  "name": "neutralizer_pairing_light",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGKV3-20"
                    ]
                  }
                }
              ]
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-sol-codex",
      "modelName": "gpt-5.6-sol",
      "harness": "openai-codex",
      "task": "q6_myeloid_subset_restricted_de",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 1,
      "passed": true,
      "score": null,
      "durationS": null,
      "cost": null,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "myeloid_subset_with_strongest_change_gate",
            "role": "gate",
            "field": "$.myeloid_subset_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "subset_gamma"
            ]
          },
          {
            "name": "gene_with_strongest_change_gate",
            "role": "gate",
            "field": "$.gene_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "AXL",
              "PROS1"
            ]
          },
          {
            "name": "direction_of_change_gate",
            "role": "gate",
            "field": "$.direction_of_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "down_in_covid"
            ]
          },
          {
            "name": "biological_function_interpretation_gate",
            "role": "gate",
            "field": "$.biological_function_interpretation",
            "kind": "value",
            "op": "in",
            "expected": [
              "efferocytosis",
              "phagocytosis"
            ]
          },
          {
            "name": "myeloid_compartment_abundance_gate",
            "role": "gate",
            "field": "$.myeloid_compartment_abundance",
            "kind": "value",
            "op": "in",
            "expected": [
              "expands_in_covid"
            ]
          },
          {
            "name": "myeloid_expansion_localization_gate",
            "role": "gate",
            "field": "$.myeloid_expansion_localization",
            "kind": "value",
            "op": "in",
            "expected": [
              "recruited_driven_resident_dysfunction"
            ]
          },
          {
            "name": "dominant_signature_gate",
            "role": "gate",
            "field": "$.dominant_signature",
            "kind": "value",
            "op": "in",
            "expected": [
              "technical_or_agonal_artifact"
            ]
          },
          {
            "name": "dominant_signature_direction_gate",
            "role": "gate",
            "field": "$.dominant_signature_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_in_covid"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-sol-codex",
      "modelName": "gpt-5.6-sol",
      "harness": "openai-codex",
      "task": "q6_myeloid_subset_restricted_de",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 946.00627,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "myeloid_subset_with_strongest_change_gate",
            "role": "gate",
            "field": "$.myeloid_subset_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "subset_gamma"
            ]
          },
          {
            "name": "gene_with_strongest_change_gate",
            "role": "gate",
            "field": "$.gene_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "AXL",
              "PROS1"
            ]
          },
          {
            "name": "direction_of_change_gate",
            "role": "gate",
            "field": "$.direction_of_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "down_in_covid"
            ]
          },
          {
            "name": "biological_function_interpretation_gate",
            "role": "gate",
            "field": "$.biological_function_interpretation",
            "kind": "value",
            "op": "in",
            "expected": [
              "efferocytosis",
              "phagocytosis"
            ]
          },
          {
            "name": "myeloid_compartment_abundance_gate",
            "role": "gate",
            "field": "$.myeloid_compartment_abundance",
            "kind": "value",
            "op": "in",
            "expected": [
              "expands_in_covid"
            ]
          },
          {
            "name": "myeloid_expansion_localization_gate",
            "role": "gate",
            "field": "$.myeloid_expansion_localization",
            "kind": "value",
            "op": "in",
            "expected": [
              "recruited_driven_resident_dysfunction"
            ]
          },
          {
            "name": "dominant_signature_gate",
            "role": "gate",
            "field": "$.dominant_signature",
            "kind": "value",
            "op": "in",
            "expected": [
              "technical_or_agonal_artifact"
            ]
          },
          {
            "name": "dominant_signature_direction_gate",
            "role": "gate",
            "field": "$.dominant_signature_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_in_covid"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-sol-codex",
      "modelName": "gpt-5.6-sol",
      "harness": "openai-codex",
      "task": "q6_myeloid_subset_restricted_de",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": null,
      "cost": null,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "myeloid_subset_with_strongest_change_gate",
            "role": "gate",
            "field": "$.myeloid_subset_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "subset_gamma"
            ]
          },
          {
            "name": "gene_with_strongest_change_gate",
            "role": "gate",
            "field": "$.gene_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "AXL",
              "PROS1"
            ]
          },
          {
            "name": "direction_of_change_gate",
            "role": "gate",
            "field": "$.direction_of_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "down_in_covid"
            ]
          },
          {
            "name": "biological_function_interpretation_gate",
            "role": "gate",
            "field": "$.biological_function_interpretation",
            "kind": "value",
            "op": "in",
            "expected": [
              "efferocytosis",
              "phagocytosis"
            ]
          },
          {
            "name": "myeloid_compartment_abundance_gate",
            "role": "gate",
            "field": "$.myeloid_compartment_abundance",
            "kind": "value",
            "op": "in",
            "expected": [
              "expands_in_covid"
            ]
          },
          {
            "name": "myeloid_expansion_localization_gate",
            "role": "gate",
            "field": "$.myeloid_expansion_localization",
            "kind": "value",
            "op": "in",
            "expected": [
              "recruited_driven_resident_dysfunction"
            ]
          },
          {
            "name": "dominant_signature_gate",
            "role": "gate",
            "field": "$.dominant_signature",
            "kind": "value",
            "op": "in",
            "expected": [
              "technical_or_agonal_artifact"
            ]
          },
          {
            "name": "dominant_signature_direction_gate",
            "role": "gate",
            "field": "$.dominant_signature_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_in_covid"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-sol-codex",
      "modelName": "gpt-5.6-sol",
      "harness": "openai-codex",
      "task": "snrnaseq_per_donor_composition_shifts_lethal_covid_lung",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 1,
      "passed": true,
      "score": null,
      "durationS": 501.640509,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dict_match",
            "role": "gate",
            "field": "cell_type_direction_map",
            "kind": "dict",
            "expected": {
              "Myeloid": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "Epithelia": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Fibroblasts": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "T_NK": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "B_plasma": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Endothelial": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Mast": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "APC": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Neuronal": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              }
            }
          },
          {
            "name": "over_merge_immune_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Immune",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_leukocyte_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Leukocyte",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_lymphoid_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Lymphoid",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_stromal_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Stromal",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-sol-codex",
      "modelName": "gpt-5.6-sol",
      "harness": "openai-codex",
      "task": "snrnaseq_per_donor_composition_shifts_lethal_covid_lung",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 759.444953,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dict_match",
            "role": "gate",
            "field": "cell_type_direction_map",
            "kind": "dict",
            "expected": {
              "Myeloid": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "Epithelia": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Fibroblasts": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "T_NK": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "B_plasma": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Endothelial": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Mast": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "APC": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Neuronal": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              }
            }
          },
          {
            "name": "over_merge_immune_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Immune",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_leukocyte_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Leukocyte",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_lymphoid_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Lymphoid",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_stromal_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Stromal",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-sol-codex",
      "modelName": "gpt-5.6-sol",
      "harness": "openai-codex",
      "task": "snrnaseq_per_donor_composition_shifts_lethal_covid_lung",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": null,
      "cost": null,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dict_match",
            "role": "gate",
            "field": "cell_type_direction_map",
            "kind": "dict",
            "expected": {
              "Myeloid": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "Epithelia": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Fibroblasts": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "T_NK": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "B_plasma": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Endothelial": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Mast": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "APC": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Neuronal": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              }
            }
          },
          {
            "name": "over_merge_immune_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Immune",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_leukocyte_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Leukocyte",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_lymphoid_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Lymphoid",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_stromal_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Stromal",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-sol-codex",
      "modelName": "gpt-5.6-sol",
      "harness": "openai-codex",
      "task": "10x_5p_melanoma_cluster_enriched_apc_states_v2",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 1837.188771,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "top_3_cluster_enriched_apc_states",
            "kind": "list",
            "matchKey": "apc_state",
            "k": 3,
            "expected": [
              {
                "key": "plasmacytoid_DC",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "T_cell_stimulatory_role_supported",
                      "role_direction_unresolved"
                    ],
                    "canonical": "T_cell_stimulatory_role_supported"
                  }
                }
              },
              {
                "key": "C1qhigh_lipid_associated_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "C1qhigh_inflammatory_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "plasma_cell",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "cohort_significant_per_patient_inconsistent"
                    ],
                    "canonical": "cohort_significant_per_patient_inconsistent"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CD80_CD86": {
                "op": "in",
                "value": [
                  "mreg_DC",
                  "shared_across_APC_states"
                ]
              },
              "PD_L1_in_APC_clusters": {
                "op": "in",
                "value": [
                  "C1qhigh_macrophage_either_subtype",
                  "shared_across_APC_states",
                  "mreg_DC"
                ]
              },
              "type_I_IFN": {
                "op": "in",
                "value": [
                  "plasmacytoid_DC",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "cf_mreg_dc_claimed_enriched",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "mreg_DC",
              "mature_DC_enriched_in_immunoregulatory_molecules"
            ]
          },
          {
            "name": "cf_framework_offcompartment_or_generic_substitution",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "cDC1",
              "Maier_mregDC",
              "Cheng_C1QC_TAM",
              "Cheng_SPP1_TAM",
              "Cheng_FOLR2_TAM",
              "Villani_pDC",
              "Zhang_inflammatory_monocyte",
              "LAMP3_pos_DC",
              "myeloid_derived_suppressor_cell",
              "MDSC",
              "tumor_associated_neutrophil",
              "mast_cell",
              "APC",
              "antigen_presenting_cell",
              "macrophage",
              "dendritic_cell",
              "tumor_associated_macrophage",
              "TAM"
            ]
          },
          {
            "name": "cf_direction_commit_on_unresolvable_slot",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_not_resolvable_hedge",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].functional_attribution_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_type_I_IFN_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.type_I_IFN",
            "kind": "value",
            "op": "in",
            "expected": [
              "mreg_DC",
              "C1qhigh_macrophage_either_subtype"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-sol-codex",
      "modelName": "gpt-5.6-sol",
      "harness": "openai-codex",
      "task": "10x_5p_melanoma_cluster_enriched_apc_states_v2",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 823.094169,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "top_3_cluster_enriched_apc_states",
            "kind": "list",
            "matchKey": "apc_state",
            "k": 3,
            "expected": [
              {
                "key": "plasmacytoid_DC",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "T_cell_stimulatory_role_supported",
                      "role_direction_unresolved"
                    ],
                    "canonical": "T_cell_stimulatory_role_supported"
                  }
                }
              },
              {
                "key": "C1qhigh_lipid_associated_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "C1qhigh_inflammatory_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "plasma_cell",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "cohort_significant_per_patient_inconsistent"
                    ],
                    "canonical": "cohort_significant_per_patient_inconsistent"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CD80_CD86": {
                "op": "in",
                "value": [
                  "mreg_DC",
                  "shared_across_APC_states"
                ]
              },
              "PD_L1_in_APC_clusters": {
                "op": "in",
                "value": [
                  "C1qhigh_macrophage_either_subtype",
                  "shared_across_APC_states",
                  "mreg_DC"
                ]
              },
              "type_I_IFN": {
                "op": "in",
                "value": [
                  "plasmacytoid_DC",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "cf_mreg_dc_claimed_enriched",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "mreg_DC",
              "mature_DC_enriched_in_immunoregulatory_molecules"
            ]
          },
          {
            "name": "cf_framework_offcompartment_or_generic_substitution",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "cDC1",
              "Maier_mregDC",
              "Cheng_C1QC_TAM",
              "Cheng_SPP1_TAM",
              "Cheng_FOLR2_TAM",
              "Villani_pDC",
              "Zhang_inflammatory_monocyte",
              "LAMP3_pos_DC",
              "myeloid_derived_suppressor_cell",
              "MDSC",
              "tumor_associated_neutrophil",
              "mast_cell",
              "APC",
              "antigen_presenting_cell",
              "macrophage",
              "dendritic_cell",
              "tumor_associated_macrophage",
              "TAM"
            ]
          },
          {
            "name": "cf_direction_commit_on_unresolvable_slot",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_not_resolvable_hedge",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].functional_attribution_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_type_I_IFN_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.type_I_IFN",
            "kind": "value",
            "op": "in",
            "expected": [
              "mreg_DC",
              "C1qhigh_macrophage_either_subtype"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-sol-codex",
      "modelName": "gpt-5.6-sol",
      "harness": "openai-codex",
      "task": "10x_5p_melanoma_cluster_enriched_apc_states_v2",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 953.434191,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "top_3_cluster_enriched_apc_states",
            "kind": "list",
            "matchKey": "apc_state",
            "k": 3,
            "expected": [
              {
                "key": "plasmacytoid_DC",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "T_cell_stimulatory_role_supported",
                      "role_direction_unresolved"
                    ],
                    "canonical": "T_cell_stimulatory_role_supported"
                  }
                }
              },
              {
                "key": "C1qhigh_lipid_associated_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "C1qhigh_inflammatory_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "plasma_cell",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "cohort_significant_per_patient_inconsistent"
                    ],
                    "canonical": "cohort_significant_per_patient_inconsistent"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CD80_CD86": {
                "op": "in",
                "value": [
                  "mreg_DC",
                  "shared_across_APC_states"
                ]
              },
              "PD_L1_in_APC_clusters": {
                "op": "in",
                "value": [
                  "C1qhigh_macrophage_either_subtype",
                  "shared_across_APC_states",
                  "mreg_DC"
                ]
              },
              "type_I_IFN": {
                "op": "in",
                "value": [
                  "plasmacytoid_DC",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "cf_mreg_dc_claimed_enriched",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "mreg_DC",
              "mature_DC_enriched_in_immunoregulatory_molecules"
            ]
          },
          {
            "name": "cf_framework_offcompartment_or_generic_substitution",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "cDC1",
              "Maier_mregDC",
              "Cheng_C1QC_TAM",
              "Cheng_SPP1_TAM",
              "Cheng_FOLR2_TAM",
              "Villani_pDC",
              "Zhang_inflammatory_monocyte",
              "LAMP3_pos_DC",
              "myeloid_derived_suppressor_cell",
              "MDSC",
              "tumor_associated_neutrophil",
              "mast_cell",
              "APC",
              "antigen_presenting_cell",
              "macrophage",
              "dendritic_cell",
              "tumor_associated_macrophage",
              "TAM"
            ]
          },
          {
            "name": "cf_direction_commit_on_unresolvable_slot",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_not_resolvable_hedge",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].functional_attribution_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_type_I_IFN_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.type_I_IFN",
            "kind": "value",
            "op": "in",
            "expected": [
              "mreg_DC",
              "C1qhigh_macrophage_either_subtype"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-sol-codex",
      "modelName": "gpt-5.6-sol",
      "harness": "openai-codex",
      "task": "10x_5p_melanoma_cluster_tumor_state_causation_v5",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 1764.743333,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "enriched_tumor_states",
            "kind": "list",
            "matchKey": "tumor_state",
            "k": 3,
            "expected": [
              {
                "key": "immune_response",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "direction_unresolved_cross_sectional"
                    ],
                    "canonical": "direction_unresolved_cross_sectional"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "IFN_response_axis",
                      "broad_antigen_presentation_MHC"
                    ],
                    "canonical": "IFN_response_axis"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "enrichment_diminished_by_decontamination",
                      "threshold_sensitive"
                    ],
                    "canonical": "enrichment_diminished_by_decontamination"
                  }
                }
              },
              {
                "key": "stress_hypoxia",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "data_supports_cell_intrinsic_cause"
                    ],
                    "canonical": "data_supports_cell_intrinsic_cause"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "HIF_hypoxia_program_specific"
                    ],
                    "canonical": "HIF_hypoxia_program_specific"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "robust_across_methods",
                      "robust_after_decontamination"
                    ],
                    "canonical": "robust_across_methods"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CXCL9_CXCL10": {
                "op": "in",
                "value": [
                  "immune_response_tumor_state"
                ]
              },
              "CCL5": {
                "op": "in",
                "value": [
                  "shared_across_tumor_states",
                  "partner_lymphocyte_derived",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "gate_conjugate_gate_confounder",
            "role": "gate",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "partner_lymphocyte_capture"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "antigen_presentation_decomposition",
            "kind": "dict",
            "expected": {
              "MHC_class_I": {
                "op": "in",
                "value": [
                  "co_regulated_with_dominant_program",
                  "tumor_intrinsic_enriched_independent"
                ]
              },
              "MHC_class_II": {
                "op": "in",
                "value": [
                  "partner_cell_attributable",
                  "flat_not_enriched",
                  "method_sensitive_no_robust_call",
                  "co_regulated_with_dominant_program"
                ]
              }
            }
          },
          {
            "name": "cf_non_robust_state_claimed_enriched",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "transitory_melanocytic",
              "intermediate_melanocytic",
              "melanocytic",
              "MITF_high",
              "neural_crest_like",
              "mitotic",
              "stress_p53",
              "undifferentiated",
              "dedifferentiated",
              "mesenchymal_tumor",
              "AXL_high_tumor",
              "invasive_tumor",
              "senescence_like",
              "EMT_transition",
              "cancer_stem_cell_like",
              "oxidative_stress_tumor"
            ]
          },
          {
            "name": "cf_framework_generic_or_offtissue_substitution",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "Tsoi_undifferentiated",
              "Tsoi_neural_crest_like",
              "Tsoi_transitory",
              "Tsoi_melanocytic",
              "Rambow_NCSC",
              "Wouters_mesenchymal",
              "Pozniak_TCF4_dependent",
              "tumor_cell",
              "melanoma_cell",
              "malignant_cell",
              "tumor_subpopulation",
              "keratinocyte",
              "hepatocyte",
              "lung_adenocarcinoma_cell",
              "colon_adenocarcinoma_cell",
              "Tirosh_cell_cycle_program",
              "Jerby_Arnon_program",
              "glioblastoma_cell",
              "breast_carcinoma_cell"
            ]
          },
          {
            "name": "cf_immune_response_direction_committed",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_immune_robustness_recall",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_ambiguous_hedge",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].causation_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_stress_generic_stress_response_evidence",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_conjugate_confounder_contamination_denied",
            "role": "hard_fail",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected"
            ]
          },
          {
            "name": "cf_ccl5_recall_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CCL5",
            "kind": "avoid",
            "expected": [
              "immune_response_tumor_state"
            ]
          },
          {
            "name": "cf_cxcl9_cxcl10_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CXCL9_CXCL10",
            "kind": "avoid",
            "expected": [
              "stress_hypoxia_tumor_state"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-sol-codex",
      "modelName": "gpt-5.6-sol",
      "harness": "openai-codex",
      "task": "10x_5p_melanoma_cluster_tumor_state_causation_v5",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 1389.944516,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "enriched_tumor_states",
            "kind": "list",
            "matchKey": "tumor_state",
            "k": 3,
            "expected": [
              {
                "key": "immune_response",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "direction_unresolved_cross_sectional"
                    ],
                    "canonical": "direction_unresolved_cross_sectional"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "IFN_response_axis",
                      "broad_antigen_presentation_MHC"
                    ],
                    "canonical": "IFN_response_axis"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "enrichment_diminished_by_decontamination",
                      "threshold_sensitive"
                    ],
                    "canonical": "enrichment_diminished_by_decontamination"
                  }
                }
              },
              {
                "key": "stress_hypoxia",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "data_supports_cell_intrinsic_cause"
                    ],
                    "canonical": "data_supports_cell_intrinsic_cause"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "HIF_hypoxia_program_specific"
                    ],
                    "canonical": "HIF_hypoxia_program_specific"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "robust_across_methods",
                      "robust_after_decontamination"
                    ],
                    "canonical": "robust_across_methods"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CXCL9_CXCL10": {
                "op": "in",
                "value": [
                  "immune_response_tumor_state"
                ]
              },
              "CCL5": {
                "op": "in",
                "value": [
                  "shared_across_tumor_states",
                  "partner_lymphocyte_derived",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "gate_conjugate_gate_confounder",
            "role": "gate",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "partner_lymphocyte_capture"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "antigen_presentation_decomposition",
            "kind": "dict",
            "expected": {
              "MHC_class_I": {
                "op": "in",
                "value": [
                  "co_regulated_with_dominant_program",
                  "tumor_intrinsic_enriched_independent"
                ]
              },
              "MHC_class_II": {
                "op": "in",
                "value": [
                  "partner_cell_attributable",
                  "flat_not_enriched",
                  "method_sensitive_no_robust_call",
                  "co_regulated_with_dominant_program"
                ]
              }
            }
          },
          {
            "name": "cf_non_robust_state_claimed_enriched",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "transitory_melanocytic",
              "intermediate_melanocytic",
              "melanocytic",
              "MITF_high",
              "neural_crest_like",
              "mitotic",
              "stress_p53",
              "undifferentiated",
              "dedifferentiated",
              "mesenchymal_tumor",
              "AXL_high_tumor",
              "invasive_tumor",
              "senescence_like",
              "EMT_transition",
              "cancer_stem_cell_like",
              "oxidative_stress_tumor"
            ]
          },
          {
            "name": "cf_framework_generic_or_offtissue_substitution",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "Tsoi_undifferentiated",
              "Tsoi_neural_crest_like",
              "Tsoi_transitory",
              "Tsoi_melanocytic",
              "Rambow_NCSC",
              "Wouters_mesenchymal",
              "Pozniak_TCF4_dependent",
              "tumor_cell",
              "melanoma_cell",
              "malignant_cell",
              "tumor_subpopulation",
              "keratinocyte",
              "hepatocyte",
              "lung_adenocarcinoma_cell",
              "colon_adenocarcinoma_cell",
              "Tirosh_cell_cycle_program",
              "Jerby_Arnon_program",
              "glioblastoma_cell",
              "breast_carcinoma_cell"
            ]
          },
          {
            "name": "cf_immune_response_direction_committed",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_immune_robustness_recall",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_ambiguous_hedge",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].causation_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_stress_generic_stress_response_evidence",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_conjugate_confounder_contamination_denied",
            "role": "hard_fail",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected"
            ]
          },
          {
            "name": "cf_ccl5_recall_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CCL5",
            "kind": "avoid",
            "expected": [
              "immune_response_tumor_state"
            ]
          },
          {
            "name": "cf_cxcl9_cxcl10_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CXCL9_CXCL10",
            "kind": "avoid",
            "expected": [
              "stress_hypoxia_tumor_state"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-sol-codex",
      "modelName": "gpt-5.6-sol",
      "harness": "openai-codex",
      "task": "10x_5p_melanoma_cluster_tumor_state_causation_v5",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1260.692401,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "enriched_tumor_states",
            "kind": "list",
            "matchKey": "tumor_state",
            "k": 3,
            "expected": [
              {
                "key": "immune_response",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "direction_unresolved_cross_sectional"
                    ],
                    "canonical": "direction_unresolved_cross_sectional"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "IFN_response_axis",
                      "broad_antigen_presentation_MHC"
                    ],
                    "canonical": "IFN_response_axis"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "enrichment_diminished_by_decontamination",
                      "threshold_sensitive"
                    ],
                    "canonical": "enrichment_diminished_by_decontamination"
                  }
                }
              },
              {
                "key": "stress_hypoxia",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "data_supports_cell_intrinsic_cause"
                    ],
                    "canonical": "data_supports_cell_intrinsic_cause"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "HIF_hypoxia_program_specific"
                    ],
                    "canonical": "HIF_hypoxia_program_specific"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "robust_across_methods",
                      "robust_after_decontamination"
                    ],
                    "canonical": "robust_across_methods"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CXCL9_CXCL10": {
                "op": "in",
                "value": [
                  "immune_response_tumor_state"
                ]
              },
              "CCL5": {
                "op": "in",
                "value": [
                  "shared_across_tumor_states",
                  "partner_lymphocyte_derived",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "gate_conjugate_gate_confounder",
            "role": "gate",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "partner_lymphocyte_capture"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "antigen_presentation_decomposition",
            "kind": "dict",
            "expected": {
              "MHC_class_I": {
                "op": "in",
                "value": [
                  "co_regulated_with_dominant_program",
                  "tumor_intrinsic_enriched_independent"
                ]
              },
              "MHC_class_II": {
                "op": "in",
                "value": [
                  "partner_cell_attributable",
                  "flat_not_enriched",
                  "method_sensitive_no_robust_call",
                  "co_regulated_with_dominant_program"
                ]
              }
            }
          },
          {
            "name": "cf_non_robust_state_claimed_enriched",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "transitory_melanocytic",
              "intermediate_melanocytic",
              "melanocytic",
              "MITF_high",
              "neural_crest_like",
              "mitotic",
              "stress_p53",
              "undifferentiated",
              "dedifferentiated",
              "mesenchymal_tumor",
              "AXL_high_tumor",
              "invasive_tumor",
              "senescence_like",
              "EMT_transition",
              "cancer_stem_cell_like",
              "oxidative_stress_tumor"
            ]
          },
          {
            "name": "cf_framework_generic_or_offtissue_substitution",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "Tsoi_undifferentiated",
              "Tsoi_neural_crest_like",
              "Tsoi_transitory",
              "Tsoi_melanocytic",
              "Rambow_NCSC",
              "Wouters_mesenchymal",
              "Pozniak_TCF4_dependent",
              "tumor_cell",
              "melanoma_cell",
              "malignant_cell",
              "tumor_subpopulation",
              "keratinocyte",
              "hepatocyte",
              "lung_adenocarcinoma_cell",
              "colon_adenocarcinoma_cell",
              "Tirosh_cell_cycle_program",
              "Jerby_Arnon_program",
              "glioblastoma_cell",
              "breast_carcinoma_cell"
            ]
          },
          {
            "name": "cf_immune_response_direction_committed",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_immune_robustness_recall",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_ambiguous_hedge",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].causation_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_stress_generic_stress_response_evidence",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_conjugate_confounder_contamination_denied",
            "role": "hard_fail",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected"
            ]
          },
          {
            "name": "cf_ccl5_recall_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CCL5",
            "kind": "avoid",
            "expected": [
              "immune_response_tumor_state"
            ]
          },
          {
            "name": "cf_cxcl9_cxcl10_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CXCL9_CXCL10",
            "kind": "avoid",
            "expected": [
              "stress_hypoxia_tumor_state"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-sol-codex",
      "modelName": "gpt-5.6-sol",
      "harness": "openai-codex",
      "task": "10x_5p_melanoma_cluster_vs_cd39_tumor_reactivity_dominance",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 1,
      "passed": true,
      "score": null,
      "durationS": 592.286082,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "population_findings",
            "kind": "list",
            "matchKey": "capture_gate",
            "k": 4,
            "expected": [
              {
                "key": "tumor_cell_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "apc_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "combined_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "unselected_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "naive",
                      "central_memory",
                      "effector_memory"
                    ],
                    "canonical": "naive"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "low"
                    ],
                    "canonical": "low"
                  },
                  "purity_caveat": {
                    "op": "equals",
                    "values": [
                      "pure_T_cell"
                    ],
                    "canonical": "pure_T_cell"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "no_reactivity_evidence"
                    ],
                    "canonical": "no_reactivity_evidence"
                  }
                }
              },
              {
                "key": "marker_sorted_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "equals",
                    "values": [
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "terminal_exhausted_LAG3_high"
                  },
                  "clonal_expansion_level": {
                    "op": "in",
                    "values": [
                      "high",
                      "intermediate"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "marker_on_non_T_lineages",
                      "single_cell_gating_loses_target"
                    ],
                    "canonical": "marker_on_non_T_lineages"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "marker_or_exhaustion_defined_only"
                    ],
                    "canonical": "marker_or_exhaustion_defined_only"
                  }
                }
              }
            ]
          },
          {
            "name": "best_capture_approach_gate",
            "role": "gate",
            "field": "best_capture_approach",
            "kind": "value",
            "expected": "heterotypic_cell_conjugate"
          },
          {
            "name": "best_selection_marker_gate",
            "role": "gate",
            "field": "best_selection_marker",
            "kind": "value",
            "expected": "none"
          },
          {
            "name": "reactivity_criterion_gate",
            "role": "gate",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "clonal_expansion_and_physical_engagement"
          },
          {
            "name": "cross_patient_robustness_gate",
            "role": "gate",
            "field": "cross_patient_robustness",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_across_patients",
              "marginal_in_some_patients"
            ]
          },
          {
            "name": "marker_selection_consequence_gate",
            "role": "gate",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "discards_clonally_expanded_reactive_cells"
          },
          {
            "name": "superiority_claim_basis_gate",
            "role": "gate",
            "field": "superiority_claim_basis",
            "kind": "value",
            "expected": "requires_functional_reactivity_assay"
          },
          {
            "name": "conjugate_enriched_state_gate",
            "role": "gate",
            "field": "conjugate_enriched_state",
            "kind": "value",
            "expected": "progenitor_exhausted_TCF7_pos_stem_like"
          },
          {
            "name": "cf_circular_exhaustion_criterion",
            "role": "hard_fail",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "exhaustion_signature_intensity"
          },
          {
            "name": "cf_marker_sort_prior",
            "role": "hard_fail",
            "field": "best_selection_marker",
            "kind": "value",
            "op": "in",
            "expected": [
              "CD39",
              "PD1",
              "PD1_high",
              "TIM3",
              "PD1_CD39",
              "CD39_CD103"
            ]
          },
          {
            "name": "cf_single_cell_capture_prior",
            "role": "hard_fail",
            "field": "best_capture_approach",
            "kind": "value",
            "op": "in",
            "expected": [
              "dissociated_single_cell",
              "dissociated_bulk_population"
            ]
          },
          {
            "name": "cf_off_data_capture_gate",
            "role": "hard_fail",
            "field": "$.population_findings[*].capture_gate",
            "kind": "avoid",
            "expected": [
              "dissociated_bulk",
              "matched_peripheral_blood",
              "matched_lymph_node",
              "post_hoc_expanded_clonotype_group"
            ]
          },
          {
            "name": "cf_conjugate_mischaracterized",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_marker_singlet_called_engaged",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "contains",
            "matchKey": "capture_gate",
            "entity": "marker_sorted_singlet",
            "valueField": "reactivity_evidence",
            "expected": [
              "engagement_and_clonal_expansion"
            ]
          },
          {
            "name": "cf_no_engaged_reactive_population",
            "role": "hard_fail",
            "field": "$.population_findings[*].reactivity_evidence",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_marker_captures_all_reactive",
            "role": "hard_fail",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "captures_full_reactive_repertoire"
          },
          {
            "name": "cf_superiority_overclaimed_from_phenotype",
            "role": "hard_fail",
            "field": "superiority_claim_basis",
            "kind": "value",
            "op": "in",
            "expected": [
              "established_by_scrna_phenotype",
              "established_by_reactivity_proxy_scores"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-sol-codex",
      "modelName": "gpt-5.6-sol",
      "harness": "openai-codex",
      "task": "10x_5p_melanoma_cluster_vs_cd39_tumor_reactivity_dominance",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 789.01199,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "population_findings",
            "kind": "list",
            "matchKey": "capture_gate",
            "k": 4,
            "expected": [
              {
                "key": "tumor_cell_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "apc_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "combined_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "unselected_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "naive",
                      "central_memory",
                      "effector_memory"
                    ],
                    "canonical": "naive"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "low"
                    ],
                    "canonical": "low"
                  },
                  "purity_caveat": {
                    "op": "equals",
                    "values": [
                      "pure_T_cell"
                    ],
                    "canonical": "pure_T_cell"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "no_reactivity_evidence"
                    ],
                    "canonical": "no_reactivity_evidence"
                  }
                }
              },
              {
                "key": "marker_sorted_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "equals",
                    "values": [
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "terminal_exhausted_LAG3_high"
                  },
                  "clonal_expansion_level": {
                    "op": "in",
                    "values": [
                      "high",
                      "intermediate"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "marker_on_non_T_lineages",
                      "single_cell_gating_loses_target"
                    ],
                    "canonical": "marker_on_non_T_lineages"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "marker_or_exhaustion_defined_only"
                    ],
                    "canonical": "marker_or_exhaustion_defined_only"
                  }
                }
              }
            ]
          },
          {
            "name": "best_capture_approach_gate",
            "role": "gate",
            "field": "best_capture_approach",
            "kind": "value",
            "expected": "heterotypic_cell_conjugate"
          },
          {
            "name": "best_selection_marker_gate",
            "role": "gate",
            "field": "best_selection_marker",
            "kind": "value",
            "expected": "none"
          },
          {
            "name": "reactivity_criterion_gate",
            "role": "gate",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "clonal_expansion_and_physical_engagement"
          },
          {
            "name": "cross_patient_robustness_gate",
            "role": "gate",
            "field": "cross_patient_robustness",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_across_patients",
              "marginal_in_some_patients"
            ]
          },
          {
            "name": "marker_selection_consequence_gate",
            "role": "gate",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "discards_clonally_expanded_reactive_cells"
          },
          {
            "name": "superiority_claim_basis_gate",
            "role": "gate",
            "field": "superiority_claim_basis",
            "kind": "value",
            "expected": "requires_functional_reactivity_assay"
          },
          {
            "name": "conjugate_enriched_state_gate",
            "role": "gate",
            "field": "conjugate_enriched_state",
            "kind": "value",
            "expected": "progenitor_exhausted_TCF7_pos_stem_like"
          },
          {
            "name": "cf_circular_exhaustion_criterion",
            "role": "hard_fail",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "exhaustion_signature_intensity"
          },
          {
            "name": "cf_marker_sort_prior",
            "role": "hard_fail",
            "field": "best_selection_marker",
            "kind": "value",
            "op": "in",
            "expected": [
              "CD39",
              "PD1",
              "PD1_high",
              "TIM3",
              "PD1_CD39",
              "CD39_CD103"
            ]
          },
          {
            "name": "cf_single_cell_capture_prior",
            "role": "hard_fail",
            "field": "best_capture_approach",
            "kind": "value",
            "op": "in",
            "expected": [
              "dissociated_single_cell",
              "dissociated_bulk_population"
            ]
          },
          {
            "name": "cf_off_data_capture_gate",
            "role": "hard_fail",
            "field": "$.population_findings[*].capture_gate",
            "kind": "avoid",
            "expected": [
              "dissociated_bulk",
              "matched_peripheral_blood",
              "matched_lymph_node",
              "post_hoc_expanded_clonotype_group"
            ]
          },
          {
            "name": "cf_conjugate_mischaracterized",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_marker_singlet_called_engaged",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "contains",
            "matchKey": "capture_gate",
            "entity": "marker_sorted_singlet",
            "valueField": "reactivity_evidence",
            "expected": [
              "engagement_and_clonal_expansion"
            ]
          },
          {
            "name": "cf_no_engaged_reactive_population",
            "role": "hard_fail",
            "field": "$.population_findings[*].reactivity_evidence",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_marker_captures_all_reactive",
            "role": "hard_fail",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "captures_full_reactive_repertoire"
          },
          {
            "name": "cf_superiority_overclaimed_from_phenotype",
            "role": "hard_fail",
            "field": "superiority_claim_basis",
            "kind": "value",
            "op": "in",
            "expected": [
              "established_by_scrna_phenotype",
              "established_by_reactivity_proxy_scores"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-sol-codex",
      "modelName": "gpt-5.6-sol",
      "harness": "openai-codex",
      "task": "10x_5p_melanoma_cluster_vs_cd39_tumor_reactivity_dominance",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 625.920023,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "population_findings",
            "kind": "list",
            "matchKey": "capture_gate",
            "k": 4,
            "expected": [
              {
                "key": "tumor_cell_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "apc_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "combined_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "unselected_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "naive",
                      "central_memory",
                      "effector_memory"
                    ],
                    "canonical": "naive"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "low"
                    ],
                    "canonical": "low"
                  },
                  "purity_caveat": {
                    "op": "equals",
                    "values": [
                      "pure_T_cell"
                    ],
                    "canonical": "pure_T_cell"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "no_reactivity_evidence"
                    ],
                    "canonical": "no_reactivity_evidence"
                  }
                }
              },
              {
                "key": "marker_sorted_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "equals",
                    "values": [
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "terminal_exhausted_LAG3_high"
                  },
                  "clonal_expansion_level": {
                    "op": "in",
                    "values": [
                      "high",
                      "intermediate"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "marker_on_non_T_lineages",
                      "single_cell_gating_loses_target"
                    ],
                    "canonical": "marker_on_non_T_lineages"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "marker_or_exhaustion_defined_only"
                    ],
                    "canonical": "marker_or_exhaustion_defined_only"
                  }
                }
              }
            ]
          },
          {
            "name": "best_capture_approach_gate",
            "role": "gate",
            "field": "best_capture_approach",
            "kind": "value",
            "expected": "heterotypic_cell_conjugate"
          },
          {
            "name": "best_selection_marker_gate",
            "role": "gate",
            "field": "best_selection_marker",
            "kind": "value",
            "expected": "none"
          },
          {
            "name": "reactivity_criterion_gate",
            "role": "gate",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "clonal_expansion_and_physical_engagement"
          },
          {
            "name": "cross_patient_robustness_gate",
            "role": "gate",
            "field": "cross_patient_robustness",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_across_patients",
              "marginal_in_some_patients"
            ]
          },
          {
            "name": "marker_selection_consequence_gate",
            "role": "gate",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "discards_clonally_expanded_reactive_cells"
          },
          {
            "name": "superiority_claim_basis_gate",
            "role": "gate",
            "field": "superiority_claim_basis",
            "kind": "value",
            "expected": "requires_functional_reactivity_assay"
          },
          {
            "name": "conjugate_enriched_state_gate",
            "role": "gate",
            "field": "conjugate_enriched_state",
            "kind": "value",
            "expected": "progenitor_exhausted_TCF7_pos_stem_like"
          },
          {
            "name": "cf_circular_exhaustion_criterion",
            "role": "hard_fail",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "exhaustion_signature_intensity"
          },
          {
            "name": "cf_marker_sort_prior",
            "role": "hard_fail",
            "field": "best_selection_marker",
            "kind": "value",
            "op": "in",
            "expected": [
              "CD39",
              "PD1",
              "PD1_high",
              "TIM3",
              "PD1_CD39",
              "CD39_CD103"
            ]
          },
          {
            "name": "cf_single_cell_capture_prior",
            "role": "hard_fail",
            "field": "best_capture_approach",
            "kind": "value",
            "op": "in",
            "expected": [
              "dissociated_single_cell",
              "dissociated_bulk_population"
            ]
          },
          {
            "name": "cf_off_data_capture_gate",
            "role": "hard_fail",
            "field": "$.population_findings[*].capture_gate",
            "kind": "avoid",
            "expected": [
              "dissociated_bulk",
              "matched_peripheral_blood",
              "matched_lymph_node",
              "post_hoc_expanded_clonotype_group"
            ]
          },
          {
            "name": "cf_conjugate_mischaracterized",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_marker_singlet_called_engaged",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "contains",
            "matchKey": "capture_gate",
            "entity": "marker_sorted_singlet",
            "valueField": "reactivity_evidence",
            "expected": [
              "engagement_and_clonal_expansion"
            ]
          },
          {
            "name": "cf_no_engaged_reactive_population",
            "role": "hard_fail",
            "field": "$.population_findings[*].reactivity_evidence",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_marker_captures_all_reactive",
            "role": "hard_fail",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "captures_full_reactive_repertoire"
          },
          {
            "name": "cf_superiority_overclaimed_from_phenotype",
            "role": "hard_fail",
            "field": "superiority_claim_basis",
            "kind": "value",
            "op": "in",
            "expected": [
              "established_by_scrna_phenotype",
              "established_by_reactivity_proxy_scores"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-sol-codex",
      "modelName": "gpt-5.6-sol",
      "harness": "openai-codex",
      "task": "10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 1114.28129,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_singlet_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "TOXhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "LAG3high_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "GZMKhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "intermediate_exhausted_gzmk"
                    ],
                    "canonical": "intermediate_exhausted_gzmk"
                  }
                }
              },
              {
                "key": "TCF7_stem_like_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "stem_like_exhausted"
                    ],
                    "canonical": "stem_like_exhausted"
                  }
                }
              },
              {
                "key": "MKI67_pos_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              },
              {
                "key": "MKI67high_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_tumor_cluster_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "Tn",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "Tn_Tmem",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "early_Tem_GZMK",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              },
              {
                "key": "Tem_GZMH",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              }
            ]
          },
          {
            "name": "clonotype_expansion_direction_data_true",
            "role": "gate",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_substantially_more_expanded_than_cluster",
              "singlet_modestly_more_expanded_than_cluster"
            ]
          },
          {
            "name": "biological_inference_data_true",
            "role": "gate",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_antigen_experienced_cluster_bystander"
            ]
          },
          {
            "name": "singlet_top3_contains_naive_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Tn",
              "Tn_Tmem",
              "early_Tem_GZMK",
              "Tem_GZMH"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "TOXhigh_Tex",
              "LAG3high_Tex",
              "GZMKhigh_Tex",
              "TCF7_stem_like_Tex",
              "MKI67_pos_Tex_Tprol",
              "MKI67high_Tex_Tprol"
            ]
          },
          {
            "name": "singlet_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "TOXhigh_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "TCF7_stem_like_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "MKI67_state_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "Tn_or_Tn_Tmem_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_stemlike_precursor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_reported_canonical_expansion_direction",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_substantially_more_expanded_than_singlet",
              "cluster_modestly_more_expanded_than_singlet"
            ]
          },
          {
            "name": "cf_expansion_lazy_defer",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "expansion_pattern_not_detectable"
            ]
          },
          {
            "name": "cf_reported_canonical_expected_biology",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_antigen_experienced_singlet_bystander"
            ]
          },
          {
            "name": "cf_biological_inference_null_artifact_or_lineage",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "differences_reflect_dissociation_or_sorting_artifact",
              "cluster_and_singlet_capture_same_biology",
              "cluster_and_singlet_capture_distinct_cd8_lineages"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "biological_relations",
            "kind": "dict",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_greater_than",
                  "greater_than"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "less_than",
                  "much_less_than"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than",
                  "approximately_equal"
                ]
              },
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal"
                ]
              },
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal",
                  "greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_exhaustion_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_naive_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_proliferation_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_expansion_richness_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_multichain_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_intermediate_gzmk_overgeneralized",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_melanocytic_conjugate_overcall",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "values": [
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_relations_mass_hedge",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-sol-codex",
      "modelName": "gpt-5.6-sol",
      "harness": "openai-codex",
      "task": "10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 1075.955197,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_singlet_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "TOXhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "LAG3high_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "GZMKhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "intermediate_exhausted_gzmk"
                    ],
                    "canonical": "intermediate_exhausted_gzmk"
                  }
                }
              },
              {
                "key": "TCF7_stem_like_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "stem_like_exhausted"
                    ],
                    "canonical": "stem_like_exhausted"
                  }
                }
              },
              {
                "key": "MKI67_pos_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              },
              {
                "key": "MKI67high_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_tumor_cluster_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "Tn",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "Tn_Tmem",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "early_Tem_GZMK",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              },
              {
                "key": "Tem_GZMH",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              }
            ]
          },
          {
            "name": "clonotype_expansion_direction_data_true",
            "role": "gate",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_substantially_more_expanded_than_cluster",
              "singlet_modestly_more_expanded_than_cluster"
            ]
          },
          {
            "name": "biological_inference_data_true",
            "role": "gate",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_antigen_experienced_cluster_bystander"
            ]
          },
          {
            "name": "singlet_top3_contains_naive_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Tn",
              "Tn_Tmem",
              "early_Tem_GZMK",
              "Tem_GZMH"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "TOXhigh_Tex",
              "LAG3high_Tex",
              "GZMKhigh_Tex",
              "TCF7_stem_like_Tex",
              "MKI67_pos_Tex_Tprol",
              "MKI67high_Tex_Tprol"
            ]
          },
          {
            "name": "singlet_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "TOXhigh_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "TCF7_stem_like_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "MKI67_state_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "Tn_or_Tn_Tmem_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_stemlike_precursor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_reported_canonical_expansion_direction",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_substantially_more_expanded_than_singlet",
              "cluster_modestly_more_expanded_than_singlet"
            ]
          },
          {
            "name": "cf_expansion_lazy_defer",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "expansion_pattern_not_detectable"
            ]
          },
          {
            "name": "cf_reported_canonical_expected_biology",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_antigen_experienced_singlet_bystander"
            ]
          },
          {
            "name": "cf_biological_inference_null_artifact_or_lineage",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "differences_reflect_dissociation_or_sorting_artifact",
              "cluster_and_singlet_capture_same_biology",
              "cluster_and_singlet_capture_distinct_cd8_lineages"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "biological_relations",
            "kind": "dict",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_greater_than",
                  "greater_than"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "less_than",
                  "much_less_than"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than",
                  "approximately_equal"
                ]
              },
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal"
                ]
              },
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal",
                  "greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_exhaustion_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_naive_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_proliferation_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_expansion_richness_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_multichain_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_intermediate_gzmk_overgeneralized",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_melanocytic_conjugate_overcall",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "values": [
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_relations_mass_hedge",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-sol-codex",
      "modelName": "gpt-5.6-sol",
      "harness": "openai-codex",
      "task": "10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1197.36982,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_singlet_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "TOXhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "LAG3high_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "GZMKhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "intermediate_exhausted_gzmk"
                    ],
                    "canonical": "intermediate_exhausted_gzmk"
                  }
                }
              },
              {
                "key": "TCF7_stem_like_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "stem_like_exhausted"
                    ],
                    "canonical": "stem_like_exhausted"
                  }
                }
              },
              {
                "key": "MKI67_pos_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              },
              {
                "key": "MKI67high_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_tumor_cluster_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "Tn",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "Tn_Tmem",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "early_Tem_GZMK",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              },
              {
                "key": "Tem_GZMH",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              }
            ]
          },
          {
            "name": "clonotype_expansion_direction_data_true",
            "role": "gate",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_substantially_more_expanded_than_cluster",
              "singlet_modestly_more_expanded_than_cluster"
            ]
          },
          {
            "name": "biological_inference_data_true",
            "role": "gate",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_antigen_experienced_cluster_bystander"
            ]
          },
          {
            "name": "singlet_top3_contains_naive_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Tn",
              "Tn_Tmem",
              "early_Tem_GZMK",
              "Tem_GZMH"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "TOXhigh_Tex",
              "LAG3high_Tex",
              "GZMKhigh_Tex",
              "TCF7_stem_like_Tex",
              "MKI67_pos_Tex_Tprol",
              "MKI67high_Tex_Tprol"
            ]
          },
          {
            "name": "singlet_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "TOXhigh_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "TCF7_stem_like_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "MKI67_state_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "Tn_or_Tn_Tmem_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_stemlike_precursor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_reported_canonical_expansion_direction",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_substantially_more_expanded_than_singlet",
              "cluster_modestly_more_expanded_than_singlet"
            ]
          },
          {
            "name": "cf_expansion_lazy_defer",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "expansion_pattern_not_detectable"
            ]
          },
          {
            "name": "cf_reported_canonical_expected_biology",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_antigen_experienced_singlet_bystander"
            ]
          },
          {
            "name": "cf_biological_inference_null_artifact_or_lineage",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "differences_reflect_dissociation_or_sorting_artifact",
              "cluster_and_singlet_capture_same_biology",
              "cluster_and_singlet_capture_distinct_cd8_lineages"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "biological_relations",
            "kind": "dict",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_greater_than",
                  "greater_than"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "less_than",
                  "much_less_than"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than",
                  "approximately_equal"
                ]
              },
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal"
                ]
              },
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal",
                  "greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_exhaustion_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_naive_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_proliferation_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_expansion_richness_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_multichain_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_intermediate_gzmk_overgeneralized",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_melanocytic_conjugate_overcall",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "values": [
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_relations_mass_hedge",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-sol-codex",
      "modelName": "gpt-5.6-sol",
      "harness": "openai-codex",
      "task": "10x_5p_melanoma_tcr_matched_apc_vs_tumor_exhaustion",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 1,
      "passed": true,
      "score": null,
      "durationS": 547.832954,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "state_discriminating_genes",
            "kind": "list",
            "matchKey": "gene",
            "k": 3,
            "expected": [
              {
                "key": "CTLA4",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TOX",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "ENTPD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "HAVCR2",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TIGIT",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "PDCD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "difference_attributable_to",
            "role": "gate",
            "field": "difference_attributable_to",
            "kind": "value",
            "expected": "covaries_with_apc_conjugate_state"
          },
          {
            "name": "causal_status",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "direction_unresolved_selection_confound"
          },
          {
            "name": "functional_interpretation",
            "role": "gate",
            "field": "functional_interpretation",
            "kind": "value",
            "expected": "costimulation_signature_function_unresolved"
          },
          {
            "name": "cf_contamination_gene_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "MLANA",
              "PMEL",
              "DCT",
              "TYR",
              "TYRP1",
              "SOX10",
              "MITF",
              "MAGEA4"
            ]
          },
          {
            "name": "cf_apc_partner_contamination_gene",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "HLA-DRA",
              "CD68",
              "LYZ",
              "CD14",
              "C1QA",
              "ITGAX",
              "CD1C"
            ]
          },
          {
            "name": "cf_ligand_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "CD274",
              "PDCD1LG2",
              "CD80",
              "CD86"
            ]
          },
          {
            "name": "cf_attribution_causal_or_baseline",
            "role": "hard_fail",
            "field": "$.difference_attributable_to",
            "kind": "avoid",
            "expected": [
              "apc_engagement",
              "tumor_cell_engagement",
              "clonotype_intrinsic_baseline"
            ]
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "engagement_induces_exhausted_state",
              "pre_exhausted_cells_select_into_apc_conjugates"
            ]
          },
          {
            "name": "cf_causal_fabrication",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "resolved_by_pseudotemporal_ordering",
              "resolved_by_perturbation_experiment"
            ]
          },
          {
            "name": "cf_functional_overclaim",
            "role": "hard_fail",
            "field": "$.functional_interpretation",
            "kind": "avoid",
            "expected": [
              "costimulation_licensed_reinvigoration_competent",
              "costimulation_licensed_checkpoint_responsive",
              "tumor_driven_terminal_exhaustion"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-sol-codex",
      "modelName": "gpt-5.6-sol",
      "harness": "openai-codex",
      "task": "10x_5p_melanoma_tcr_matched_apc_vs_tumor_exhaustion",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 2,
      "passed": true,
      "score": null,
      "durationS": 320.2512,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "state_discriminating_genes",
            "kind": "list",
            "matchKey": "gene",
            "k": 3,
            "expected": [
              {
                "key": "CTLA4",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TOX",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "ENTPD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "HAVCR2",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TIGIT",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "PDCD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "difference_attributable_to",
            "role": "gate",
            "field": "difference_attributable_to",
            "kind": "value",
            "expected": "covaries_with_apc_conjugate_state"
          },
          {
            "name": "causal_status",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "direction_unresolved_selection_confound"
          },
          {
            "name": "functional_interpretation",
            "role": "gate",
            "field": "functional_interpretation",
            "kind": "value",
            "expected": "costimulation_signature_function_unresolved"
          },
          {
            "name": "cf_contamination_gene_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "MLANA",
              "PMEL",
              "DCT",
              "TYR",
              "TYRP1",
              "SOX10",
              "MITF",
              "MAGEA4"
            ]
          },
          {
            "name": "cf_apc_partner_contamination_gene",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "HLA-DRA",
              "CD68",
              "LYZ",
              "CD14",
              "C1QA",
              "ITGAX",
              "CD1C"
            ]
          },
          {
            "name": "cf_ligand_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "CD274",
              "PDCD1LG2",
              "CD80",
              "CD86"
            ]
          },
          {
            "name": "cf_attribution_causal_or_baseline",
            "role": "hard_fail",
            "field": "$.difference_attributable_to",
            "kind": "avoid",
            "expected": [
              "apc_engagement",
              "tumor_cell_engagement",
              "clonotype_intrinsic_baseline"
            ]
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "engagement_induces_exhausted_state",
              "pre_exhausted_cells_select_into_apc_conjugates"
            ]
          },
          {
            "name": "cf_causal_fabrication",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "resolved_by_pseudotemporal_ordering",
              "resolved_by_perturbation_experiment"
            ]
          },
          {
            "name": "cf_functional_overclaim",
            "role": "hard_fail",
            "field": "$.functional_interpretation",
            "kind": "avoid",
            "expected": [
              "costimulation_licensed_reinvigoration_competent",
              "costimulation_licensed_checkpoint_responsive",
              "tumor_driven_terminal_exhaustion"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-sol-codex",
      "modelName": "gpt-5.6-sol",
      "harness": "openai-codex",
      "task": "10x_5p_melanoma_tcr_matched_apc_vs_tumor_exhaustion",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 610.179704,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "state_discriminating_genes",
            "kind": "list",
            "matchKey": "gene",
            "k": 3,
            "expected": [
              {
                "key": "CTLA4",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TOX",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "ENTPD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "HAVCR2",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TIGIT",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "PDCD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "difference_attributable_to",
            "role": "gate",
            "field": "difference_attributable_to",
            "kind": "value",
            "expected": "covaries_with_apc_conjugate_state"
          },
          {
            "name": "causal_status",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "direction_unresolved_selection_confound"
          },
          {
            "name": "functional_interpretation",
            "role": "gate",
            "field": "functional_interpretation",
            "kind": "value",
            "expected": "costimulation_signature_function_unresolved"
          },
          {
            "name": "cf_contamination_gene_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "MLANA",
              "PMEL",
              "DCT",
              "TYR",
              "TYRP1",
              "SOX10",
              "MITF",
              "MAGEA4"
            ]
          },
          {
            "name": "cf_apc_partner_contamination_gene",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "HLA-DRA",
              "CD68",
              "LYZ",
              "CD14",
              "C1QA",
              "ITGAX",
              "CD1C"
            ]
          },
          {
            "name": "cf_ligand_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "CD274",
              "PDCD1LG2",
              "CD80",
              "CD86"
            ]
          },
          {
            "name": "cf_attribution_causal_or_baseline",
            "role": "hard_fail",
            "field": "$.difference_attributable_to",
            "kind": "avoid",
            "expected": [
              "apc_engagement",
              "tumor_cell_engagement",
              "clonotype_intrinsic_baseline"
            ]
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "engagement_induces_exhausted_state",
              "pre_exhausted_cells_select_into_apc_conjugates"
            ]
          },
          {
            "name": "cf_causal_fabrication",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "resolved_by_pseudotemporal_ordering",
              "resolved_by_perturbation_experiment"
            ]
          },
          {
            "name": "cf_functional_overclaim",
            "role": "hard_fail",
            "field": "$.functional_interpretation",
            "kind": "avoid",
            "expected": [
              "costimulation_licensed_reinvigoration_competent",
              "costimulation_licensed_checkpoint_responsive",
              "tumor_driven_terminal_exhaustion"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-sol-codex",
      "modelName": "gpt-5.6-sol",
      "harness": "openai-codex",
      "task": "green_2_tme_context_dependence",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 1612.75038,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "program_map",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "sig_includes_itgae",
            "role": "gate",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Itgae"
            ]
          },
          {
            "name": "declared_a_known_confounder",
            "role": "gate",
            "field": "$.excluded_confounders[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "cf_signature_gut_contaminant",
            "role": "hard_fail",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "generic_signature_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd103_program_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "<"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "per_model_regulatory_state",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "poised_unfired"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "late_activation"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "closing_committed"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_tuples",
            "kind": "list",
            "matchKey": "regulator",
            "k": 1,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "role": {
                    "op": "equals",
                    "values": [
                      "repressor"
                    ]
                  },
                  "gating_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  },
                  "evidence_basis": {
                    "op": "equals",
                    "values": [
                      "target_program_behavior"
                    ]
                  },
                  "expr_vs_activity": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "locus_differential_gate",
            "role": "gate",
            "field": "locus_differential",
            "kind": "value",
            "expected": "uniform_across_models"
          },
          {
            "name": "formation_limited_by_gate",
            "role": "gate",
            "field": "formation_limited_by",
            "kind": "value",
            "expected": "absent_induction"
          },
          {
            "name": "locus_competent_in_naive_gate",
            "role": "gate",
            "field": "locus_competent_in_naive",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "ready_silent_fraction_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 3,
            "expected": [
              {
                "key": "B16_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_KPC_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "activator_chromatin_footprint_gate",
            "role": "gate",
            "field": "activator_chromatin_footprint",
            "kind": "value",
            "expected": "inconsistent_across_formers"
          },
          {
            "name": "exhaustion_comparable_gate",
            "role": "gate",
            "field": "exhaustion_comparable_across_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_nondeterminant_gate",
            "role": "gate",
            "field": "exhaustion_explains_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_separable_program_gate",
            "role": "gate",
            "field": "exhaustion_separable_program",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "exhaustion_counterexample_gate",
            "role": "gate",
            "field": "exhaustion_counterexample_model",
            "kind": "value",
            "expected": "KPC_GP"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "exhaustion_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "cf_inconsistent_present_without_activator",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "program",
            "entity": "present",
            "valueField": "activator_state",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_no_block_but_absent",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "none",
            "valueField": "program",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_late_block_without_poised_locus",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "late_activation",
            "valueField": "locus_state",
            "expected": [
              "closing_committed",
              "active"
            ]
          },
          {
            "name": "cf_inconsistent_early_block_with_brake_released",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "early_release",
            "valueField": "brake_state",
            "expected": [
              "released"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-sol-codex",
      "modelName": "gpt-5.6-sol",
      "harness": "openai-codex",
      "task": "green_2_tme_context_dependence",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 1106.800516,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "program_map",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "sig_includes_itgae",
            "role": "gate",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Itgae"
            ]
          },
          {
            "name": "declared_a_known_confounder",
            "role": "gate",
            "field": "$.excluded_confounders[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "cf_signature_gut_contaminant",
            "role": "hard_fail",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "generic_signature_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd103_program_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "<"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "per_model_regulatory_state",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "poised_unfired"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "late_activation"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "closing_committed"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_tuples",
            "kind": "list",
            "matchKey": "regulator",
            "k": 1,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "role": {
                    "op": "equals",
                    "values": [
                      "repressor"
                    ]
                  },
                  "gating_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  },
                  "evidence_basis": {
                    "op": "equals",
                    "values": [
                      "target_program_behavior"
                    ]
                  },
                  "expr_vs_activity": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "locus_differential_gate",
            "role": "gate",
            "field": "locus_differential",
            "kind": "value",
            "expected": "uniform_across_models"
          },
          {
            "name": "formation_limited_by_gate",
            "role": "gate",
            "field": "formation_limited_by",
            "kind": "value",
            "expected": "absent_induction"
          },
          {
            "name": "locus_competent_in_naive_gate",
            "role": "gate",
            "field": "locus_competent_in_naive",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "ready_silent_fraction_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 3,
            "expected": [
              {
                "key": "B16_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_KPC_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "activator_chromatin_footprint_gate",
            "role": "gate",
            "field": "activator_chromatin_footprint",
            "kind": "value",
            "expected": "inconsistent_across_formers"
          },
          {
            "name": "exhaustion_comparable_gate",
            "role": "gate",
            "field": "exhaustion_comparable_across_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_nondeterminant_gate",
            "role": "gate",
            "field": "exhaustion_explains_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_separable_program_gate",
            "role": "gate",
            "field": "exhaustion_separable_program",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "exhaustion_counterexample_gate",
            "role": "gate",
            "field": "exhaustion_counterexample_model",
            "kind": "value",
            "expected": "KPC_GP"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "exhaustion_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "cf_inconsistent_present_without_activator",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "program",
            "entity": "present",
            "valueField": "activator_state",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_no_block_but_absent",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "none",
            "valueField": "program",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_late_block_without_poised_locus",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "late_activation",
            "valueField": "locus_state",
            "expected": [
              "closing_committed",
              "active"
            ]
          },
          {
            "name": "cf_inconsistent_early_block_with_brake_released",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "early_release",
            "valueField": "brake_state",
            "expected": [
              "released"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-sol-codex",
      "modelName": "gpt-5.6-sol",
      "harness": "openai-codex",
      "task": "green_2_tme_context_dependence",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1818.361825,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "program_map",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "sig_includes_itgae",
            "role": "gate",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Itgae"
            ]
          },
          {
            "name": "declared_a_known_confounder",
            "role": "gate",
            "field": "$.excluded_confounders[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "cf_signature_gut_contaminant",
            "role": "hard_fail",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "generic_signature_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd103_program_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "<"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "per_model_regulatory_state",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "poised_unfired"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "late_activation"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "closing_committed"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_tuples",
            "kind": "list",
            "matchKey": "regulator",
            "k": 1,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "role": {
                    "op": "equals",
                    "values": [
                      "repressor"
                    ]
                  },
                  "gating_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  },
                  "evidence_basis": {
                    "op": "equals",
                    "values": [
                      "target_program_behavior"
                    ]
                  },
                  "expr_vs_activity": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "locus_differential_gate",
            "role": "gate",
            "field": "locus_differential",
            "kind": "value",
            "expected": "uniform_across_models"
          },
          {
            "name": "formation_limited_by_gate",
            "role": "gate",
            "field": "formation_limited_by",
            "kind": "value",
            "expected": "absent_induction"
          },
          {
            "name": "locus_competent_in_naive_gate",
            "role": "gate",
            "field": "locus_competent_in_naive",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "ready_silent_fraction_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 3,
            "expected": [
              {
                "key": "B16_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_KPC_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "activator_chromatin_footprint_gate",
            "role": "gate",
            "field": "activator_chromatin_footprint",
            "kind": "value",
            "expected": "inconsistent_across_formers"
          },
          {
            "name": "exhaustion_comparable_gate",
            "role": "gate",
            "field": "exhaustion_comparable_across_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_nondeterminant_gate",
            "role": "gate",
            "field": "exhaustion_explains_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_separable_program_gate",
            "role": "gate",
            "field": "exhaustion_separable_program",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "exhaustion_counterexample_gate",
            "role": "gate",
            "field": "exhaustion_counterexample_model",
            "kind": "value",
            "expected": "KPC_GP"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "exhaustion_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "cf_inconsistent_present_without_activator",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "program",
            "entity": "present",
            "valueField": "activator_state",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_no_block_but_absent",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "none",
            "valueField": "program",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_late_block_without_poised_locus",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "late_activation",
            "valueField": "locus_state",
            "expected": [
              "closing_committed",
              "active"
            ]
          },
          {
            "name": "cf_inconsistent_early_block_with_brake_released",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "early_release",
            "valueField": "brake_state",
            "expected": [
              "released"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-sol-codex",
      "modelName": "gpt-5.6-sol",
      "harness": "openai-codex",
      "task": "green_klf2_hidden_repressor_of_trm_like_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 448.65127,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "regulator_gate",
            "role": "gate",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "KLF2",
              "Klf2"
            ]
          },
          {
            "name": "controlling_node_character_gate",
            "role": "gate",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "high_differential_expression"
          },
          {
            "name": "expression_vs_activity_gate",
            "role": "gate",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "expression_exceeds_activity"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "target_program_partition",
            "kind": "list",
            "matchKey": "gene",
            "k": 7,
            "expected": [
              {
                "key": "S1pr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "S1pr4",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Klrg1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Cx3cr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Itgae",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Hic1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "P2rx7",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              }
            ]
          },
          {
            "name": "causal_status_gate",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "association_only_sufficiency_requires_perturbation"
          },
          {
            "name": "residency_control_logic_gate",
            "role": "gate",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_release_necessary_not_sufficient_multi_regulon"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "trm_like_program_by_model",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_absent"
                    ],
                    "canonical": "niche_absent"
                  }
                }
              }
            ]
          },
          {
            "name": "cf_regulator_activator_or_wrong_node",
            "role": "hard_fail",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "RUNX3",
              "Runx3",
              "PRDM1",
              "Prdm1",
              "BLIMP1",
              "Blimp1",
              "BHLHE40",
              "Bhlhe40",
              "ZNF683",
              "Zfp683",
              "HOBIT",
              "Hobit",
              "AHR",
              "Ahr",
              "BATF",
              "Batf",
              "TOX",
              "Tox",
              "NR4A1",
              "Nr4a1",
              "NR4A2",
              "Nr4a2",
              "NR4A3",
              "Nr4a3",
              "TCF7",
              "Tcf7",
              "TCF1",
              "Tcf1",
              "BACH2",
              "Bach2",
              "TBX21",
              "Tbx21",
              "TBET",
              "Tbet",
              "FLI1",
              "Fli1",
              "NR3C1",
              "Nr3c1",
              "EOMES",
              "Eomes",
              "ZEB2",
              "Zeb2",
              "ID2",
              "Id2",
              "ID3",
              "Id3",
              "IRF4",
              "Irf4"
            ]
          },
          {
            "name": "cf_node_character_hidden_driver_recall",
            "role": "hard_fail",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "hidden_driver_activity_not_expression"
          },
          {
            "name": "cf_effect_relation_hidden_pattern",
            "role": "hard_fail",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "activity_exceeds_expression"
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "causal_status",
            "kind": "value",
            "op": "in",
            "expected": [
              "sufficiency_established",
              "necessity_established",
              "causal_established_in_dataset",
              "sufficiency_via_perturbation_in_dataset"
            ]
          },
          {
            "name": "cf_residency_logic_activator_prior",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "residency_activator_induction_primary"
          },
          {
            "name": "cf_residency_logic_single_regulon",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_controls_entire_residency_program"
          },
          {
            "name": "cf_decoy_called_target",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_distractor_model_reported",
            "role": "hard_fail",
            "field": "$.trm_like_program_by_model[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_chromatin_only_decoy",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-sol-codex",
      "modelName": "gpt-5.6-sol",
      "harness": "openai-codex",
      "task": "green_klf2_hidden_repressor_of_trm_like_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 1264.067273,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "regulator_gate",
            "role": "gate",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "KLF2",
              "Klf2"
            ]
          },
          {
            "name": "controlling_node_character_gate",
            "role": "gate",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "high_differential_expression"
          },
          {
            "name": "expression_vs_activity_gate",
            "role": "gate",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "expression_exceeds_activity"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "target_program_partition",
            "kind": "list",
            "matchKey": "gene",
            "k": 7,
            "expected": [
              {
                "key": "S1pr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "S1pr4",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Klrg1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Cx3cr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Itgae",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Hic1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "P2rx7",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              }
            ]
          },
          {
            "name": "causal_status_gate",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "association_only_sufficiency_requires_perturbation"
          },
          {
            "name": "residency_control_logic_gate",
            "role": "gate",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_release_necessary_not_sufficient_multi_regulon"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "trm_like_program_by_model",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_absent"
                    ],
                    "canonical": "niche_absent"
                  }
                }
              }
            ]
          },
          {
            "name": "cf_regulator_activator_or_wrong_node",
            "role": "hard_fail",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "RUNX3",
              "Runx3",
              "PRDM1",
              "Prdm1",
              "BLIMP1",
              "Blimp1",
              "BHLHE40",
              "Bhlhe40",
              "ZNF683",
              "Zfp683",
              "HOBIT",
              "Hobit",
              "AHR",
              "Ahr",
              "BATF",
              "Batf",
              "TOX",
              "Tox",
              "NR4A1",
              "Nr4a1",
              "NR4A2",
              "Nr4a2",
              "NR4A3",
              "Nr4a3",
              "TCF7",
              "Tcf7",
              "TCF1",
              "Tcf1",
              "BACH2",
              "Bach2",
              "TBX21",
              "Tbx21",
              "TBET",
              "Tbet",
              "FLI1",
              "Fli1",
              "NR3C1",
              "Nr3c1",
              "EOMES",
              "Eomes",
              "ZEB2",
              "Zeb2",
              "ID2",
              "Id2",
              "ID3",
              "Id3",
              "IRF4",
              "Irf4"
            ]
          },
          {
            "name": "cf_node_character_hidden_driver_recall",
            "role": "hard_fail",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "hidden_driver_activity_not_expression"
          },
          {
            "name": "cf_effect_relation_hidden_pattern",
            "role": "hard_fail",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "activity_exceeds_expression"
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "causal_status",
            "kind": "value",
            "op": "in",
            "expected": [
              "sufficiency_established",
              "necessity_established",
              "causal_established_in_dataset",
              "sufficiency_via_perturbation_in_dataset"
            ]
          },
          {
            "name": "cf_residency_logic_activator_prior",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "residency_activator_induction_primary"
          },
          {
            "name": "cf_residency_logic_single_regulon",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_controls_entire_residency_program"
          },
          {
            "name": "cf_decoy_called_target",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_distractor_model_reported",
            "role": "hard_fail",
            "field": "$.trm_like_program_by_model[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_chromatin_only_decoy",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-sol-codex",
      "modelName": "gpt-5.6-sol",
      "harness": "openai-codex",
      "task": "green_klf2_hidden_repressor_of_trm_like_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 697.51953,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "regulator_gate",
            "role": "gate",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "KLF2",
              "Klf2"
            ]
          },
          {
            "name": "controlling_node_character_gate",
            "role": "gate",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "high_differential_expression"
          },
          {
            "name": "expression_vs_activity_gate",
            "role": "gate",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "expression_exceeds_activity"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "target_program_partition",
            "kind": "list",
            "matchKey": "gene",
            "k": 7,
            "expected": [
              {
                "key": "S1pr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "S1pr4",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Klrg1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Cx3cr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Itgae",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Hic1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "P2rx7",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              }
            ]
          },
          {
            "name": "causal_status_gate",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "association_only_sufficiency_requires_perturbation"
          },
          {
            "name": "residency_control_logic_gate",
            "role": "gate",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_release_necessary_not_sufficient_multi_regulon"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "trm_like_program_by_model",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_absent"
                    ],
                    "canonical": "niche_absent"
                  }
                }
              }
            ]
          },
          {
            "name": "cf_regulator_activator_or_wrong_node",
            "role": "hard_fail",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "RUNX3",
              "Runx3",
              "PRDM1",
              "Prdm1",
              "BLIMP1",
              "Blimp1",
              "BHLHE40",
              "Bhlhe40",
              "ZNF683",
              "Zfp683",
              "HOBIT",
              "Hobit",
              "AHR",
              "Ahr",
              "BATF",
              "Batf",
              "TOX",
              "Tox",
              "NR4A1",
              "Nr4a1",
              "NR4A2",
              "Nr4a2",
              "NR4A3",
              "Nr4a3",
              "TCF7",
              "Tcf7",
              "TCF1",
              "Tcf1",
              "BACH2",
              "Bach2",
              "TBX21",
              "Tbx21",
              "TBET",
              "Tbet",
              "FLI1",
              "Fli1",
              "NR3C1",
              "Nr3c1",
              "EOMES",
              "Eomes",
              "ZEB2",
              "Zeb2",
              "ID2",
              "Id2",
              "ID3",
              "Id3",
              "IRF4",
              "Irf4"
            ]
          },
          {
            "name": "cf_node_character_hidden_driver_recall",
            "role": "hard_fail",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "hidden_driver_activity_not_expression"
          },
          {
            "name": "cf_effect_relation_hidden_pattern",
            "role": "hard_fail",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "activity_exceeds_expression"
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "causal_status",
            "kind": "value",
            "op": "in",
            "expected": [
              "sufficiency_established",
              "necessity_established",
              "causal_established_in_dataset",
              "sufficiency_via_perturbation_in_dataset"
            ]
          },
          {
            "name": "cf_residency_logic_activator_prior",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "residency_activator_induction_primary"
          },
          {
            "name": "cf_residency_logic_single_regulon",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_controls_entire_residency_program"
          },
          {
            "name": "cf_decoy_called_target",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_distractor_model_reported",
            "role": "hard_fail",
            "field": "$.trm_like_program_by_model[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_chromatin_only_decoy",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-sol-codex",
      "modelName": "gpt-5.6-sol",
      "harness": "openai-codex",
      "task": "green_shared_vs_distinct_regulators_trm_vs_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 341.417621,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_program_map",
            "kind": "list",
            "matchKey": "regulator",
            "k": 3,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "both"
                    ],
                    "canonical": "both"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "represses"
                    ],
                    "canonical": "represses"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Batf",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "trm_like_til_specific"
                    ],
                    "canonical": "trm_like_til_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "chromatin_over_rna"
                    ],
                    "canonical": "chromatin_over_rna"
                  }
                }
              },
              {
                "key": "Ahr",
                "fields": {
                  "program_specificity": {
                    "op": "in",
                    "values": [
                      "trm_specific",
                      "both"
                    ],
                    "canonical": "trm_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "rna_over_chromatin"
                    ],
                    "canonical": "rna_over_chromatin"
                  }
                }
              }
            ]
          },
          {
            "name": "regulator_set_no_extras_gate",
            "role": "gate",
            "field": "$.regulator_program_map[*]",
            "kind": "other",
            "expected": null
          },
          {
            "name": "program_identity_conclusion_gate",
            "role": "gate",
            "field": "program_identity_conclusion",
            "kind": "value",
            "expected": "distinct_state_shared_brake_divergent_activators"
          },
          {
            "name": "residency_install_logic_gate",
            "role": "gate",
            "field": "residency_install_logic",
            "kind": "value",
            "expected": "brake_release_permits_context_matched_activator"
          },
          {
            "name": "activator_context_dependence_gate",
            "role": "gate",
            "field": "activator_context_dependence",
            "kind": "value",
            "expected": "context_matched_crossover"
          },
          {
            "name": "evidence_ceiling_gate",
            "role": "gate",
            "field": "evidence_ceiling",
            "kind": "value",
            "expected": "specificity_dissociations_require_perturbation"
          },
          {
            "name": "cf_install_logic_flat_or_single_step",
            "role": "hard_fail",
            "field": "residency_install_logic",
            "kind": "value",
            "op": "in",
            "expected": [
              "flat_shared_residency_toolkit",
              "activator_induction_sufficient",
              "brake_release_alone_sufficient"
            ]
          },
          {
            "name": "cf_program_identity_lumped_or_wrong",
            "role": "hard_fail",
            "field": "program_identity_conclusion",
            "kind": "value",
            "op": "in",
            "expected": [
              "trm_like_til_is_tumor_trm",
              "trm_like_til_is_exhausted_subset",
              "trm_like_til_is_effector_state",
              "trm_like_til_is_transitional_intermediate",
              "trm_like_til_is_shared_program_distinct_niche"
            ]
          },
          {
            "name": "cf_ahr_assigned_to_til",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Ahr",
            "valueField": "program_specificity",
            "expected": [
              "trm_like_til_specific"
            ]
          },
          {
            "name": "cf_batf_assigned_to_trm_or_shared",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Batf",
            "valueField": "program_specificity",
            "expected": [
              "trm_specific",
              "both"
            ]
          },
          {
            "name": "cf_klf2_called_activator",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Klf2",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          },
          {
            "name": "cf_evidence_ceiling_overclaim",
            "role": "hard_fail",
            "field": "evidence_ceiling",
            "kind": "value",
            "op": "in",
            "expected": [
              "specificity_established_from_regulon_activity",
              "specificity_established_in_dataset",
              "causation_established_by_cross_sectional_concordance",
              "necessity_and_sufficiency_established"
            ]
          },
          {
            "name": "cf_activator_context_shared_or_single",
            "role": "hard_fail",
            "field": "activator_context_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "shared_activators_both_contexts",
              "single_global_activator",
              "redundant_activators"
            ]
          },
          {
            "name": "cf_single_modality_rule",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_hic1_motif_brake_mislabel",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_runx3_anticanonical_as_controller",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Runx3",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-sol-codex",
      "modelName": "gpt-5.6-sol",
      "harness": "openai-codex",
      "task": "green_shared_vs_distinct_regulators_trm_vs_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 467.730985,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_program_map",
            "kind": "list",
            "matchKey": "regulator",
            "k": 3,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "both"
                    ],
                    "canonical": "both"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "represses"
                    ],
                    "canonical": "represses"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Batf",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "trm_like_til_specific"
                    ],
                    "canonical": "trm_like_til_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "chromatin_over_rna"
                    ],
                    "canonical": "chromatin_over_rna"
                  }
                }
              },
              {
                "key": "Ahr",
                "fields": {
                  "program_specificity": {
                    "op": "in",
                    "values": [
                      "trm_specific",
                      "both"
                    ],
                    "canonical": "trm_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "rna_over_chromatin"
                    ],
                    "canonical": "rna_over_chromatin"
                  }
                }
              }
            ]
          },
          {
            "name": "regulator_set_no_extras_gate",
            "role": "gate",
            "field": "$.regulator_program_map[*]",
            "kind": "other",
            "expected": null
          },
          {
            "name": "program_identity_conclusion_gate",
            "role": "gate",
            "field": "program_identity_conclusion",
            "kind": "value",
            "expected": "distinct_state_shared_brake_divergent_activators"
          },
          {
            "name": "residency_install_logic_gate",
            "role": "gate",
            "field": "residency_install_logic",
            "kind": "value",
            "expected": "brake_release_permits_context_matched_activator"
          },
          {
            "name": "activator_context_dependence_gate",
            "role": "gate",
            "field": "activator_context_dependence",
            "kind": "value",
            "expected": "context_matched_crossover"
          },
          {
            "name": "evidence_ceiling_gate",
            "role": "gate",
            "field": "evidence_ceiling",
            "kind": "value",
            "expected": "specificity_dissociations_require_perturbation"
          },
          {
            "name": "cf_install_logic_flat_or_single_step",
            "role": "hard_fail",
            "field": "residency_install_logic",
            "kind": "value",
            "op": "in",
            "expected": [
              "flat_shared_residency_toolkit",
              "activator_induction_sufficient",
              "brake_release_alone_sufficient"
            ]
          },
          {
            "name": "cf_program_identity_lumped_or_wrong",
            "role": "hard_fail",
            "field": "program_identity_conclusion",
            "kind": "value",
            "op": "in",
            "expected": [
              "trm_like_til_is_tumor_trm",
              "trm_like_til_is_exhausted_subset",
              "trm_like_til_is_effector_state",
              "trm_like_til_is_transitional_intermediate",
              "trm_like_til_is_shared_program_distinct_niche"
            ]
          },
          {
            "name": "cf_ahr_assigned_to_til",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Ahr",
            "valueField": "program_specificity",
            "expected": [
              "trm_like_til_specific"
            ]
          },
          {
            "name": "cf_batf_assigned_to_trm_or_shared",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Batf",
            "valueField": "program_specificity",
            "expected": [
              "trm_specific",
              "both"
            ]
          },
          {
            "name": "cf_klf2_called_activator",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Klf2",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          },
          {
            "name": "cf_evidence_ceiling_overclaim",
            "role": "hard_fail",
            "field": "evidence_ceiling",
            "kind": "value",
            "op": "in",
            "expected": [
              "specificity_established_from_regulon_activity",
              "specificity_established_in_dataset",
              "causation_established_by_cross_sectional_concordance",
              "necessity_and_sufficiency_established"
            ]
          },
          {
            "name": "cf_activator_context_shared_or_single",
            "role": "hard_fail",
            "field": "activator_context_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "shared_activators_both_contexts",
              "single_global_activator",
              "redundant_activators"
            ]
          },
          {
            "name": "cf_single_modality_rule",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_hic1_motif_brake_mislabel",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_runx3_anticanonical_as_controller",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Runx3",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-sol-codex",
      "modelName": "gpt-5.6-sol",
      "harness": "openai-codex",
      "task": "green_shared_vs_distinct_regulators_trm_vs_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 374.028981,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_program_map",
            "kind": "list",
            "matchKey": "regulator",
            "k": 3,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "both"
                    ],
                    "canonical": "both"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "represses"
                    ],
                    "canonical": "represses"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Batf",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "trm_like_til_specific"
                    ],
                    "canonical": "trm_like_til_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "chromatin_over_rna"
                    ],
                    "canonical": "chromatin_over_rna"
                  }
                }
              },
              {
                "key": "Ahr",
                "fields": {
                  "program_specificity": {
                    "op": "in",
                    "values": [
                      "trm_specific",
                      "both"
                    ],
                    "canonical": "trm_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "rna_over_chromatin"
                    ],
                    "canonical": "rna_over_chromatin"
                  }
                }
              }
            ]
          },
          {
            "name": "regulator_set_no_extras_gate",
            "role": "gate",
            "field": "$.regulator_program_map[*]",
            "kind": "other",
            "expected": null
          },
          {
            "name": "program_identity_conclusion_gate",
            "role": "gate",
            "field": "program_identity_conclusion",
            "kind": "value",
            "expected": "distinct_state_shared_brake_divergent_activators"
          },
          {
            "name": "residency_install_logic_gate",
            "role": "gate",
            "field": "residency_install_logic",
            "kind": "value",
            "expected": "brake_release_permits_context_matched_activator"
          },
          {
            "name": "activator_context_dependence_gate",
            "role": "gate",
            "field": "activator_context_dependence",
            "kind": "value",
            "expected": "context_matched_crossover"
          },
          {
            "name": "evidence_ceiling_gate",
            "role": "gate",
            "field": "evidence_ceiling",
            "kind": "value",
            "expected": "specificity_dissociations_require_perturbation"
          },
          {
            "name": "cf_install_logic_flat_or_single_step",
            "role": "hard_fail",
            "field": "residency_install_logic",
            "kind": "value",
            "op": "in",
            "expected": [
              "flat_shared_residency_toolkit",
              "activator_induction_sufficient",
              "brake_release_alone_sufficient"
            ]
          },
          {
            "name": "cf_program_identity_lumped_or_wrong",
            "role": "hard_fail",
            "field": "program_identity_conclusion",
            "kind": "value",
            "op": "in",
            "expected": [
              "trm_like_til_is_tumor_trm",
              "trm_like_til_is_exhausted_subset",
              "trm_like_til_is_effector_state",
              "trm_like_til_is_transitional_intermediate",
              "trm_like_til_is_shared_program_distinct_niche"
            ]
          },
          {
            "name": "cf_ahr_assigned_to_til",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Ahr",
            "valueField": "program_specificity",
            "expected": [
              "trm_like_til_specific"
            ]
          },
          {
            "name": "cf_batf_assigned_to_trm_or_shared",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Batf",
            "valueField": "program_specificity",
            "expected": [
              "trm_specific",
              "both"
            ]
          },
          {
            "name": "cf_klf2_called_activator",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Klf2",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          },
          {
            "name": "cf_evidence_ceiling_overclaim",
            "role": "hard_fail",
            "field": "evidence_ceiling",
            "kind": "value",
            "op": "in",
            "expected": [
              "specificity_established_from_regulon_activity",
              "specificity_established_in_dataset",
              "causation_established_by_cross_sectional_concordance",
              "necessity_and_sufficiency_established"
            ]
          },
          {
            "name": "cf_activator_context_shared_or_single",
            "role": "hard_fail",
            "field": "activator_context_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "shared_activators_both_contexts",
              "single_global_activator",
              "redundant_activators"
            ]
          },
          {
            "name": "cf_single_modality_rule",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_hic1_motif_brake_mislabel",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_runx3_anticanonical_as_controller",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Runx3",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-sol-codex",
      "modelName": "gpt-5.6-sol",
      "harness": "openai-codex",
      "task": "gse297023_q2_senescence_not_mechanism",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 1,
      "passed": true,
      "score": null,
      "durationS": 366.084674,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "candidate_is_senescence",
            "role": "gate",
            "field": "$.candidate_mechanism",
            "kind": "value",
            "op": "in",
            "expected": [
              "cellular_senescence",
              "oncogene_induced_senescence",
              "replicative_senescence"
            ]
          },
          {
            "name": "arrest_core_not_engaged",
            "role": "gate",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_not_engaged"
            ]
          },
          {
            "name": "secretory_arm_is_inflammaging",
            "role": "gate",
            "field": "$.secretory_arm_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "acute_phase_inflammaging_response"
            ]
          },
          {
            "name": "secretory_arm_broadly_elevated",
            "role": "gate",
            "field": "$.secretory_arm_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "broadly_elevated"
            ]
          },
          {
            "name": "carries_aging_program",
            "role": "gate",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "carries_bona_fide_aging_program_distinct_from_nominated"
            ]
          },
          {
            "name": "conclusion_senescence_unlikely",
            "role": "gate",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "unlikely_major_contributor"
            ]
          },
          {
            "name": "cf_arrest_core_engaged_recall",
            "role": "hard_fail",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_engaged"
            ]
          },
          {
            "name": "cf_aging_program_conflation_or_explain_away",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "aging_program_indistinguishable_from_nominated_program",
              "aging_program_is_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_no_aging_program_sensitivity_failure",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_detectable_aging_program"
            ]
          },
          {
            "name": "cf_senescence_driver_conclusion",
            "role": "hard_fail",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "likely_primary_driver",
              "plausible_supported_contributor"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-sol-codex",
      "modelName": "gpt-5.6-sol",
      "harness": "openai-codex",
      "task": "gse297023_q2_senescence_not_mechanism",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 697.449743,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "candidate_is_senescence",
            "role": "gate",
            "field": "$.candidate_mechanism",
            "kind": "value",
            "op": "in",
            "expected": [
              "cellular_senescence",
              "oncogene_induced_senescence",
              "replicative_senescence"
            ]
          },
          {
            "name": "arrest_core_not_engaged",
            "role": "gate",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_not_engaged"
            ]
          },
          {
            "name": "secretory_arm_is_inflammaging",
            "role": "gate",
            "field": "$.secretory_arm_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "acute_phase_inflammaging_response"
            ]
          },
          {
            "name": "secretory_arm_broadly_elevated",
            "role": "gate",
            "field": "$.secretory_arm_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "broadly_elevated"
            ]
          },
          {
            "name": "carries_aging_program",
            "role": "gate",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "carries_bona_fide_aging_program_distinct_from_nominated"
            ]
          },
          {
            "name": "conclusion_senescence_unlikely",
            "role": "gate",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "unlikely_major_contributor"
            ]
          },
          {
            "name": "cf_arrest_core_engaged_recall",
            "role": "hard_fail",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_engaged"
            ]
          },
          {
            "name": "cf_aging_program_conflation_or_explain_away",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "aging_program_indistinguishable_from_nominated_program",
              "aging_program_is_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_no_aging_program_sensitivity_failure",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_detectable_aging_program"
            ]
          },
          {
            "name": "cf_senescence_driver_conclusion",
            "role": "hard_fail",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "likely_primary_driver",
              "plausible_supported_contributor"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-sol-codex",
      "modelName": "gpt-5.6-sol",
      "harness": "openai-codex",
      "task": "gse297023_q2_senescence_not_mechanism",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 446.773991,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "candidate_is_senescence",
            "role": "gate",
            "field": "$.candidate_mechanism",
            "kind": "value",
            "op": "in",
            "expected": [
              "cellular_senescence",
              "oncogene_induced_senescence",
              "replicative_senescence"
            ]
          },
          {
            "name": "arrest_core_not_engaged",
            "role": "gate",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_not_engaged"
            ]
          },
          {
            "name": "secretory_arm_is_inflammaging",
            "role": "gate",
            "field": "$.secretory_arm_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "acute_phase_inflammaging_response"
            ]
          },
          {
            "name": "secretory_arm_broadly_elevated",
            "role": "gate",
            "field": "$.secretory_arm_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "broadly_elevated"
            ]
          },
          {
            "name": "carries_aging_program",
            "role": "gate",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "carries_bona_fide_aging_program_distinct_from_nominated"
            ]
          },
          {
            "name": "conclusion_senescence_unlikely",
            "role": "gate",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "unlikely_major_contributor"
            ]
          },
          {
            "name": "cf_arrest_core_engaged_recall",
            "role": "hard_fail",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_engaged"
            ]
          },
          {
            "name": "cf_aging_program_conflation_or_explain_away",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "aging_program_indistinguishable_from_nominated_program",
              "aging_program_is_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_no_aging_program_sensitivity_failure",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_detectable_aging_program"
            ]
          },
          {
            "name": "cf_senescence_driver_conclusion",
            "role": "hard_fail",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "likely_primary_driver",
              "plausible_supported_contributor"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-sol-codex",
      "modelName": "gpt-5.6-sol",
      "harness": "openai-codex",
      "task": "gse297023_q3_age_retained_mapk_reduced",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 1,
      "passed": true,
      "score": null,
      "durationS": 1065.439299,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "age_retained",
            "role": "gate",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_retained"
            ]
          },
          {
            "name": "identity_is_aging",
            "role": "gate",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_aging_signatures"
            ]
          },
          {
            "name": "nature_state_shift",
            "role": "gate",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_state_expression_shift"
            ]
          },
          {
            "name": "pathway_is_mapk",
            "role": "gate",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "mapk"
            ]
          },
          {
            "name": "pathway_reduced_in_aged",
            "role": "gate",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "reduced_in_aged"
            ]
          },
          {
            "name": "cf_mapk_increased_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_in_aged"
            ]
          },
          {
            "name": "cf_age_erased",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_erased"
            ]
          },
          {
            "name": "cf_pervasive_overcall",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_pervasive_artifactual"
            ]
          },
          {
            "name": "cf_explain_away_aging",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_compositional_or_identity",
            "role": "hard_fail",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_proportion_shift",
              "cell_identity_shift",
              "composition_dominant_state_secondary",
              "both_state_and_composition_equally"
            ]
          },
          {
            "name": "cf_biggest_inflammatory_signal_not_mapk",
            "role": "hard_fail",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "jak_stat",
              "nfkb",
              "tnfa"
            ]
          },
          {
            "name": "cf_senescence_misattribution",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_senescence_signature_specifically"
            ]
          },
          {
            "name": "cf_dismiss_as_batch_artifact",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_batch_or_technical_artifact"
            ]
          },
          {
            "name": "cf_mapk_comparable_constitutive_kras_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "comparable_across_ages"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-sol-codex",
      "modelName": "gpt-5.6-sol",
      "harness": "openai-codex",
      "task": "gse297023_q3_age_retained_mapk_reduced",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 2,
      "passed": true,
      "score": null,
      "durationS": 1039.780675,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "age_retained",
            "role": "gate",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_retained"
            ]
          },
          {
            "name": "identity_is_aging",
            "role": "gate",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_aging_signatures"
            ]
          },
          {
            "name": "nature_state_shift",
            "role": "gate",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_state_expression_shift"
            ]
          },
          {
            "name": "pathway_is_mapk",
            "role": "gate",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "mapk"
            ]
          },
          {
            "name": "pathway_reduced_in_aged",
            "role": "gate",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "reduced_in_aged"
            ]
          },
          {
            "name": "cf_mapk_increased_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_in_aged"
            ]
          },
          {
            "name": "cf_age_erased",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_erased"
            ]
          },
          {
            "name": "cf_pervasive_overcall",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_pervasive_artifactual"
            ]
          },
          {
            "name": "cf_explain_away_aging",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_compositional_or_identity",
            "role": "hard_fail",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_proportion_shift",
              "cell_identity_shift",
              "composition_dominant_state_secondary",
              "both_state_and_composition_equally"
            ]
          },
          {
            "name": "cf_biggest_inflammatory_signal_not_mapk",
            "role": "hard_fail",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "jak_stat",
              "nfkb",
              "tnfa"
            ]
          },
          {
            "name": "cf_senescence_misattribution",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_senescence_signature_specifically"
            ]
          },
          {
            "name": "cf_dismiss_as_batch_artifact",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_batch_or_technical_artifact"
            ]
          },
          {
            "name": "cf_mapk_comparable_constitutive_kras_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "comparable_across_ages"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-sol-codex",
      "modelName": "gpt-5.6-sol",
      "harness": "openai-codex",
      "task": "gse297023_q3_age_retained_mapk_reduced",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1205.266991,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "age_retained",
            "role": "gate",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_retained"
            ]
          },
          {
            "name": "identity_is_aging",
            "role": "gate",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_aging_signatures"
            ]
          },
          {
            "name": "nature_state_shift",
            "role": "gate",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_state_expression_shift"
            ]
          },
          {
            "name": "pathway_is_mapk",
            "role": "gate",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "mapk"
            ]
          },
          {
            "name": "pathway_reduced_in_aged",
            "role": "gate",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "reduced_in_aged"
            ]
          },
          {
            "name": "cf_mapk_increased_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_in_aged"
            ]
          },
          {
            "name": "cf_age_erased",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_erased"
            ]
          },
          {
            "name": "cf_pervasive_overcall",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_pervasive_artifactual"
            ]
          },
          {
            "name": "cf_explain_away_aging",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_compositional_or_identity",
            "role": "hard_fail",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_proportion_shift",
              "cell_identity_shift",
              "composition_dominant_state_secondary",
              "both_state_and_composition_equally"
            ]
          },
          {
            "name": "cf_biggest_inflammatory_signal_not_mapk",
            "role": "hard_fail",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "jak_stat",
              "nfkb",
              "tnfa"
            ]
          },
          {
            "name": "cf_senescence_misattribution",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_senescence_signature_specifically"
            ]
          },
          {
            "name": "cf_dismiss_as_batch_artifact",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_batch_or_technical_artifact"
            ]
          },
          {
            "name": "cf_mapk_comparable_constitutive_kras_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "comparable_across_ages"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-sol-codex",
      "modelName": "gpt-5.6-sol",
      "harness": "openai-codex",
      "task": "gse297023_q5_tme_age_dependent_remodeling",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 1,
      "passed": true,
      "score": null,
      "durationS": 1415.262709,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "perturbation_is_pten",
            "role": "gate",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "pten"
            ]
          },
          {
            "name": "tme_non_cell_autonomous",
            "role": "gate",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "substantial_non_cell_autonomous_remodeling"
            ]
          },
          {
            "name": "tme_state_not_composition",
            "role": "gate",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "tme_age_dependent",
            "role": "gate",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_dependent"
            ]
          },
          {
            "name": "tme_response_broader_in_aged",
            "role": "gate",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_aged"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "most_responsive_noncancer_cell_types",
            "kind": "list",
            "matchKey": "cell_type",
            "k": 2,
            "expected": [
              {
                "key": "alveolar_macrophage",
                "fields": {}
              },
              {
                "key": "endothelial",
                "fields": {}
              }
            ]
          },
          {
            "name": "cf_cell_intrinsic_only",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "negligible_cell_intrinsic_only_effect",
              "remodeling_unresolved",
              "modest_remodeling_few_cell_types"
            ]
          },
          {
            "name": "cf_remodeling_is_contamination",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "apparent_remodeling_reflects_cancer_contamination"
            ]
          },
          {
            "name": "cf_perturbation_design_misread",
            "role": "hard_fail",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "multiple_tumor_suppressors_pooled",
              "no_additional_gene_difference_kras_only",
              "rb1_cdkn2a_double",
              "pcna_essential_gene_control",
              "difference_is_oncogene_not_suppressor"
            ]
          },
          {
            "name": "cf_age_independent_tme",
            "role": "hard_fail",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_independent"
            ]
          },
          {
            "name": "cf_compositional_shift",
            "role": "hard_fail",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_frequency_shift",
              "compositional_immune_infiltration",
              "composition_dominant_state_secondary",
              "both_state_and_composition"
            ]
          },
          {
            "name": "cf_epithelial_or_cancer_substitution",
            "role": "hard_fail",
            "field": "$.most_responsive_noncancer_cell_types[*].cell_type",
            "kind": "avoid",
            "expected": [
              "AT2_cell",
              "AT1_cell",
              "cancer_cell",
              "ciliated",
              "secretory",
              "club_cell",
              "basal_cell",
              "goblet_cell"
            ]
          },
          {
            "name": "cf_tme_breadth_broader_in_young",
            "role": "hard_fail",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_young"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-sol-codex",
      "modelName": "gpt-5.6-sol",
      "harness": "openai-codex",
      "task": "gse297023_q5_tme_age_dependent_remodeling",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 2,
      "passed": true,
      "score": null,
      "durationS": 1574.566797,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "perturbation_is_pten",
            "role": "gate",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "pten"
            ]
          },
          {
            "name": "tme_non_cell_autonomous",
            "role": "gate",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "substantial_non_cell_autonomous_remodeling"
            ]
          },
          {
            "name": "tme_state_not_composition",
            "role": "gate",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "tme_age_dependent",
            "role": "gate",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_dependent"
            ]
          },
          {
            "name": "tme_response_broader_in_aged",
            "role": "gate",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_aged"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "most_responsive_noncancer_cell_types",
            "kind": "list",
            "matchKey": "cell_type",
            "k": 2,
            "expected": [
              {
                "key": "alveolar_macrophage",
                "fields": {}
              },
              {
                "key": "endothelial",
                "fields": {}
              }
            ]
          },
          {
            "name": "cf_cell_intrinsic_only",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "negligible_cell_intrinsic_only_effect",
              "remodeling_unresolved",
              "modest_remodeling_few_cell_types"
            ]
          },
          {
            "name": "cf_remodeling_is_contamination",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "apparent_remodeling_reflects_cancer_contamination"
            ]
          },
          {
            "name": "cf_perturbation_design_misread",
            "role": "hard_fail",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "multiple_tumor_suppressors_pooled",
              "no_additional_gene_difference_kras_only",
              "rb1_cdkn2a_double",
              "pcna_essential_gene_control",
              "difference_is_oncogene_not_suppressor"
            ]
          },
          {
            "name": "cf_age_independent_tme",
            "role": "hard_fail",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_independent"
            ]
          },
          {
            "name": "cf_compositional_shift",
            "role": "hard_fail",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_frequency_shift",
              "compositional_immune_infiltration",
              "composition_dominant_state_secondary",
              "both_state_and_composition"
            ]
          },
          {
            "name": "cf_epithelial_or_cancer_substitution",
            "role": "hard_fail",
            "field": "$.most_responsive_noncancer_cell_types[*].cell_type",
            "kind": "avoid",
            "expected": [
              "AT2_cell",
              "AT1_cell",
              "cancer_cell",
              "ciliated",
              "secretory",
              "club_cell",
              "basal_cell",
              "goblet_cell"
            ]
          },
          {
            "name": "cf_tme_breadth_broader_in_young",
            "role": "hard_fail",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_young"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-sol-codex",
      "modelName": "gpt-5.6-sol",
      "harness": "openai-codex",
      "task": "gse297023_q5_tme_age_dependent_remodeling",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 3,
      "passed": true,
      "score": null,
      "durationS": 1518.952934,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "perturbation_is_pten",
            "role": "gate",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "pten"
            ]
          },
          {
            "name": "tme_non_cell_autonomous",
            "role": "gate",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "substantial_non_cell_autonomous_remodeling"
            ]
          },
          {
            "name": "tme_state_not_composition",
            "role": "gate",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "tme_age_dependent",
            "role": "gate",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_dependent"
            ]
          },
          {
            "name": "tme_response_broader_in_aged",
            "role": "gate",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_aged"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "most_responsive_noncancer_cell_types",
            "kind": "list",
            "matchKey": "cell_type",
            "k": 2,
            "expected": [
              {
                "key": "alveolar_macrophage",
                "fields": {}
              },
              {
                "key": "endothelial",
                "fields": {}
              }
            ]
          },
          {
            "name": "cf_cell_intrinsic_only",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "negligible_cell_intrinsic_only_effect",
              "remodeling_unresolved",
              "modest_remodeling_few_cell_types"
            ]
          },
          {
            "name": "cf_remodeling_is_contamination",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "apparent_remodeling_reflects_cancer_contamination"
            ]
          },
          {
            "name": "cf_perturbation_design_misread",
            "role": "hard_fail",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "multiple_tumor_suppressors_pooled",
              "no_additional_gene_difference_kras_only",
              "rb1_cdkn2a_double",
              "pcna_essential_gene_control",
              "difference_is_oncogene_not_suppressor"
            ]
          },
          {
            "name": "cf_age_independent_tme",
            "role": "hard_fail",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_independent"
            ]
          },
          {
            "name": "cf_compositional_shift",
            "role": "hard_fail",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_frequency_shift",
              "compositional_immune_infiltration",
              "composition_dominant_state_secondary",
              "both_state_and_composition"
            ]
          },
          {
            "name": "cf_epithelial_or_cancer_substitution",
            "role": "hard_fail",
            "field": "$.most_responsive_noncancer_cell_types[*].cell_type",
            "kind": "avoid",
            "expected": [
              "AT2_cell",
              "AT1_cell",
              "cancer_cell",
              "ciliated",
              "secretory",
              "club_cell",
              "basal_cell",
              "goblet_cell"
            ]
          },
          {
            "name": "cf_tme_breadth_broader_in_young",
            "role": "hard_fail",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_young"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-sol-codex",
      "modelName": "gpt-5.6-sol",
      "harness": "openai-codex",
      "task": "gse297023_q6_age_immune_infiltration",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 1,
      "passed": true,
      "score": null,
      "durationS": 397.448268,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "composition_stable_with_age",
            "role": "gate",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "frequencies_largely_stable"
            ]
          },
          {
            "name": "age_signal_is_state_not_composition",
            "role": "gate",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "frequencies_uninformative_for_infiltration",
            "role": "gate",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_cannot_establish_infiltration"
            ]
          },
          {
            "name": "infiltration_increases_with_age",
            "role": "gate",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_with_age"
            ]
          },
          {
            "name": "infiltration_strength_calibrated",
            "role": "gate",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "significant_increase_strongest_markers_only",
              "consistent_increase_not_statistically_robust"
            ]
          },
          {
            "name": "genotype_marker_fraction_lower_in_pten",
            "role": "gate",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "lower_in_pten_deficient"
            ]
          },
          {
            "name": "cf_composition_shift_reading",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "major_compositional_shift_with_age",
              "increased_immune_fraction_with_age"
            ]
          },
          {
            "name": "cf_lineage_specific_composition_claim",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "myeloid_expansion_with_age",
              "lymphoid_expansion_with_age",
              "minor_shifts_in_some_cell_types"
            ]
          },
          {
            "name": "cf_immunosenescence_depletion_prior",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "immune_depletion_with_age"
            ]
          },
          {
            "name": "cf_age_modality_compositional",
            "role": "hard_fail",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_type_composition_shift",
              "both_equally",
              "composition_dominant_state_secondary"
            ]
          },
          {
            "name": "cf_overread_frequencies_as_infiltration",
            "role": "hard_fail",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_shows_no_infiltration_change",
              "single_cell_quantifies_infiltration",
              "frequencies_show_increased_infiltration",
              "frequencies_show_decreased_infiltration"
            ]
          },
          {
            "name": "cf_infiltration_decreases_immunosenescence",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "decreased_with_age"
            ]
          },
          {
            "name": "cf_infiltration_unchanged_scrna_overread",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "unchanged_with_age"
            ]
          },
          {
            "name": "cf_infiltration_indeterminate_unused_node",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "direction_indeterminate_from_data"
            ]
          },
          {
            "name": "cf_infiltration_mixed_direction",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "mixed_by_immune_cell_type"
            ]
          },
          {
            "name": "cf_infiltration_genotype_confusion",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_only_in_pten_deficient",
              "decreased_only_in_pten_deficient"
            ]
          },
          {
            "name": "cf_pten_infiltration_paper_text_recall",
            "role": "hard_fail",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "higher_in_pten_deficient",
              "higher_in_pten_deficient_only_in_young",
              "higher_in_pten_deficient_only_in_aged"
            ]
          },
          {
            "name": "cf_pseudoreplication_overclaim",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_increase_across_markers"
            ]
          },
          {
            "name": "cf_infiltration_strength_decrease_direction",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_decrease_across_markers",
              "significant_decrease_strongest_markers_only",
              "consistent_decrease_not_statistically_robust"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-sol-codex",
      "modelName": "gpt-5.6-sol",
      "harness": "openai-codex",
      "task": "gse297023_q6_age_immune_infiltration",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 2,
      "passed": true,
      "score": null,
      "durationS": 1072.278832,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "composition_stable_with_age",
            "role": "gate",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "frequencies_largely_stable"
            ]
          },
          {
            "name": "age_signal_is_state_not_composition",
            "role": "gate",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "frequencies_uninformative_for_infiltration",
            "role": "gate",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_cannot_establish_infiltration"
            ]
          },
          {
            "name": "infiltration_increases_with_age",
            "role": "gate",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_with_age"
            ]
          },
          {
            "name": "infiltration_strength_calibrated",
            "role": "gate",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "significant_increase_strongest_markers_only",
              "consistent_increase_not_statistically_robust"
            ]
          },
          {
            "name": "genotype_marker_fraction_lower_in_pten",
            "role": "gate",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "lower_in_pten_deficient"
            ]
          },
          {
            "name": "cf_composition_shift_reading",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "major_compositional_shift_with_age",
              "increased_immune_fraction_with_age"
            ]
          },
          {
            "name": "cf_lineage_specific_composition_claim",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "myeloid_expansion_with_age",
              "lymphoid_expansion_with_age",
              "minor_shifts_in_some_cell_types"
            ]
          },
          {
            "name": "cf_immunosenescence_depletion_prior",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "immune_depletion_with_age"
            ]
          },
          {
            "name": "cf_age_modality_compositional",
            "role": "hard_fail",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_type_composition_shift",
              "both_equally",
              "composition_dominant_state_secondary"
            ]
          },
          {
            "name": "cf_overread_frequencies_as_infiltration",
            "role": "hard_fail",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_shows_no_infiltration_change",
              "single_cell_quantifies_infiltration",
              "frequencies_show_increased_infiltration",
              "frequencies_show_decreased_infiltration"
            ]
          },
          {
            "name": "cf_infiltration_decreases_immunosenescence",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "decreased_with_age"
            ]
          },
          {
            "name": "cf_infiltration_unchanged_scrna_overread",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "unchanged_with_age"
            ]
          },
          {
            "name": "cf_infiltration_indeterminate_unused_node",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "direction_indeterminate_from_data"
            ]
          },
          {
            "name": "cf_infiltration_mixed_direction",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "mixed_by_immune_cell_type"
            ]
          },
          {
            "name": "cf_infiltration_genotype_confusion",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_only_in_pten_deficient",
              "decreased_only_in_pten_deficient"
            ]
          },
          {
            "name": "cf_pten_infiltration_paper_text_recall",
            "role": "hard_fail",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "higher_in_pten_deficient",
              "higher_in_pten_deficient_only_in_young",
              "higher_in_pten_deficient_only_in_aged"
            ]
          },
          {
            "name": "cf_pseudoreplication_overclaim",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_increase_across_markers"
            ]
          },
          {
            "name": "cf_infiltration_strength_decrease_direction",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_decrease_across_markers",
              "significant_decrease_strongest_markers_only",
              "consistent_decrease_not_statistically_robust"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-sol-codex",
      "modelName": "gpt-5.6-sol",
      "harness": "openai-codex",
      "task": "gse297023_q6_age_immune_infiltration",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 713.104841,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "composition_stable_with_age",
            "role": "gate",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "frequencies_largely_stable"
            ]
          },
          {
            "name": "age_signal_is_state_not_composition",
            "role": "gate",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "frequencies_uninformative_for_infiltration",
            "role": "gate",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_cannot_establish_infiltration"
            ]
          },
          {
            "name": "infiltration_increases_with_age",
            "role": "gate",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_with_age"
            ]
          },
          {
            "name": "infiltration_strength_calibrated",
            "role": "gate",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "significant_increase_strongest_markers_only",
              "consistent_increase_not_statistically_robust"
            ]
          },
          {
            "name": "genotype_marker_fraction_lower_in_pten",
            "role": "gate",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "lower_in_pten_deficient"
            ]
          },
          {
            "name": "cf_composition_shift_reading",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "major_compositional_shift_with_age",
              "increased_immune_fraction_with_age"
            ]
          },
          {
            "name": "cf_lineage_specific_composition_claim",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "myeloid_expansion_with_age",
              "lymphoid_expansion_with_age",
              "minor_shifts_in_some_cell_types"
            ]
          },
          {
            "name": "cf_immunosenescence_depletion_prior",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "immune_depletion_with_age"
            ]
          },
          {
            "name": "cf_age_modality_compositional",
            "role": "hard_fail",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_type_composition_shift",
              "both_equally",
              "composition_dominant_state_secondary"
            ]
          },
          {
            "name": "cf_overread_frequencies_as_infiltration",
            "role": "hard_fail",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_shows_no_infiltration_change",
              "single_cell_quantifies_infiltration",
              "frequencies_show_increased_infiltration",
              "frequencies_show_decreased_infiltration"
            ]
          },
          {
            "name": "cf_infiltration_decreases_immunosenescence",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "decreased_with_age"
            ]
          },
          {
            "name": "cf_infiltration_unchanged_scrna_overread",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "unchanged_with_age"
            ]
          },
          {
            "name": "cf_infiltration_indeterminate_unused_node",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "direction_indeterminate_from_data"
            ]
          },
          {
            "name": "cf_infiltration_mixed_direction",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "mixed_by_immune_cell_type"
            ]
          },
          {
            "name": "cf_infiltration_genotype_confusion",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_only_in_pten_deficient",
              "decreased_only_in_pten_deficient"
            ]
          },
          {
            "name": "cf_pten_infiltration_paper_text_recall",
            "role": "hard_fail",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "higher_in_pten_deficient",
              "higher_in_pten_deficient_only_in_young",
              "higher_in_pten_deficient_only_in_aged"
            ]
          },
          {
            "name": "cf_pseudoreplication_overclaim",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_increase_across_markers"
            ]
          },
          {
            "name": "cf_infiltration_strength_decrease_direction",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_decrease_across_markers",
              "significant_decrease_strongest_markers_only",
              "consistent_decrease_not_statistically_robust"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-terra-codex",
      "modelName": "gpt-5.6-terra",
      "harness": "openai-codex",
      "task": "gse155381_q4_pseudotime_delay",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 357.481986,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "pseudotime_direction_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "E17_chronological_match_weighted",
            "role": "additive",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pluripotency_exit_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pseudotime_direction_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_synchronous_with_monkey_EPI",
              "chimera_human_EPI_developmentally_advanced",
              "chimera_human_EPI_delayed_at_pre_implantation_only_resolved_post_implantation"
            ]
          },
          {
            "name": "E17_chronological_match_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "resembles_monkey_E17_Gast_EPI",
              "resembles_chimera_monkey_E17_EPI_specifically",
              "chimera_human_E17_EPI_more_advanced_than_monkey_E17_Gast"
            ]
          },
          {
            "name": "pluripotency_exit_paper_inverted_or_prior_driven",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_advances_pluripotency_exit_prematurely",
              "naive_pluripotency_program_specifically_engaged",
              "pluripotency_core_retained_via_KLF17_naive_marker_specifically"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-terra-codex",
      "modelName": "gpt-5.6-terra",
      "harness": "openai-codex",
      "task": "gse155381_q4_pseudotime_delay",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 359.571493,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "pseudotime_direction_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "E17_chronological_match_weighted",
            "role": "additive",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pluripotency_exit_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pseudotime_direction_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_synchronous_with_monkey_EPI",
              "chimera_human_EPI_developmentally_advanced",
              "chimera_human_EPI_delayed_at_pre_implantation_only_resolved_post_implantation"
            ]
          },
          {
            "name": "E17_chronological_match_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "resembles_monkey_E17_Gast_EPI",
              "resembles_chimera_monkey_E17_EPI_specifically",
              "chimera_human_E17_EPI_more_advanced_than_monkey_E17_Gast"
            ]
          },
          {
            "name": "pluripotency_exit_paper_inverted_or_prior_driven",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_advances_pluripotency_exit_prematurely",
              "naive_pluripotency_program_specifically_engaged",
              "pluripotency_core_retained_via_KLF17_naive_marker_specifically"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-terra-codex",
      "modelName": "gpt-5.6-terra",
      "harness": "openai-codex",
      "task": "gse155381_q4_pseudotime_delay",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 359.87738,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "pseudotime_direction_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "E17_chronological_match_weighted",
            "role": "additive",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pluripotency_exit_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pseudotime_direction_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_synchronous_with_monkey_EPI",
              "chimera_human_EPI_developmentally_advanced",
              "chimera_human_EPI_delayed_at_pre_implantation_only_resolved_post_implantation"
            ]
          },
          {
            "name": "E17_chronological_match_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "resembles_monkey_E17_Gast_EPI",
              "resembles_chimera_monkey_E17_EPI_specifically",
              "chimera_human_E17_EPI_more_advanced_than_monkey_E17_Gast"
            ]
          },
          {
            "name": "pluripotency_exit_paper_inverted_or_prior_driven",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_advances_pluripotency_exit_prematurely",
              "naive_pluripotency_program_specifically_engaged",
              "pluripotency_core_retained_via_KLF17_naive_marker_specifically"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-terra-codex",
      "modelName": "gpt-5.6-terra",
      "harness": "openai-codex",
      "task": "gse155381_q5_chimera_monkey_perturbation",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 304.88818,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "deg_direction_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "comparison_subset_choice_weighted",
            "role": "additive",
            "field": "$.comparison_subset_choice",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "panel_resolved_strong_target_ccn1_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN1",
              "CYR61"
            ]
          },
          {
            "name": "panel_resolved_strong_target_ccn2_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN2",
              "CTGF"
            ]
          },
          {
            "name": "hippo_pathway_weighted",
            "role": "additive",
            "field": "$.hippo_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "tgf_beta_pathway_weighted",
            "role": "additive",
            "field": "$.tgf_beta_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "deg_direction_batch_effect_uncorrected",
            "role": "hard_fail",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_dominated_global",
              "up_dominated_in_chosen_subset"
            ]
          },
          {
            "name": "silent_panel_dropout_paired_with_directional_pathway_claim",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "hippo_pathway_direction": {
                "op": "in",
                "values": [
                  "hippo_pathway_upregulated",
                  "hippo_pathway_unchanged",
                  "wnt_pathway_specifically_perturbed",
                  "stress_response_pathway_dominantly_engaged",
                  "no_pathway_signal_above_confound",
                  "yap_tead_direct_target_panel_downregulated",
                  "yap_tead_panel_dependent_direction",
                  "hippo_pathway_downregulated"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-terra-codex",
      "modelName": "gpt-5.6-terra",
      "harness": "openai-codex",
      "task": "gse155381_q5_chimera_monkey_perturbation",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 605.330237,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "deg_direction_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "comparison_subset_choice_weighted",
            "role": "additive",
            "field": "$.comparison_subset_choice",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "panel_resolved_strong_target_ccn1_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN1",
              "CYR61"
            ]
          },
          {
            "name": "panel_resolved_strong_target_ccn2_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN2",
              "CTGF"
            ]
          },
          {
            "name": "hippo_pathway_weighted",
            "role": "additive",
            "field": "$.hippo_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "tgf_beta_pathway_weighted",
            "role": "additive",
            "field": "$.tgf_beta_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "deg_direction_batch_effect_uncorrected",
            "role": "hard_fail",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_dominated_global",
              "up_dominated_in_chosen_subset"
            ]
          },
          {
            "name": "silent_panel_dropout_paired_with_directional_pathway_claim",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "hippo_pathway_direction": {
                "op": "in",
                "values": [
                  "hippo_pathway_upregulated",
                  "hippo_pathway_unchanged",
                  "wnt_pathway_specifically_perturbed",
                  "stress_response_pathway_dominantly_engaged",
                  "no_pathway_signal_above_confound",
                  "yap_tead_direct_target_panel_downregulated",
                  "yap_tead_panel_dependent_direction",
                  "hippo_pathway_downregulated"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-terra-codex",
      "modelName": "gpt-5.6-terra",
      "harness": "openai-codex",
      "task": "gse155381_q5_chimera_monkey_perturbation",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 379.375688,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "deg_direction_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "comparison_subset_choice_weighted",
            "role": "additive",
            "field": "$.comparison_subset_choice",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "panel_resolved_strong_target_ccn1_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN1",
              "CYR61"
            ]
          },
          {
            "name": "panel_resolved_strong_target_ccn2_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN2",
              "CTGF"
            ]
          },
          {
            "name": "hippo_pathway_weighted",
            "role": "additive",
            "field": "$.hippo_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "tgf_beta_pathway_weighted",
            "role": "additive",
            "field": "$.tgf_beta_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "deg_direction_batch_effect_uncorrected",
            "role": "hard_fail",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_dominated_global",
              "up_dominated_in_chosen_subset"
            ]
          },
          {
            "name": "silent_panel_dropout_paired_with_directional_pathway_claim",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "hippo_pathway_direction": {
                "op": "in",
                "values": [
                  "hippo_pathway_upregulated",
                  "hippo_pathway_unchanged",
                  "wnt_pathway_specifically_perturbed",
                  "stress_response_pathway_dominantly_engaged",
                  "no_pathway_signal_above_confound",
                  "yap_tead_direct_target_panel_downregulated",
                  "yap_tead_panel_dependent_direction",
                  "hippo_pathway_downregulated"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-terra-codex",
      "modelName": "gpt-5.6-terra",
      "harness": "openai-codex",
      "task": "gse155381_q7_xspecies_epi_communication",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 385.741425,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "richness_weighted",
            "role": "additive",
            "field": "$.xspecies_signaling_richness",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_weighted",
            "role": "additive",
            "field": "$.dominant_signaling_axis",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_wnt_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "WNT_FZD_LRP_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_fgf_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "FGF_FGFR_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_notch_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "NOTCH_DLL_JAG_family"
                ]
              }
            }
          },
          {
            "name": "signaling_directionality_weighted",
            "role": "additive",
            "field": "$.signaling_directionality",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "top_signal_interaction_class_weighted",
            "role": "additive",
            "field": "$.top_signal_interaction_class",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-terra-codex",
      "modelName": "gpt-5.6-terra",
      "harness": "openai-codex",
      "task": "gse155381_q7_xspecies_epi_communication",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 647.518966,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "richness_weighted",
            "role": "additive",
            "field": "$.xspecies_signaling_richness",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_weighted",
            "role": "additive",
            "field": "$.dominant_signaling_axis",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_wnt_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "WNT_FZD_LRP_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_fgf_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "FGF_FGFR_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_notch_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "NOTCH_DLL_JAG_family"
                ]
              }
            }
          },
          {
            "name": "signaling_directionality_weighted",
            "role": "additive",
            "field": "$.signaling_directionality",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "top_signal_interaction_class_weighted",
            "role": "additive",
            "field": "$.top_signal_interaction_class",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-terra-codex",
      "modelName": "gpt-5.6-terra",
      "harness": "openai-codex",
      "task": "gse155381_q7_xspecies_epi_communication",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 380.69024,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "richness_weighted",
            "role": "additive",
            "field": "$.xspecies_signaling_richness",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_weighted",
            "role": "additive",
            "field": "$.dominant_signaling_axis",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_wnt_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "WNT_FZD_LRP_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_fgf_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "FGF_FGFR_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_notch_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "NOTCH_DLL_JAG_family"
                ]
              }
            }
          },
          {
            "name": "signaling_directionality_weighted",
            "role": "additive",
            "field": "$.signaling_directionality",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "top_signal_interaction_class_weighted",
            "role": "additive",
            "field": "$.top_signal_interaction_class",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-terra-codex",
      "modelName": "gpt-5.6-terra",
      "harness": "openai-codex",
      "task": "gse155381_q8_bidirectional_convergence",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 246.569911,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "chimera_human_identity_shift_target_weighted",
            "role": "additive",
            "field": "$.chimera_human_identity_shift_target",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_in_vivo_state_assignment_weighted",
            "role": "additive",
            "field": "$.chimera_human_in_vivo_state_assignment",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_shift_magnitude_weighted",
            "role": "additive",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_monkey_identity_shift_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "both_arms_null_contradicted_by_calibration",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "chimera_human_identity_shift_target": {
                "op": "equals",
                "values": [
                  "chimera_human_maintains_origin_identity_no_measurable_shift"
                ]
              },
              "chimera_monkey_identity_shift": {
                "op": "equals",
                "values": [
                  "chimera_monkey_unchanged_relative_to_control"
                ]
              }
            }
          },
          {
            "name": "chimera_monkey_paper_claim_on_confounded_input",
            "role": "hard_fail",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "value",
            "expected": "chimera_monkey_shifts_toward_chimera_human_identity"
          },
          {
            "name": "chimera_human_magnitude_wrong_attribution_signature",
            "role": "hard_fail",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "value",
            "op": "in",
            "expected": [
              "dramatic_complete_replacement_of_origin_identity",
              "magnitude_dominated_by_culture_or_metabolic_axis",
              "magnitude_dominated_by_developmental_stage_axis",
              "magnitude_dominated_by_cell_cycle_axis",
              "magnitude_dominated_by_batch_effect_axis"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-terra-codex",
      "modelName": "gpt-5.6-terra",
      "harness": "openai-codex",
      "task": "gse155381_q8_bidirectional_convergence",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 489.861441,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "chimera_human_identity_shift_target_weighted",
            "role": "additive",
            "field": "$.chimera_human_identity_shift_target",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_in_vivo_state_assignment_weighted",
            "role": "additive",
            "field": "$.chimera_human_in_vivo_state_assignment",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_shift_magnitude_weighted",
            "role": "additive",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_monkey_identity_shift_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "both_arms_null_contradicted_by_calibration",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "chimera_human_identity_shift_target": {
                "op": "equals",
                "values": [
                  "chimera_human_maintains_origin_identity_no_measurable_shift"
                ]
              },
              "chimera_monkey_identity_shift": {
                "op": "equals",
                "values": [
                  "chimera_monkey_unchanged_relative_to_control"
                ]
              }
            }
          },
          {
            "name": "chimera_monkey_paper_claim_on_confounded_input",
            "role": "hard_fail",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "value",
            "expected": "chimera_monkey_shifts_toward_chimera_human_identity"
          },
          {
            "name": "chimera_human_magnitude_wrong_attribution_signature",
            "role": "hard_fail",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "value",
            "op": "in",
            "expected": [
              "dramatic_complete_replacement_of_origin_identity",
              "magnitude_dominated_by_culture_or_metabolic_axis",
              "magnitude_dominated_by_developmental_stage_axis",
              "magnitude_dominated_by_cell_cycle_axis",
              "magnitude_dominated_by_batch_effect_axis"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-terra-codex",
      "modelName": "gpt-5.6-terra",
      "harness": "openai-codex",
      "task": "gse155381_q8_bidirectional_convergence",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1233.970058,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "chimera_human_identity_shift_target_weighted",
            "role": "additive",
            "field": "$.chimera_human_identity_shift_target",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_in_vivo_state_assignment_weighted",
            "role": "additive",
            "field": "$.chimera_human_in_vivo_state_assignment",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_shift_magnitude_weighted",
            "role": "additive",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_monkey_identity_shift_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "both_arms_null_contradicted_by_calibration",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "chimera_human_identity_shift_target": {
                "op": "equals",
                "values": [
                  "chimera_human_maintains_origin_identity_no_measurable_shift"
                ]
              },
              "chimera_monkey_identity_shift": {
                "op": "equals",
                "values": [
                  "chimera_monkey_unchanged_relative_to_control"
                ]
              }
            }
          },
          {
            "name": "chimera_monkey_paper_claim_on_confounded_input",
            "role": "hard_fail",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "value",
            "expected": "chimera_monkey_shifts_toward_chimera_human_identity"
          },
          {
            "name": "chimera_human_magnitude_wrong_attribution_signature",
            "role": "hard_fail",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "value",
            "op": "in",
            "expected": [
              "dramatic_complete_replacement_of_origin_identity",
              "magnitude_dominated_by_culture_or_metabolic_axis",
              "magnitude_dominated_by_developmental_stage_axis",
              "magnitude_dominated_by_cell_cycle_axis",
              "magnitude_dominated_by_batch_effect_axis"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-terra-codex",
      "modelName": "gpt-5.6-terra",
      "harness": "openai-codex",
      "task": "gse171524_ectopic_airway_rare_cell",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 1,
      "passed": true,
      "score": null,
      "durationS": 197.757799,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "focus_population_identity_gate",
            "role": "gate",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "chemosensory_marker_expressing_population_alpha"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "focus_population_marker_calls",
            "kind": "list",
            "matchKey": "marker",
            "k": 3,
            "expected": [
              {
                "key": "POU2F3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "CHAT",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "TRPM5",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "LRMP",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNAT3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "AVIL",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNG13",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              }
            ]
          },
          {
            "name": "focus_population_condition_distribution_gate",
            "role": "gate",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_enriched_with_substantial_control_presence",
              "covid_predominant",
              "covid_only",
              "covid_enriched_trend_not_significant"
            ]
          },
          {
            "name": "focus_population_location_gate",
            "role": "gate",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "ectopic_parenchyma_in_covid",
              "parenchyma_only",
              "bronchiolized_or_peribronchial_parenchyma",
              "airway_and_parenchyma"
            ]
          },
          {
            "name": "focus_population_recovery_mode_gate",
            "role": "gate",
            "field": "$.focus_population_recovery_mode",
            "kind": "value",
            "op": "in",
            "expected": [
              "continuum_not_discrete_cluster"
            ]
          },
          {
            "name": "focus_population_identity_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "ionocyte_like_beta",
              "neuroendocrine_like_gamma",
              "basal_stem_progenitor_like_delta",
              "aberrant_basaloid_transitional_like_epsilon",
              "neuronal_or_neural_gene_minor_population_zeta",
              "deuterosomal_ciliated_progenitor_like_eta",
              "hillock_transitional_like_theta",
              "low_quality_doublet_or_debris",
              "no_rare_cell_recovered",
              "not_assessed"
            ]
          },
          {
            "name": "cf_wrong_lineage_marker_called_positive",
            "role": "hard_fail",
            "field": "$.focus_population_marker_calls[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "focus_population_condition_distribution_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "equally_distributed",
              "control_predominant",
              "not_assessed"
            ]
          },
          {
            "name": "focus_population_location_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "airway_only",
              "not_assessed"
            ]
          },
          {
            "name": "cross_axis_canonical_identity_with_airway_only_location_incoherence",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "focus_population_identity": {
                "op": "in",
                "values": [
                  "chemosensory_marker_expressing_population_alpha"
                ]
              },
              "focus_population_location": {
                "op": "in",
                "values": [
                  "airway_only"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-terra-codex",
      "modelName": "gpt-5.6-terra",
      "harness": "openai-codex",
      "task": "gse171524_ectopic_airway_rare_cell",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 2,
      "passed": true,
      "score": null,
      "durationS": 545.900923,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "focus_population_identity_gate",
            "role": "gate",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "chemosensory_marker_expressing_population_alpha"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "focus_population_marker_calls",
            "kind": "list",
            "matchKey": "marker",
            "k": 3,
            "expected": [
              {
                "key": "POU2F3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "CHAT",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "TRPM5",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "LRMP",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNAT3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "AVIL",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNG13",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              }
            ]
          },
          {
            "name": "focus_population_condition_distribution_gate",
            "role": "gate",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_enriched_with_substantial_control_presence",
              "covid_predominant",
              "covid_only",
              "covid_enriched_trend_not_significant"
            ]
          },
          {
            "name": "focus_population_location_gate",
            "role": "gate",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "ectopic_parenchyma_in_covid",
              "parenchyma_only",
              "bronchiolized_or_peribronchial_parenchyma",
              "airway_and_parenchyma"
            ]
          },
          {
            "name": "focus_population_recovery_mode_gate",
            "role": "gate",
            "field": "$.focus_population_recovery_mode",
            "kind": "value",
            "op": "in",
            "expected": [
              "continuum_not_discrete_cluster"
            ]
          },
          {
            "name": "focus_population_identity_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "ionocyte_like_beta",
              "neuroendocrine_like_gamma",
              "basal_stem_progenitor_like_delta",
              "aberrant_basaloid_transitional_like_epsilon",
              "neuronal_or_neural_gene_minor_population_zeta",
              "deuterosomal_ciliated_progenitor_like_eta",
              "hillock_transitional_like_theta",
              "low_quality_doublet_or_debris",
              "no_rare_cell_recovered",
              "not_assessed"
            ]
          },
          {
            "name": "cf_wrong_lineage_marker_called_positive",
            "role": "hard_fail",
            "field": "$.focus_population_marker_calls[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "focus_population_condition_distribution_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "equally_distributed",
              "control_predominant",
              "not_assessed"
            ]
          },
          {
            "name": "focus_population_location_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "airway_only",
              "not_assessed"
            ]
          },
          {
            "name": "cross_axis_canonical_identity_with_airway_only_location_incoherence",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "focus_population_identity": {
                "op": "in",
                "values": [
                  "chemosensory_marker_expressing_population_alpha"
                ]
              },
              "focus_population_location": {
                "op": "in",
                "values": [
                  "airway_only"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-terra-codex",
      "modelName": "gpt-5.6-terra",
      "harness": "openai-codex",
      "task": "gse171524_ectopic_airway_rare_cell",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 3,
      "passed": true,
      "score": null,
      "durationS": 409.407895,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "focus_population_identity_gate",
            "role": "gate",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "chemosensory_marker_expressing_population_alpha"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "focus_population_marker_calls",
            "kind": "list",
            "matchKey": "marker",
            "k": 3,
            "expected": [
              {
                "key": "POU2F3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "CHAT",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "TRPM5",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "LRMP",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNAT3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "AVIL",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNG13",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              }
            ]
          },
          {
            "name": "focus_population_condition_distribution_gate",
            "role": "gate",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_enriched_with_substantial_control_presence",
              "covid_predominant",
              "covid_only",
              "covid_enriched_trend_not_significant"
            ]
          },
          {
            "name": "focus_population_location_gate",
            "role": "gate",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "ectopic_parenchyma_in_covid",
              "parenchyma_only",
              "bronchiolized_or_peribronchial_parenchyma",
              "airway_and_parenchyma"
            ]
          },
          {
            "name": "focus_population_recovery_mode_gate",
            "role": "gate",
            "field": "$.focus_population_recovery_mode",
            "kind": "value",
            "op": "in",
            "expected": [
              "continuum_not_discrete_cluster"
            ]
          },
          {
            "name": "focus_population_identity_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "ionocyte_like_beta",
              "neuroendocrine_like_gamma",
              "basal_stem_progenitor_like_delta",
              "aberrant_basaloid_transitional_like_epsilon",
              "neuronal_or_neural_gene_minor_population_zeta",
              "deuterosomal_ciliated_progenitor_like_eta",
              "hillock_transitional_like_theta",
              "low_quality_doublet_or_debris",
              "no_rare_cell_recovered",
              "not_assessed"
            ]
          },
          {
            "name": "cf_wrong_lineage_marker_called_positive",
            "role": "hard_fail",
            "field": "$.focus_population_marker_calls[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "focus_population_condition_distribution_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "equally_distributed",
              "control_predominant",
              "not_assessed"
            ]
          },
          {
            "name": "focus_population_location_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "airway_only",
              "not_assessed"
            ]
          },
          {
            "name": "cross_axis_canonical_identity_with_airway_only_location_incoherence",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "focus_population_identity": {
                "op": "in",
                "values": [
                  "chemosensory_marker_expressing_population_alpha"
                ]
              },
              "focus_population_location": {
                "op": "in",
                "values": [
                  "airway_only"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-terra-codex",
      "modelName": "gpt-5.6-terra",
      "harness": "openai-codex",
      "task": "gse171524_pathological_fibroblast_master_regulator_druggable_targets",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 1,
      "passed": true,
      "score": null,
      "durationS": 362.621776,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "disease_associated_fibroblast_subtype_weighted",
            "role": "additive",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "mr_tf_junb_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_alpha"
            ]
          },
          {
            "name": "mr_tf_jund_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_beta"
            ]
          },
          {
            "name": "drug_mmp14_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_alpha"
            ]
          },
          {
            "name": "drug_stat3_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_beta"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_weighted",
            "role": "additive",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "disease_associated_fibroblast_subtype_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "value",
            "op": "in",
            "expected": [
              "subtype_lambda",
              "subtype_epsilon",
              "subtype_iota",
              "subtype_beta",
              "subtype_eta",
              "subtype_zeta",
              "subtype_not_assessed"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "simple_de_ranking_by_log_fc",
              "marker_based_de_only",
              "not_computed"
            ]
          },
          {
            "name": "top_master_regulator_transcription_factors_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_not_computed"
            ]
          },
          {
            "name": "top_druggable_proteins_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_not_assessed"
            ]
          },
          {
            "name": "cross_axis_circular_marker_in_druggable_proteins",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_gamma",
              "drug_target_delta",
              "drug_target_eta"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-terra-codex",
      "modelName": "gpt-5.6-terra",
      "harness": "openai-codex",
      "task": "gse171524_pathological_fibroblast_master_regulator_druggable_targets",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 2,
      "passed": true,
      "score": null,
      "durationS": 808.237934,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "disease_associated_fibroblast_subtype_weighted",
            "role": "additive",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "mr_tf_junb_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_alpha"
            ]
          },
          {
            "name": "mr_tf_jund_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_beta"
            ]
          },
          {
            "name": "drug_mmp14_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_alpha"
            ]
          },
          {
            "name": "drug_stat3_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_beta"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_weighted",
            "role": "additive",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "disease_associated_fibroblast_subtype_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "value",
            "op": "in",
            "expected": [
              "subtype_lambda",
              "subtype_epsilon",
              "subtype_iota",
              "subtype_beta",
              "subtype_eta",
              "subtype_zeta",
              "subtype_not_assessed"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "simple_de_ranking_by_log_fc",
              "marker_based_de_only",
              "not_computed"
            ]
          },
          {
            "name": "top_master_regulator_transcription_factors_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_not_computed"
            ]
          },
          {
            "name": "top_druggable_proteins_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_not_assessed"
            ]
          },
          {
            "name": "cross_axis_circular_marker_in_druggable_proteins",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_gamma",
              "drug_target_delta",
              "drug_target_eta"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-terra-codex",
      "modelName": "gpt-5.6-terra",
      "harness": "openai-codex",
      "task": "gse171524_pathological_fibroblast_master_regulator_druggable_targets",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 3,
      "passed": true,
      "score": null,
      "durationS": 645.689435,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "disease_associated_fibroblast_subtype_weighted",
            "role": "additive",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "mr_tf_junb_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_alpha"
            ]
          },
          {
            "name": "mr_tf_jund_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_beta"
            ]
          },
          {
            "name": "drug_mmp14_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_alpha"
            ]
          },
          {
            "name": "drug_stat3_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_beta"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_weighted",
            "role": "additive",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "disease_associated_fibroblast_subtype_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "value",
            "op": "in",
            "expected": [
              "subtype_lambda",
              "subtype_epsilon",
              "subtype_iota",
              "subtype_beta",
              "subtype_eta",
              "subtype_zeta",
              "subtype_not_assessed"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "simple_de_ranking_by_log_fc",
              "marker_based_de_only",
              "not_computed"
            ]
          },
          {
            "name": "top_master_regulator_transcription_factors_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_not_computed"
            ]
          },
          {
            "name": "top_druggable_proteins_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_not_assessed"
            ]
          },
          {
            "name": "cross_axis_circular_marker_in_druggable_proteins",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_gamma",
              "drug_target_delta",
              "drug_target_eta"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-terra-codex",
      "modelName": "gpt-5.6-terra",
      "harness": "openai-codex",
      "task": "gse171524_plasma_bcr_neutralizing_clonotype",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 280.742463,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dominant_pairing_heavy_gate",
            "role": "gate",
            "field": "$.dominant_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV4-34"
            ]
          },
          {
            "name": "dominant_pairing_light_gate",
            "role": "gate",
            "field": "$.dominant_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGLV3-19"
            ]
          },
          {
            "name": "dominant_neutralizer_match_gate",
            "role": "gate",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_known_neutralizer_match"
            ]
          },
          {
            "name": "neutralizer_pairing_heavy_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV1-18"
            ]
          },
          {
            "name": "neutralizer_pairing_light_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGKV3-20"
            ]
          },
          {
            "name": "neutralizer_recurrence_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "observed_in_2_to_5_donors"
            ]
          },
          {
            "name": "neutralizer_rank_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "top_5_pairing",
              "top_6_to_20_pairing"
            ]
          },
          {
            "name": "dominant_clone_isotype_gate",
            "role": "gate",
            "field": "$.dominant_clone_isotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "IgG"
            ]
          },
          {
            "name": "blineage_class_switch_contrast_gate",
            "role": "gate",
            "field": "$.blineage_class_switch_contrast",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_class_switched_control_higher_unswitched",
              "class_switched_in_both_cohorts"
            ]
          },
          {
            "name": "dominant_match_narrative_trap_or_null",
            "role": "hard_fail",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_rbd_neutralizer",
              "matches_known_autoreactive_clone",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_heavy",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_light",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_recurrence_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_rank_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "cross_axis_found_dominant_but_failed_to_scan_for_embedded_neutralizer",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_pairing_neutralizer_match": {
                "op": "in",
                "values": [
                  "no_known_neutralizer_match"
                ]
              },
              "neutralizer_pairing_heavy": {
                "op": "in",
                "values": [
                  "none_detected"
                ]
              }
            }
          },
          {
            "name": "neutralizer_pairing_not_exactly_the_s309_pair",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "op": "not",
            "expected": {
              "op": "and",
              "args": [
                {
                  "op": "field",
                  "name": "neutralizer_pairing_heavy",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGHV1-18"
                    ]
                  }
                },
                {
                  "op": "field",
                  "name": "neutralizer_pairing_light",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGKV3-20"
                    ]
                  }
                }
              ]
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-terra-codex",
      "modelName": "gpt-5.6-terra",
      "harness": "openai-codex",
      "task": "gse171524_plasma_bcr_neutralizing_clonotype",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 164.814753,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dominant_pairing_heavy_gate",
            "role": "gate",
            "field": "$.dominant_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV4-34"
            ]
          },
          {
            "name": "dominant_pairing_light_gate",
            "role": "gate",
            "field": "$.dominant_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGLV3-19"
            ]
          },
          {
            "name": "dominant_neutralizer_match_gate",
            "role": "gate",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_known_neutralizer_match"
            ]
          },
          {
            "name": "neutralizer_pairing_heavy_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV1-18"
            ]
          },
          {
            "name": "neutralizer_pairing_light_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGKV3-20"
            ]
          },
          {
            "name": "neutralizer_recurrence_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "observed_in_2_to_5_donors"
            ]
          },
          {
            "name": "neutralizer_rank_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "top_5_pairing",
              "top_6_to_20_pairing"
            ]
          },
          {
            "name": "dominant_clone_isotype_gate",
            "role": "gate",
            "field": "$.dominant_clone_isotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "IgG"
            ]
          },
          {
            "name": "blineage_class_switch_contrast_gate",
            "role": "gate",
            "field": "$.blineage_class_switch_contrast",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_class_switched_control_higher_unswitched",
              "class_switched_in_both_cohorts"
            ]
          },
          {
            "name": "dominant_match_narrative_trap_or_null",
            "role": "hard_fail",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_rbd_neutralizer",
              "matches_known_autoreactive_clone",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_heavy",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_light",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_recurrence_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_rank_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "cross_axis_found_dominant_but_failed_to_scan_for_embedded_neutralizer",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_pairing_neutralizer_match": {
                "op": "in",
                "values": [
                  "no_known_neutralizer_match"
                ]
              },
              "neutralizer_pairing_heavy": {
                "op": "in",
                "values": [
                  "none_detected"
                ]
              }
            }
          },
          {
            "name": "neutralizer_pairing_not_exactly_the_s309_pair",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "op": "not",
            "expected": {
              "op": "and",
              "args": [
                {
                  "op": "field",
                  "name": "neutralizer_pairing_heavy",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGHV1-18"
                    ]
                  }
                },
                {
                  "op": "field",
                  "name": "neutralizer_pairing_light",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGKV3-20"
                    ]
                  }
                }
              ]
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-terra-codex",
      "modelName": "gpt-5.6-terra",
      "harness": "openai-codex",
      "task": "gse171524_plasma_bcr_neutralizing_clonotype",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 184.574708,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dominant_pairing_heavy_gate",
            "role": "gate",
            "field": "$.dominant_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV4-34"
            ]
          },
          {
            "name": "dominant_pairing_light_gate",
            "role": "gate",
            "field": "$.dominant_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGLV3-19"
            ]
          },
          {
            "name": "dominant_neutralizer_match_gate",
            "role": "gate",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_known_neutralizer_match"
            ]
          },
          {
            "name": "neutralizer_pairing_heavy_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV1-18"
            ]
          },
          {
            "name": "neutralizer_pairing_light_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGKV3-20"
            ]
          },
          {
            "name": "neutralizer_recurrence_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "observed_in_2_to_5_donors"
            ]
          },
          {
            "name": "neutralizer_rank_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "top_5_pairing",
              "top_6_to_20_pairing"
            ]
          },
          {
            "name": "dominant_clone_isotype_gate",
            "role": "gate",
            "field": "$.dominant_clone_isotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "IgG"
            ]
          },
          {
            "name": "blineage_class_switch_contrast_gate",
            "role": "gate",
            "field": "$.blineage_class_switch_contrast",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_class_switched_control_higher_unswitched",
              "class_switched_in_both_cohorts"
            ]
          },
          {
            "name": "dominant_match_narrative_trap_or_null",
            "role": "hard_fail",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_rbd_neutralizer",
              "matches_known_autoreactive_clone",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_heavy",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_light",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_recurrence_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_rank_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "cross_axis_found_dominant_but_failed_to_scan_for_embedded_neutralizer",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_pairing_neutralizer_match": {
                "op": "in",
                "values": [
                  "no_known_neutralizer_match"
                ]
              },
              "neutralizer_pairing_heavy": {
                "op": "in",
                "values": [
                  "none_detected"
                ]
              }
            }
          },
          {
            "name": "neutralizer_pairing_not_exactly_the_s309_pair",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "op": "not",
            "expected": {
              "op": "and",
              "args": [
                {
                  "op": "field",
                  "name": "neutralizer_pairing_heavy",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGHV1-18"
                    ]
                  }
                },
                {
                  "op": "field",
                  "name": "neutralizer_pairing_light",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGKV3-20"
                    ]
                  }
                }
              ]
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-terra-codex",
      "modelName": "gpt-5.6-terra",
      "harness": "openai-codex",
      "task": "q6_myeloid_subset_restricted_de",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 1,
      "passed": true,
      "score": null,
      "durationS": 707.438605,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "myeloid_subset_with_strongest_change_gate",
            "role": "gate",
            "field": "$.myeloid_subset_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "subset_gamma"
            ]
          },
          {
            "name": "gene_with_strongest_change_gate",
            "role": "gate",
            "field": "$.gene_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "AXL",
              "PROS1"
            ]
          },
          {
            "name": "direction_of_change_gate",
            "role": "gate",
            "field": "$.direction_of_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "down_in_covid"
            ]
          },
          {
            "name": "biological_function_interpretation_gate",
            "role": "gate",
            "field": "$.biological_function_interpretation",
            "kind": "value",
            "op": "in",
            "expected": [
              "efferocytosis",
              "phagocytosis"
            ]
          },
          {
            "name": "myeloid_compartment_abundance_gate",
            "role": "gate",
            "field": "$.myeloid_compartment_abundance",
            "kind": "value",
            "op": "in",
            "expected": [
              "expands_in_covid"
            ]
          },
          {
            "name": "myeloid_expansion_localization_gate",
            "role": "gate",
            "field": "$.myeloid_expansion_localization",
            "kind": "value",
            "op": "in",
            "expected": [
              "recruited_driven_resident_dysfunction"
            ]
          },
          {
            "name": "dominant_signature_gate",
            "role": "gate",
            "field": "$.dominant_signature",
            "kind": "value",
            "op": "in",
            "expected": [
              "technical_or_agonal_artifact"
            ]
          },
          {
            "name": "dominant_signature_direction_gate",
            "role": "gate",
            "field": "$.dominant_signature_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_in_covid"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-terra-codex",
      "modelName": "gpt-5.6-terra",
      "harness": "openai-codex",
      "task": "q6_myeloid_subset_restricted_de",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 2,
      "passed": true,
      "score": null,
      "durationS": 502.638037,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "myeloid_subset_with_strongest_change_gate",
            "role": "gate",
            "field": "$.myeloid_subset_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "subset_gamma"
            ]
          },
          {
            "name": "gene_with_strongest_change_gate",
            "role": "gate",
            "field": "$.gene_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "AXL",
              "PROS1"
            ]
          },
          {
            "name": "direction_of_change_gate",
            "role": "gate",
            "field": "$.direction_of_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "down_in_covid"
            ]
          },
          {
            "name": "biological_function_interpretation_gate",
            "role": "gate",
            "field": "$.biological_function_interpretation",
            "kind": "value",
            "op": "in",
            "expected": [
              "efferocytosis",
              "phagocytosis"
            ]
          },
          {
            "name": "myeloid_compartment_abundance_gate",
            "role": "gate",
            "field": "$.myeloid_compartment_abundance",
            "kind": "value",
            "op": "in",
            "expected": [
              "expands_in_covid"
            ]
          },
          {
            "name": "myeloid_expansion_localization_gate",
            "role": "gate",
            "field": "$.myeloid_expansion_localization",
            "kind": "value",
            "op": "in",
            "expected": [
              "recruited_driven_resident_dysfunction"
            ]
          },
          {
            "name": "dominant_signature_gate",
            "role": "gate",
            "field": "$.dominant_signature",
            "kind": "value",
            "op": "in",
            "expected": [
              "technical_or_agonal_artifact"
            ]
          },
          {
            "name": "dominant_signature_direction_gate",
            "role": "gate",
            "field": "$.dominant_signature_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_in_covid"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-terra-codex",
      "modelName": "gpt-5.6-terra",
      "harness": "openai-codex",
      "task": "q6_myeloid_subset_restricted_de",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 3,
      "passed": true,
      "score": null,
      "durationS": 360.012286,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "myeloid_subset_with_strongest_change_gate",
            "role": "gate",
            "field": "$.myeloid_subset_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "subset_gamma"
            ]
          },
          {
            "name": "gene_with_strongest_change_gate",
            "role": "gate",
            "field": "$.gene_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "AXL",
              "PROS1"
            ]
          },
          {
            "name": "direction_of_change_gate",
            "role": "gate",
            "field": "$.direction_of_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "down_in_covid"
            ]
          },
          {
            "name": "biological_function_interpretation_gate",
            "role": "gate",
            "field": "$.biological_function_interpretation",
            "kind": "value",
            "op": "in",
            "expected": [
              "efferocytosis",
              "phagocytosis"
            ]
          },
          {
            "name": "myeloid_compartment_abundance_gate",
            "role": "gate",
            "field": "$.myeloid_compartment_abundance",
            "kind": "value",
            "op": "in",
            "expected": [
              "expands_in_covid"
            ]
          },
          {
            "name": "myeloid_expansion_localization_gate",
            "role": "gate",
            "field": "$.myeloid_expansion_localization",
            "kind": "value",
            "op": "in",
            "expected": [
              "recruited_driven_resident_dysfunction"
            ]
          },
          {
            "name": "dominant_signature_gate",
            "role": "gate",
            "field": "$.dominant_signature",
            "kind": "value",
            "op": "in",
            "expected": [
              "technical_or_agonal_artifact"
            ]
          },
          {
            "name": "dominant_signature_direction_gate",
            "role": "gate",
            "field": "$.dominant_signature_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_in_covid"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-terra-codex",
      "modelName": "gpt-5.6-terra",
      "harness": "openai-codex",
      "task": "snrnaseq_per_donor_composition_shifts_lethal_covid_lung",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 1,
      "passed": true,
      "score": null,
      "durationS": 412.98837,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dict_match",
            "role": "gate",
            "field": "cell_type_direction_map",
            "kind": "dict",
            "expected": {
              "Myeloid": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "Epithelia": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Fibroblasts": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "T_NK": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "B_plasma": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Endothelial": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Mast": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "APC": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Neuronal": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              }
            }
          },
          {
            "name": "over_merge_immune_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Immune",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_leukocyte_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Leukocyte",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_lymphoid_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Lymphoid",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_stromal_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Stromal",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-terra-codex",
      "modelName": "gpt-5.6-terra",
      "harness": "openai-codex",
      "task": "snrnaseq_per_donor_composition_shifts_lethal_covid_lung",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 506.088936,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dict_match",
            "role": "gate",
            "field": "cell_type_direction_map",
            "kind": "dict",
            "expected": {
              "Myeloid": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "Epithelia": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Fibroblasts": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "T_NK": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "B_plasma": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Endothelial": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Mast": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "APC": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Neuronal": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              }
            }
          },
          {
            "name": "over_merge_immune_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Immune",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_leukocyte_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Leukocyte",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_lymphoid_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Lymphoid",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_stromal_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Stromal",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-terra-codex",
      "modelName": "gpt-5.6-terra",
      "harness": "openai-codex",
      "task": "snrnaseq_per_donor_composition_shifts_lethal_covid_lung",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 602.401799,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dict_match",
            "role": "gate",
            "field": "cell_type_direction_map",
            "kind": "dict",
            "expected": {
              "Myeloid": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "Epithelia": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Fibroblasts": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "T_NK": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "B_plasma": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Endothelial": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Mast": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "APC": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Neuronal": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              }
            }
          },
          {
            "name": "over_merge_immune_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Immune",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_leukocyte_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Leukocyte",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_lymphoid_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Lymphoid",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_stromal_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Stromal",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-terra-codex",
      "modelName": "gpt-5.6-terra",
      "harness": "openai-codex",
      "task": "10x_5p_melanoma_cluster_enriched_apc_states_v2",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 581.240337,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "top_3_cluster_enriched_apc_states",
            "kind": "list",
            "matchKey": "apc_state",
            "k": 3,
            "expected": [
              {
                "key": "plasmacytoid_DC",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "T_cell_stimulatory_role_supported",
                      "role_direction_unresolved"
                    ],
                    "canonical": "T_cell_stimulatory_role_supported"
                  }
                }
              },
              {
                "key": "C1qhigh_lipid_associated_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "C1qhigh_inflammatory_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "plasma_cell",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "cohort_significant_per_patient_inconsistent"
                    ],
                    "canonical": "cohort_significant_per_patient_inconsistent"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CD80_CD86": {
                "op": "in",
                "value": [
                  "mreg_DC",
                  "shared_across_APC_states"
                ]
              },
              "PD_L1_in_APC_clusters": {
                "op": "in",
                "value": [
                  "C1qhigh_macrophage_either_subtype",
                  "shared_across_APC_states",
                  "mreg_DC"
                ]
              },
              "type_I_IFN": {
                "op": "in",
                "value": [
                  "plasmacytoid_DC",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "cf_mreg_dc_claimed_enriched",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "mreg_DC",
              "mature_DC_enriched_in_immunoregulatory_molecules"
            ]
          },
          {
            "name": "cf_framework_offcompartment_or_generic_substitution",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "cDC1",
              "Maier_mregDC",
              "Cheng_C1QC_TAM",
              "Cheng_SPP1_TAM",
              "Cheng_FOLR2_TAM",
              "Villani_pDC",
              "Zhang_inflammatory_monocyte",
              "LAMP3_pos_DC",
              "myeloid_derived_suppressor_cell",
              "MDSC",
              "tumor_associated_neutrophil",
              "mast_cell",
              "APC",
              "antigen_presenting_cell",
              "macrophage",
              "dendritic_cell",
              "tumor_associated_macrophage",
              "TAM"
            ]
          },
          {
            "name": "cf_direction_commit_on_unresolvable_slot",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_not_resolvable_hedge",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].functional_attribution_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_type_I_IFN_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.type_I_IFN",
            "kind": "value",
            "op": "in",
            "expected": [
              "mreg_DC",
              "C1qhigh_macrophage_either_subtype"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-terra-codex",
      "modelName": "gpt-5.6-terra",
      "harness": "openai-codex",
      "task": "10x_5p_melanoma_cluster_enriched_apc_states_v2",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 641.455442,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "top_3_cluster_enriched_apc_states",
            "kind": "list",
            "matchKey": "apc_state",
            "k": 3,
            "expected": [
              {
                "key": "plasmacytoid_DC",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "T_cell_stimulatory_role_supported",
                      "role_direction_unresolved"
                    ],
                    "canonical": "T_cell_stimulatory_role_supported"
                  }
                }
              },
              {
                "key": "C1qhigh_lipid_associated_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "C1qhigh_inflammatory_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "plasma_cell",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "cohort_significant_per_patient_inconsistent"
                    ],
                    "canonical": "cohort_significant_per_patient_inconsistent"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CD80_CD86": {
                "op": "in",
                "value": [
                  "mreg_DC",
                  "shared_across_APC_states"
                ]
              },
              "PD_L1_in_APC_clusters": {
                "op": "in",
                "value": [
                  "C1qhigh_macrophage_either_subtype",
                  "shared_across_APC_states",
                  "mreg_DC"
                ]
              },
              "type_I_IFN": {
                "op": "in",
                "value": [
                  "plasmacytoid_DC",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "cf_mreg_dc_claimed_enriched",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "mreg_DC",
              "mature_DC_enriched_in_immunoregulatory_molecules"
            ]
          },
          {
            "name": "cf_framework_offcompartment_or_generic_substitution",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "cDC1",
              "Maier_mregDC",
              "Cheng_C1QC_TAM",
              "Cheng_SPP1_TAM",
              "Cheng_FOLR2_TAM",
              "Villani_pDC",
              "Zhang_inflammatory_monocyte",
              "LAMP3_pos_DC",
              "myeloid_derived_suppressor_cell",
              "MDSC",
              "tumor_associated_neutrophil",
              "mast_cell",
              "APC",
              "antigen_presenting_cell",
              "macrophage",
              "dendritic_cell",
              "tumor_associated_macrophage",
              "TAM"
            ]
          },
          {
            "name": "cf_direction_commit_on_unresolvable_slot",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_not_resolvable_hedge",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].functional_attribution_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_type_I_IFN_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.type_I_IFN",
            "kind": "value",
            "op": "in",
            "expected": [
              "mreg_DC",
              "C1qhigh_macrophage_either_subtype"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-terra-codex",
      "modelName": "gpt-5.6-terra",
      "harness": "openai-codex",
      "task": "10x_5p_melanoma_cluster_enriched_apc_states_v2",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 452.983197,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "top_3_cluster_enriched_apc_states",
            "kind": "list",
            "matchKey": "apc_state",
            "k": 3,
            "expected": [
              {
                "key": "plasmacytoid_DC",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "T_cell_stimulatory_role_supported",
                      "role_direction_unresolved"
                    ],
                    "canonical": "T_cell_stimulatory_role_supported"
                  }
                }
              },
              {
                "key": "C1qhigh_lipid_associated_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "C1qhigh_inflammatory_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "plasma_cell",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "cohort_significant_per_patient_inconsistent"
                    ],
                    "canonical": "cohort_significant_per_patient_inconsistent"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CD80_CD86": {
                "op": "in",
                "value": [
                  "mreg_DC",
                  "shared_across_APC_states"
                ]
              },
              "PD_L1_in_APC_clusters": {
                "op": "in",
                "value": [
                  "C1qhigh_macrophage_either_subtype",
                  "shared_across_APC_states",
                  "mreg_DC"
                ]
              },
              "type_I_IFN": {
                "op": "in",
                "value": [
                  "plasmacytoid_DC",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "cf_mreg_dc_claimed_enriched",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "mreg_DC",
              "mature_DC_enriched_in_immunoregulatory_molecules"
            ]
          },
          {
            "name": "cf_framework_offcompartment_or_generic_substitution",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "cDC1",
              "Maier_mregDC",
              "Cheng_C1QC_TAM",
              "Cheng_SPP1_TAM",
              "Cheng_FOLR2_TAM",
              "Villani_pDC",
              "Zhang_inflammatory_monocyte",
              "LAMP3_pos_DC",
              "myeloid_derived_suppressor_cell",
              "MDSC",
              "tumor_associated_neutrophil",
              "mast_cell",
              "APC",
              "antigen_presenting_cell",
              "macrophage",
              "dendritic_cell",
              "tumor_associated_macrophage",
              "TAM"
            ]
          },
          {
            "name": "cf_direction_commit_on_unresolvable_slot",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_not_resolvable_hedge",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].functional_attribution_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_type_I_IFN_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.type_I_IFN",
            "kind": "value",
            "op": "in",
            "expected": [
              "mreg_DC",
              "C1qhigh_macrophage_either_subtype"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-terra-codex",
      "modelName": "gpt-5.6-terra",
      "harness": "openai-codex",
      "task": "10x_5p_melanoma_cluster_tumor_state_causation_v5",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 675.15949,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "enriched_tumor_states",
            "kind": "list",
            "matchKey": "tumor_state",
            "k": 3,
            "expected": [
              {
                "key": "immune_response",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "direction_unresolved_cross_sectional"
                    ],
                    "canonical": "direction_unresolved_cross_sectional"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "IFN_response_axis",
                      "broad_antigen_presentation_MHC"
                    ],
                    "canonical": "IFN_response_axis"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "enrichment_diminished_by_decontamination",
                      "threshold_sensitive"
                    ],
                    "canonical": "enrichment_diminished_by_decontamination"
                  }
                }
              },
              {
                "key": "stress_hypoxia",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "data_supports_cell_intrinsic_cause"
                    ],
                    "canonical": "data_supports_cell_intrinsic_cause"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "HIF_hypoxia_program_specific"
                    ],
                    "canonical": "HIF_hypoxia_program_specific"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "robust_across_methods",
                      "robust_after_decontamination"
                    ],
                    "canonical": "robust_across_methods"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CXCL9_CXCL10": {
                "op": "in",
                "value": [
                  "immune_response_tumor_state"
                ]
              },
              "CCL5": {
                "op": "in",
                "value": [
                  "shared_across_tumor_states",
                  "partner_lymphocyte_derived",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "gate_conjugate_gate_confounder",
            "role": "gate",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "partner_lymphocyte_capture"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "antigen_presentation_decomposition",
            "kind": "dict",
            "expected": {
              "MHC_class_I": {
                "op": "in",
                "value": [
                  "co_regulated_with_dominant_program",
                  "tumor_intrinsic_enriched_independent"
                ]
              },
              "MHC_class_II": {
                "op": "in",
                "value": [
                  "partner_cell_attributable",
                  "flat_not_enriched",
                  "method_sensitive_no_robust_call",
                  "co_regulated_with_dominant_program"
                ]
              }
            }
          },
          {
            "name": "cf_non_robust_state_claimed_enriched",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "transitory_melanocytic",
              "intermediate_melanocytic",
              "melanocytic",
              "MITF_high",
              "neural_crest_like",
              "mitotic",
              "stress_p53",
              "undifferentiated",
              "dedifferentiated",
              "mesenchymal_tumor",
              "AXL_high_tumor",
              "invasive_tumor",
              "senescence_like",
              "EMT_transition",
              "cancer_stem_cell_like",
              "oxidative_stress_tumor"
            ]
          },
          {
            "name": "cf_framework_generic_or_offtissue_substitution",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "Tsoi_undifferentiated",
              "Tsoi_neural_crest_like",
              "Tsoi_transitory",
              "Tsoi_melanocytic",
              "Rambow_NCSC",
              "Wouters_mesenchymal",
              "Pozniak_TCF4_dependent",
              "tumor_cell",
              "melanoma_cell",
              "malignant_cell",
              "tumor_subpopulation",
              "keratinocyte",
              "hepatocyte",
              "lung_adenocarcinoma_cell",
              "colon_adenocarcinoma_cell",
              "Tirosh_cell_cycle_program",
              "Jerby_Arnon_program",
              "glioblastoma_cell",
              "breast_carcinoma_cell"
            ]
          },
          {
            "name": "cf_immune_response_direction_committed",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_immune_robustness_recall",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_ambiguous_hedge",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].causation_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_stress_generic_stress_response_evidence",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_conjugate_confounder_contamination_denied",
            "role": "hard_fail",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected"
            ]
          },
          {
            "name": "cf_ccl5_recall_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CCL5",
            "kind": "avoid",
            "expected": [
              "immune_response_tumor_state"
            ]
          },
          {
            "name": "cf_cxcl9_cxcl10_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CXCL9_CXCL10",
            "kind": "avoid",
            "expected": [
              "stress_hypoxia_tumor_state"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-terra-codex",
      "modelName": "gpt-5.6-terra",
      "harness": "openai-codex",
      "task": "10x_5p_melanoma_cluster_tumor_state_causation_v5",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 636.182937,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "enriched_tumor_states",
            "kind": "list",
            "matchKey": "tumor_state",
            "k": 3,
            "expected": [
              {
                "key": "immune_response",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "direction_unresolved_cross_sectional"
                    ],
                    "canonical": "direction_unresolved_cross_sectional"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "IFN_response_axis",
                      "broad_antigen_presentation_MHC"
                    ],
                    "canonical": "IFN_response_axis"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "enrichment_diminished_by_decontamination",
                      "threshold_sensitive"
                    ],
                    "canonical": "enrichment_diminished_by_decontamination"
                  }
                }
              },
              {
                "key": "stress_hypoxia",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "data_supports_cell_intrinsic_cause"
                    ],
                    "canonical": "data_supports_cell_intrinsic_cause"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "HIF_hypoxia_program_specific"
                    ],
                    "canonical": "HIF_hypoxia_program_specific"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "robust_across_methods",
                      "robust_after_decontamination"
                    ],
                    "canonical": "robust_across_methods"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CXCL9_CXCL10": {
                "op": "in",
                "value": [
                  "immune_response_tumor_state"
                ]
              },
              "CCL5": {
                "op": "in",
                "value": [
                  "shared_across_tumor_states",
                  "partner_lymphocyte_derived",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "gate_conjugate_gate_confounder",
            "role": "gate",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "partner_lymphocyte_capture"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "antigen_presentation_decomposition",
            "kind": "dict",
            "expected": {
              "MHC_class_I": {
                "op": "in",
                "value": [
                  "co_regulated_with_dominant_program",
                  "tumor_intrinsic_enriched_independent"
                ]
              },
              "MHC_class_II": {
                "op": "in",
                "value": [
                  "partner_cell_attributable",
                  "flat_not_enriched",
                  "method_sensitive_no_robust_call",
                  "co_regulated_with_dominant_program"
                ]
              }
            }
          },
          {
            "name": "cf_non_robust_state_claimed_enriched",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "transitory_melanocytic",
              "intermediate_melanocytic",
              "melanocytic",
              "MITF_high",
              "neural_crest_like",
              "mitotic",
              "stress_p53",
              "undifferentiated",
              "dedifferentiated",
              "mesenchymal_tumor",
              "AXL_high_tumor",
              "invasive_tumor",
              "senescence_like",
              "EMT_transition",
              "cancer_stem_cell_like",
              "oxidative_stress_tumor"
            ]
          },
          {
            "name": "cf_framework_generic_or_offtissue_substitution",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "Tsoi_undifferentiated",
              "Tsoi_neural_crest_like",
              "Tsoi_transitory",
              "Tsoi_melanocytic",
              "Rambow_NCSC",
              "Wouters_mesenchymal",
              "Pozniak_TCF4_dependent",
              "tumor_cell",
              "melanoma_cell",
              "malignant_cell",
              "tumor_subpopulation",
              "keratinocyte",
              "hepatocyte",
              "lung_adenocarcinoma_cell",
              "colon_adenocarcinoma_cell",
              "Tirosh_cell_cycle_program",
              "Jerby_Arnon_program",
              "glioblastoma_cell",
              "breast_carcinoma_cell"
            ]
          },
          {
            "name": "cf_immune_response_direction_committed",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_immune_robustness_recall",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_ambiguous_hedge",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].causation_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_stress_generic_stress_response_evidence",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_conjugate_confounder_contamination_denied",
            "role": "hard_fail",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected"
            ]
          },
          {
            "name": "cf_ccl5_recall_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CCL5",
            "kind": "avoid",
            "expected": [
              "immune_response_tumor_state"
            ]
          },
          {
            "name": "cf_cxcl9_cxcl10_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CXCL9_CXCL10",
            "kind": "avoid",
            "expected": [
              "stress_hypoxia_tumor_state"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-terra-codex",
      "modelName": "gpt-5.6-terra",
      "harness": "openai-codex",
      "task": "10x_5p_melanoma_cluster_tumor_state_causation_v5",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 363.334161,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "enriched_tumor_states",
            "kind": "list",
            "matchKey": "tumor_state",
            "k": 3,
            "expected": [
              {
                "key": "immune_response",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "direction_unresolved_cross_sectional"
                    ],
                    "canonical": "direction_unresolved_cross_sectional"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "IFN_response_axis",
                      "broad_antigen_presentation_MHC"
                    ],
                    "canonical": "IFN_response_axis"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "enrichment_diminished_by_decontamination",
                      "threshold_sensitive"
                    ],
                    "canonical": "enrichment_diminished_by_decontamination"
                  }
                }
              },
              {
                "key": "stress_hypoxia",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "data_supports_cell_intrinsic_cause"
                    ],
                    "canonical": "data_supports_cell_intrinsic_cause"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "HIF_hypoxia_program_specific"
                    ],
                    "canonical": "HIF_hypoxia_program_specific"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "robust_across_methods",
                      "robust_after_decontamination"
                    ],
                    "canonical": "robust_across_methods"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CXCL9_CXCL10": {
                "op": "in",
                "value": [
                  "immune_response_tumor_state"
                ]
              },
              "CCL5": {
                "op": "in",
                "value": [
                  "shared_across_tumor_states",
                  "partner_lymphocyte_derived",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "gate_conjugate_gate_confounder",
            "role": "gate",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "partner_lymphocyte_capture"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "antigen_presentation_decomposition",
            "kind": "dict",
            "expected": {
              "MHC_class_I": {
                "op": "in",
                "value": [
                  "co_regulated_with_dominant_program",
                  "tumor_intrinsic_enriched_independent"
                ]
              },
              "MHC_class_II": {
                "op": "in",
                "value": [
                  "partner_cell_attributable",
                  "flat_not_enriched",
                  "method_sensitive_no_robust_call",
                  "co_regulated_with_dominant_program"
                ]
              }
            }
          },
          {
            "name": "cf_non_robust_state_claimed_enriched",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "transitory_melanocytic",
              "intermediate_melanocytic",
              "melanocytic",
              "MITF_high",
              "neural_crest_like",
              "mitotic",
              "stress_p53",
              "undifferentiated",
              "dedifferentiated",
              "mesenchymal_tumor",
              "AXL_high_tumor",
              "invasive_tumor",
              "senescence_like",
              "EMT_transition",
              "cancer_stem_cell_like",
              "oxidative_stress_tumor"
            ]
          },
          {
            "name": "cf_framework_generic_or_offtissue_substitution",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "Tsoi_undifferentiated",
              "Tsoi_neural_crest_like",
              "Tsoi_transitory",
              "Tsoi_melanocytic",
              "Rambow_NCSC",
              "Wouters_mesenchymal",
              "Pozniak_TCF4_dependent",
              "tumor_cell",
              "melanoma_cell",
              "malignant_cell",
              "tumor_subpopulation",
              "keratinocyte",
              "hepatocyte",
              "lung_adenocarcinoma_cell",
              "colon_adenocarcinoma_cell",
              "Tirosh_cell_cycle_program",
              "Jerby_Arnon_program",
              "glioblastoma_cell",
              "breast_carcinoma_cell"
            ]
          },
          {
            "name": "cf_immune_response_direction_committed",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_immune_robustness_recall",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_ambiguous_hedge",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].causation_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_stress_generic_stress_response_evidence",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_conjugate_confounder_contamination_denied",
            "role": "hard_fail",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected"
            ]
          },
          {
            "name": "cf_ccl5_recall_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CCL5",
            "kind": "avoid",
            "expected": [
              "immune_response_tumor_state"
            ]
          },
          {
            "name": "cf_cxcl9_cxcl10_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CXCL9_CXCL10",
            "kind": "avoid",
            "expected": [
              "stress_hypoxia_tumor_state"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-terra-codex",
      "modelName": "gpt-5.6-terra",
      "harness": "openai-codex",
      "task": "10x_5p_melanoma_cluster_vs_cd39_tumor_reactivity_dominance",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 245.824884,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "population_findings",
            "kind": "list",
            "matchKey": "capture_gate",
            "k": 4,
            "expected": [
              {
                "key": "tumor_cell_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "apc_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "combined_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "unselected_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "naive",
                      "central_memory",
                      "effector_memory"
                    ],
                    "canonical": "naive"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "low"
                    ],
                    "canonical": "low"
                  },
                  "purity_caveat": {
                    "op": "equals",
                    "values": [
                      "pure_T_cell"
                    ],
                    "canonical": "pure_T_cell"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "no_reactivity_evidence"
                    ],
                    "canonical": "no_reactivity_evidence"
                  }
                }
              },
              {
                "key": "marker_sorted_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "equals",
                    "values": [
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "terminal_exhausted_LAG3_high"
                  },
                  "clonal_expansion_level": {
                    "op": "in",
                    "values": [
                      "high",
                      "intermediate"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "marker_on_non_T_lineages",
                      "single_cell_gating_loses_target"
                    ],
                    "canonical": "marker_on_non_T_lineages"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "marker_or_exhaustion_defined_only"
                    ],
                    "canonical": "marker_or_exhaustion_defined_only"
                  }
                }
              }
            ]
          },
          {
            "name": "best_capture_approach_gate",
            "role": "gate",
            "field": "best_capture_approach",
            "kind": "value",
            "expected": "heterotypic_cell_conjugate"
          },
          {
            "name": "best_selection_marker_gate",
            "role": "gate",
            "field": "best_selection_marker",
            "kind": "value",
            "expected": "none"
          },
          {
            "name": "reactivity_criterion_gate",
            "role": "gate",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "clonal_expansion_and_physical_engagement"
          },
          {
            "name": "cross_patient_robustness_gate",
            "role": "gate",
            "field": "cross_patient_robustness",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_across_patients",
              "marginal_in_some_patients"
            ]
          },
          {
            "name": "marker_selection_consequence_gate",
            "role": "gate",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "discards_clonally_expanded_reactive_cells"
          },
          {
            "name": "superiority_claim_basis_gate",
            "role": "gate",
            "field": "superiority_claim_basis",
            "kind": "value",
            "expected": "requires_functional_reactivity_assay"
          },
          {
            "name": "conjugate_enriched_state_gate",
            "role": "gate",
            "field": "conjugate_enriched_state",
            "kind": "value",
            "expected": "progenitor_exhausted_TCF7_pos_stem_like"
          },
          {
            "name": "cf_circular_exhaustion_criterion",
            "role": "hard_fail",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "exhaustion_signature_intensity"
          },
          {
            "name": "cf_marker_sort_prior",
            "role": "hard_fail",
            "field": "best_selection_marker",
            "kind": "value",
            "op": "in",
            "expected": [
              "CD39",
              "PD1",
              "PD1_high",
              "TIM3",
              "PD1_CD39",
              "CD39_CD103"
            ]
          },
          {
            "name": "cf_single_cell_capture_prior",
            "role": "hard_fail",
            "field": "best_capture_approach",
            "kind": "value",
            "op": "in",
            "expected": [
              "dissociated_single_cell",
              "dissociated_bulk_population"
            ]
          },
          {
            "name": "cf_off_data_capture_gate",
            "role": "hard_fail",
            "field": "$.population_findings[*].capture_gate",
            "kind": "avoid",
            "expected": [
              "dissociated_bulk",
              "matched_peripheral_blood",
              "matched_lymph_node",
              "post_hoc_expanded_clonotype_group"
            ]
          },
          {
            "name": "cf_conjugate_mischaracterized",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_marker_singlet_called_engaged",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "contains",
            "matchKey": "capture_gate",
            "entity": "marker_sorted_singlet",
            "valueField": "reactivity_evidence",
            "expected": [
              "engagement_and_clonal_expansion"
            ]
          },
          {
            "name": "cf_no_engaged_reactive_population",
            "role": "hard_fail",
            "field": "$.population_findings[*].reactivity_evidence",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_marker_captures_all_reactive",
            "role": "hard_fail",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "captures_full_reactive_repertoire"
          },
          {
            "name": "cf_superiority_overclaimed_from_phenotype",
            "role": "hard_fail",
            "field": "superiority_claim_basis",
            "kind": "value",
            "op": "in",
            "expected": [
              "established_by_scrna_phenotype",
              "established_by_reactivity_proxy_scores"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-terra-codex",
      "modelName": "gpt-5.6-terra",
      "harness": "openai-codex",
      "task": "10x_5p_melanoma_cluster_vs_cd39_tumor_reactivity_dominance",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 368.110505,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "population_findings",
            "kind": "list",
            "matchKey": "capture_gate",
            "k": 4,
            "expected": [
              {
                "key": "tumor_cell_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "apc_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "combined_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "unselected_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "naive",
                      "central_memory",
                      "effector_memory"
                    ],
                    "canonical": "naive"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "low"
                    ],
                    "canonical": "low"
                  },
                  "purity_caveat": {
                    "op": "equals",
                    "values": [
                      "pure_T_cell"
                    ],
                    "canonical": "pure_T_cell"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "no_reactivity_evidence"
                    ],
                    "canonical": "no_reactivity_evidence"
                  }
                }
              },
              {
                "key": "marker_sorted_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "equals",
                    "values": [
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "terminal_exhausted_LAG3_high"
                  },
                  "clonal_expansion_level": {
                    "op": "in",
                    "values": [
                      "high",
                      "intermediate"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "marker_on_non_T_lineages",
                      "single_cell_gating_loses_target"
                    ],
                    "canonical": "marker_on_non_T_lineages"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "marker_or_exhaustion_defined_only"
                    ],
                    "canonical": "marker_or_exhaustion_defined_only"
                  }
                }
              }
            ]
          },
          {
            "name": "best_capture_approach_gate",
            "role": "gate",
            "field": "best_capture_approach",
            "kind": "value",
            "expected": "heterotypic_cell_conjugate"
          },
          {
            "name": "best_selection_marker_gate",
            "role": "gate",
            "field": "best_selection_marker",
            "kind": "value",
            "expected": "none"
          },
          {
            "name": "reactivity_criterion_gate",
            "role": "gate",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "clonal_expansion_and_physical_engagement"
          },
          {
            "name": "cross_patient_robustness_gate",
            "role": "gate",
            "field": "cross_patient_robustness",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_across_patients",
              "marginal_in_some_patients"
            ]
          },
          {
            "name": "marker_selection_consequence_gate",
            "role": "gate",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "discards_clonally_expanded_reactive_cells"
          },
          {
            "name": "superiority_claim_basis_gate",
            "role": "gate",
            "field": "superiority_claim_basis",
            "kind": "value",
            "expected": "requires_functional_reactivity_assay"
          },
          {
            "name": "conjugate_enriched_state_gate",
            "role": "gate",
            "field": "conjugate_enriched_state",
            "kind": "value",
            "expected": "progenitor_exhausted_TCF7_pos_stem_like"
          },
          {
            "name": "cf_circular_exhaustion_criterion",
            "role": "hard_fail",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "exhaustion_signature_intensity"
          },
          {
            "name": "cf_marker_sort_prior",
            "role": "hard_fail",
            "field": "best_selection_marker",
            "kind": "value",
            "op": "in",
            "expected": [
              "CD39",
              "PD1",
              "PD1_high",
              "TIM3",
              "PD1_CD39",
              "CD39_CD103"
            ]
          },
          {
            "name": "cf_single_cell_capture_prior",
            "role": "hard_fail",
            "field": "best_capture_approach",
            "kind": "value",
            "op": "in",
            "expected": [
              "dissociated_single_cell",
              "dissociated_bulk_population"
            ]
          },
          {
            "name": "cf_off_data_capture_gate",
            "role": "hard_fail",
            "field": "$.population_findings[*].capture_gate",
            "kind": "avoid",
            "expected": [
              "dissociated_bulk",
              "matched_peripheral_blood",
              "matched_lymph_node",
              "post_hoc_expanded_clonotype_group"
            ]
          },
          {
            "name": "cf_conjugate_mischaracterized",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_marker_singlet_called_engaged",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "contains",
            "matchKey": "capture_gate",
            "entity": "marker_sorted_singlet",
            "valueField": "reactivity_evidence",
            "expected": [
              "engagement_and_clonal_expansion"
            ]
          },
          {
            "name": "cf_no_engaged_reactive_population",
            "role": "hard_fail",
            "field": "$.population_findings[*].reactivity_evidence",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_marker_captures_all_reactive",
            "role": "hard_fail",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "captures_full_reactive_repertoire"
          },
          {
            "name": "cf_superiority_overclaimed_from_phenotype",
            "role": "hard_fail",
            "field": "superiority_claim_basis",
            "kind": "value",
            "op": "in",
            "expected": [
              "established_by_scrna_phenotype",
              "established_by_reactivity_proxy_scores"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-terra-codex",
      "modelName": "gpt-5.6-terra",
      "harness": "openai-codex",
      "task": "10x_5p_melanoma_cluster_vs_cd39_tumor_reactivity_dominance",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 452.187052,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "population_findings",
            "kind": "list",
            "matchKey": "capture_gate",
            "k": 4,
            "expected": [
              {
                "key": "tumor_cell_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "apc_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "combined_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "unselected_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "naive",
                      "central_memory",
                      "effector_memory"
                    ],
                    "canonical": "naive"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "low"
                    ],
                    "canonical": "low"
                  },
                  "purity_caveat": {
                    "op": "equals",
                    "values": [
                      "pure_T_cell"
                    ],
                    "canonical": "pure_T_cell"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "no_reactivity_evidence"
                    ],
                    "canonical": "no_reactivity_evidence"
                  }
                }
              },
              {
                "key": "marker_sorted_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "equals",
                    "values": [
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "terminal_exhausted_LAG3_high"
                  },
                  "clonal_expansion_level": {
                    "op": "in",
                    "values": [
                      "high",
                      "intermediate"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "marker_on_non_T_lineages",
                      "single_cell_gating_loses_target"
                    ],
                    "canonical": "marker_on_non_T_lineages"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "marker_or_exhaustion_defined_only"
                    ],
                    "canonical": "marker_or_exhaustion_defined_only"
                  }
                }
              }
            ]
          },
          {
            "name": "best_capture_approach_gate",
            "role": "gate",
            "field": "best_capture_approach",
            "kind": "value",
            "expected": "heterotypic_cell_conjugate"
          },
          {
            "name": "best_selection_marker_gate",
            "role": "gate",
            "field": "best_selection_marker",
            "kind": "value",
            "expected": "none"
          },
          {
            "name": "reactivity_criterion_gate",
            "role": "gate",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "clonal_expansion_and_physical_engagement"
          },
          {
            "name": "cross_patient_robustness_gate",
            "role": "gate",
            "field": "cross_patient_robustness",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_across_patients",
              "marginal_in_some_patients"
            ]
          },
          {
            "name": "marker_selection_consequence_gate",
            "role": "gate",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "discards_clonally_expanded_reactive_cells"
          },
          {
            "name": "superiority_claim_basis_gate",
            "role": "gate",
            "field": "superiority_claim_basis",
            "kind": "value",
            "expected": "requires_functional_reactivity_assay"
          },
          {
            "name": "conjugate_enriched_state_gate",
            "role": "gate",
            "field": "conjugate_enriched_state",
            "kind": "value",
            "expected": "progenitor_exhausted_TCF7_pos_stem_like"
          },
          {
            "name": "cf_circular_exhaustion_criterion",
            "role": "hard_fail",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "exhaustion_signature_intensity"
          },
          {
            "name": "cf_marker_sort_prior",
            "role": "hard_fail",
            "field": "best_selection_marker",
            "kind": "value",
            "op": "in",
            "expected": [
              "CD39",
              "PD1",
              "PD1_high",
              "TIM3",
              "PD1_CD39",
              "CD39_CD103"
            ]
          },
          {
            "name": "cf_single_cell_capture_prior",
            "role": "hard_fail",
            "field": "best_capture_approach",
            "kind": "value",
            "op": "in",
            "expected": [
              "dissociated_single_cell",
              "dissociated_bulk_population"
            ]
          },
          {
            "name": "cf_off_data_capture_gate",
            "role": "hard_fail",
            "field": "$.population_findings[*].capture_gate",
            "kind": "avoid",
            "expected": [
              "dissociated_bulk",
              "matched_peripheral_blood",
              "matched_lymph_node",
              "post_hoc_expanded_clonotype_group"
            ]
          },
          {
            "name": "cf_conjugate_mischaracterized",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_marker_singlet_called_engaged",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "contains",
            "matchKey": "capture_gate",
            "entity": "marker_sorted_singlet",
            "valueField": "reactivity_evidence",
            "expected": [
              "engagement_and_clonal_expansion"
            ]
          },
          {
            "name": "cf_no_engaged_reactive_population",
            "role": "hard_fail",
            "field": "$.population_findings[*].reactivity_evidence",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_marker_captures_all_reactive",
            "role": "hard_fail",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "captures_full_reactive_repertoire"
          },
          {
            "name": "cf_superiority_overclaimed_from_phenotype",
            "role": "hard_fail",
            "field": "superiority_claim_basis",
            "kind": "value",
            "op": "in",
            "expected": [
              "established_by_scrna_phenotype",
              "established_by_reactivity_proxy_scores"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-terra-codex",
      "modelName": "gpt-5.6-terra",
      "harness": "openai-codex",
      "task": "10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 1,
      "passed": true,
      "score": null,
      "durationS": 590.440454,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_singlet_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "TOXhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "LAG3high_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "GZMKhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "intermediate_exhausted_gzmk"
                    ],
                    "canonical": "intermediate_exhausted_gzmk"
                  }
                }
              },
              {
                "key": "TCF7_stem_like_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "stem_like_exhausted"
                    ],
                    "canonical": "stem_like_exhausted"
                  }
                }
              },
              {
                "key": "MKI67_pos_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              },
              {
                "key": "MKI67high_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_tumor_cluster_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "Tn",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "Tn_Tmem",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "early_Tem_GZMK",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              },
              {
                "key": "Tem_GZMH",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              }
            ]
          },
          {
            "name": "clonotype_expansion_direction_data_true",
            "role": "gate",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_substantially_more_expanded_than_cluster",
              "singlet_modestly_more_expanded_than_cluster"
            ]
          },
          {
            "name": "biological_inference_data_true",
            "role": "gate",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_antigen_experienced_cluster_bystander"
            ]
          },
          {
            "name": "singlet_top3_contains_naive_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Tn",
              "Tn_Tmem",
              "early_Tem_GZMK",
              "Tem_GZMH"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "TOXhigh_Tex",
              "LAG3high_Tex",
              "GZMKhigh_Tex",
              "TCF7_stem_like_Tex",
              "MKI67_pos_Tex_Tprol",
              "MKI67high_Tex_Tprol"
            ]
          },
          {
            "name": "singlet_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "TOXhigh_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "TCF7_stem_like_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "MKI67_state_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "Tn_or_Tn_Tmem_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_stemlike_precursor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_reported_canonical_expansion_direction",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_substantially_more_expanded_than_singlet",
              "cluster_modestly_more_expanded_than_singlet"
            ]
          },
          {
            "name": "cf_expansion_lazy_defer",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "expansion_pattern_not_detectable"
            ]
          },
          {
            "name": "cf_reported_canonical_expected_biology",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_antigen_experienced_singlet_bystander"
            ]
          },
          {
            "name": "cf_biological_inference_null_artifact_or_lineage",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "differences_reflect_dissociation_or_sorting_artifact",
              "cluster_and_singlet_capture_same_biology",
              "cluster_and_singlet_capture_distinct_cd8_lineages"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "biological_relations",
            "kind": "dict",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_greater_than",
                  "greater_than"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "less_than",
                  "much_less_than"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than",
                  "approximately_equal"
                ]
              },
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal"
                ]
              },
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal",
                  "greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_exhaustion_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_naive_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_proliferation_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_expansion_richness_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_multichain_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_intermediate_gzmk_overgeneralized",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_melanocytic_conjugate_overcall",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "values": [
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_relations_mass_hedge",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-terra-codex",
      "modelName": "gpt-5.6-terra",
      "harness": "openai-codex",
      "task": "10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 427.001382,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_singlet_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "TOXhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "LAG3high_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "GZMKhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "intermediate_exhausted_gzmk"
                    ],
                    "canonical": "intermediate_exhausted_gzmk"
                  }
                }
              },
              {
                "key": "TCF7_stem_like_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "stem_like_exhausted"
                    ],
                    "canonical": "stem_like_exhausted"
                  }
                }
              },
              {
                "key": "MKI67_pos_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              },
              {
                "key": "MKI67high_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_tumor_cluster_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "Tn",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "Tn_Tmem",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "early_Tem_GZMK",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              },
              {
                "key": "Tem_GZMH",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              }
            ]
          },
          {
            "name": "clonotype_expansion_direction_data_true",
            "role": "gate",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_substantially_more_expanded_than_cluster",
              "singlet_modestly_more_expanded_than_cluster"
            ]
          },
          {
            "name": "biological_inference_data_true",
            "role": "gate",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_antigen_experienced_cluster_bystander"
            ]
          },
          {
            "name": "singlet_top3_contains_naive_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Tn",
              "Tn_Tmem",
              "early_Tem_GZMK",
              "Tem_GZMH"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "TOXhigh_Tex",
              "LAG3high_Tex",
              "GZMKhigh_Tex",
              "TCF7_stem_like_Tex",
              "MKI67_pos_Tex_Tprol",
              "MKI67high_Tex_Tprol"
            ]
          },
          {
            "name": "singlet_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "TOXhigh_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "TCF7_stem_like_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "MKI67_state_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "Tn_or_Tn_Tmem_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_stemlike_precursor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_reported_canonical_expansion_direction",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_substantially_more_expanded_than_singlet",
              "cluster_modestly_more_expanded_than_singlet"
            ]
          },
          {
            "name": "cf_expansion_lazy_defer",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "expansion_pattern_not_detectable"
            ]
          },
          {
            "name": "cf_reported_canonical_expected_biology",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_antigen_experienced_singlet_bystander"
            ]
          },
          {
            "name": "cf_biological_inference_null_artifact_or_lineage",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "differences_reflect_dissociation_or_sorting_artifact",
              "cluster_and_singlet_capture_same_biology",
              "cluster_and_singlet_capture_distinct_cd8_lineages"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "biological_relations",
            "kind": "dict",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_greater_than",
                  "greater_than"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "less_than",
                  "much_less_than"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than",
                  "approximately_equal"
                ]
              },
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal"
                ]
              },
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal",
                  "greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_exhaustion_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_naive_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_proliferation_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_expansion_richness_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_multichain_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_intermediate_gzmk_overgeneralized",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_melanocytic_conjugate_overcall",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "values": [
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_relations_mass_hedge",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-terra-codex",
      "modelName": "gpt-5.6-terra",
      "harness": "openai-codex",
      "task": "10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 598.525859,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_singlet_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "TOXhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "LAG3high_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "GZMKhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "intermediate_exhausted_gzmk"
                    ],
                    "canonical": "intermediate_exhausted_gzmk"
                  }
                }
              },
              {
                "key": "TCF7_stem_like_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "stem_like_exhausted"
                    ],
                    "canonical": "stem_like_exhausted"
                  }
                }
              },
              {
                "key": "MKI67_pos_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              },
              {
                "key": "MKI67high_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_tumor_cluster_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "Tn",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "Tn_Tmem",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "early_Tem_GZMK",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              },
              {
                "key": "Tem_GZMH",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              }
            ]
          },
          {
            "name": "clonotype_expansion_direction_data_true",
            "role": "gate",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_substantially_more_expanded_than_cluster",
              "singlet_modestly_more_expanded_than_cluster"
            ]
          },
          {
            "name": "biological_inference_data_true",
            "role": "gate",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_antigen_experienced_cluster_bystander"
            ]
          },
          {
            "name": "singlet_top3_contains_naive_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Tn",
              "Tn_Tmem",
              "early_Tem_GZMK",
              "Tem_GZMH"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "TOXhigh_Tex",
              "LAG3high_Tex",
              "GZMKhigh_Tex",
              "TCF7_stem_like_Tex",
              "MKI67_pos_Tex_Tprol",
              "MKI67high_Tex_Tprol"
            ]
          },
          {
            "name": "singlet_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "TOXhigh_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "TCF7_stem_like_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "MKI67_state_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "Tn_or_Tn_Tmem_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_stemlike_precursor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_reported_canonical_expansion_direction",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_substantially_more_expanded_than_singlet",
              "cluster_modestly_more_expanded_than_singlet"
            ]
          },
          {
            "name": "cf_expansion_lazy_defer",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "expansion_pattern_not_detectable"
            ]
          },
          {
            "name": "cf_reported_canonical_expected_biology",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_antigen_experienced_singlet_bystander"
            ]
          },
          {
            "name": "cf_biological_inference_null_artifact_or_lineage",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "differences_reflect_dissociation_or_sorting_artifact",
              "cluster_and_singlet_capture_same_biology",
              "cluster_and_singlet_capture_distinct_cd8_lineages"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "biological_relations",
            "kind": "dict",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_greater_than",
                  "greater_than"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "less_than",
                  "much_less_than"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than",
                  "approximately_equal"
                ]
              },
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal"
                ]
              },
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal",
                  "greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_exhaustion_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_naive_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_proliferation_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_expansion_richness_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_multichain_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_intermediate_gzmk_overgeneralized",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_melanocytic_conjugate_overcall",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "values": [
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_relations_mass_hedge",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-terra-codex",
      "modelName": "gpt-5.6-terra",
      "harness": "openai-codex",
      "task": "10x_5p_melanoma_tcr_matched_apc_vs_tumor_exhaustion",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 1,
      "passed": true,
      "score": null,
      "durationS": 239.954975,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "state_discriminating_genes",
            "kind": "list",
            "matchKey": "gene",
            "k": 3,
            "expected": [
              {
                "key": "CTLA4",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TOX",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "ENTPD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "HAVCR2",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TIGIT",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "PDCD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "difference_attributable_to",
            "role": "gate",
            "field": "difference_attributable_to",
            "kind": "value",
            "expected": "covaries_with_apc_conjugate_state"
          },
          {
            "name": "causal_status",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "direction_unresolved_selection_confound"
          },
          {
            "name": "functional_interpretation",
            "role": "gate",
            "field": "functional_interpretation",
            "kind": "value",
            "expected": "costimulation_signature_function_unresolved"
          },
          {
            "name": "cf_contamination_gene_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "MLANA",
              "PMEL",
              "DCT",
              "TYR",
              "TYRP1",
              "SOX10",
              "MITF",
              "MAGEA4"
            ]
          },
          {
            "name": "cf_apc_partner_contamination_gene",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "HLA-DRA",
              "CD68",
              "LYZ",
              "CD14",
              "C1QA",
              "ITGAX",
              "CD1C"
            ]
          },
          {
            "name": "cf_ligand_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "CD274",
              "PDCD1LG2",
              "CD80",
              "CD86"
            ]
          },
          {
            "name": "cf_attribution_causal_or_baseline",
            "role": "hard_fail",
            "field": "$.difference_attributable_to",
            "kind": "avoid",
            "expected": [
              "apc_engagement",
              "tumor_cell_engagement",
              "clonotype_intrinsic_baseline"
            ]
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "engagement_induces_exhausted_state",
              "pre_exhausted_cells_select_into_apc_conjugates"
            ]
          },
          {
            "name": "cf_causal_fabrication",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "resolved_by_pseudotemporal_ordering",
              "resolved_by_perturbation_experiment"
            ]
          },
          {
            "name": "cf_functional_overclaim",
            "role": "hard_fail",
            "field": "$.functional_interpretation",
            "kind": "avoid",
            "expected": [
              "costimulation_licensed_reinvigoration_competent",
              "costimulation_licensed_checkpoint_responsive",
              "tumor_driven_terminal_exhaustion"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-terra-codex",
      "modelName": "gpt-5.6-terra",
      "harness": "openai-codex",
      "task": "10x_5p_melanoma_tcr_matched_apc_vs_tumor_exhaustion",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 299.510735,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "state_discriminating_genes",
            "kind": "list",
            "matchKey": "gene",
            "k": 3,
            "expected": [
              {
                "key": "CTLA4",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TOX",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "ENTPD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "HAVCR2",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TIGIT",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "PDCD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "difference_attributable_to",
            "role": "gate",
            "field": "difference_attributable_to",
            "kind": "value",
            "expected": "covaries_with_apc_conjugate_state"
          },
          {
            "name": "causal_status",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "direction_unresolved_selection_confound"
          },
          {
            "name": "functional_interpretation",
            "role": "gate",
            "field": "functional_interpretation",
            "kind": "value",
            "expected": "costimulation_signature_function_unresolved"
          },
          {
            "name": "cf_contamination_gene_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "MLANA",
              "PMEL",
              "DCT",
              "TYR",
              "TYRP1",
              "SOX10",
              "MITF",
              "MAGEA4"
            ]
          },
          {
            "name": "cf_apc_partner_contamination_gene",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "HLA-DRA",
              "CD68",
              "LYZ",
              "CD14",
              "C1QA",
              "ITGAX",
              "CD1C"
            ]
          },
          {
            "name": "cf_ligand_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "CD274",
              "PDCD1LG2",
              "CD80",
              "CD86"
            ]
          },
          {
            "name": "cf_attribution_causal_or_baseline",
            "role": "hard_fail",
            "field": "$.difference_attributable_to",
            "kind": "avoid",
            "expected": [
              "apc_engagement",
              "tumor_cell_engagement",
              "clonotype_intrinsic_baseline"
            ]
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "engagement_induces_exhausted_state",
              "pre_exhausted_cells_select_into_apc_conjugates"
            ]
          },
          {
            "name": "cf_causal_fabrication",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "resolved_by_pseudotemporal_ordering",
              "resolved_by_perturbation_experiment"
            ]
          },
          {
            "name": "cf_functional_overclaim",
            "role": "hard_fail",
            "field": "$.functional_interpretation",
            "kind": "avoid",
            "expected": [
              "costimulation_licensed_reinvigoration_competent",
              "costimulation_licensed_checkpoint_responsive",
              "tumor_driven_terminal_exhaustion"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-terra-codex",
      "modelName": "gpt-5.6-terra",
      "harness": "openai-codex",
      "task": "10x_5p_melanoma_tcr_matched_apc_vs_tumor_exhaustion",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 334.236808,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "state_discriminating_genes",
            "kind": "list",
            "matchKey": "gene",
            "k": 3,
            "expected": [
              {
                "key": "CTLA4",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TOX",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "ENTPD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "HAVCR2",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TIGIT",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "PDCD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "difference_attributable_to",
            "role": "gate",
            "field": "difference_attributable_to",
            "kind": "value",
            "expected": "covaries_with_apc_conjugate_state"
          },
          {
            "name": "causal_status",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "direction_unresolved_selection_confound"
          },
          {
            "name": "functional_interpretation",
            "role": "gate",
            "field": "functional_interpretation",
            "kind": "value",
            "expected": "costimulation_signature_function_unresolved"
          },
          {
            "name": "cf_contamination_gene_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "MLANA",
              "PMEL",
              "DCT",
              "TYR",
              "TYRP1",
              "SOX10",
              "MITF",
              "MAGEA4"
            ]
          },
          {
            "name": "cf_apc_partner_contamination_gene",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "HLA-DRA",
              "CD68",
              "LYZ",
              "CD14",
              "C1QA",
              "ITGAX",
              "CD1C"
            ]
          },
          {
            "name": "cf_ligand_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "CD274",
              "PDCD1LG2",
              "CD80",
              "CD86"
            ]
          },
          {
            "name": "cf_attribution_causal_or_baseline",
            "role": "hard_fail",
            "field": "$.difference_attributable_to",
            "kind": "avoid",
            "expected": [
              "apc_engagement",
              "tumor_cell_engagement",
              "clonotype_intrinsic_baseline"
            ]
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "engagement_induces_exhausted_state",
              "pre_exhausted_cells_select_into_apc_conjugates"
            ]
          },
          {
            "name": "cf_causal_fabrication",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "resolved_by_pseudotemporal_ordering",
              "resolved_by_perturbation_experiment"
            ]
          },
          {
            "name": "cf_functional_overclaim",
            "role": "hard_fail",
            "field": "$.functional_interpretation",
            "kind": "avoid",
            "expected": [
              "costimulation_licensed_reinvigoration_competent",
              "costimulation_licensed_checkpoint_responsive",
              "tumor_driven_terminal_exhaustion"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-terra-codex",
      "modelName": "gpt-5.6-terra",
      "harness": "openai-codex",
      "task": "green_2_tme_context_dependence",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 895.077691,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "program_map",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "sig_includes_itgae",
            "role": "gate",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Itgae"
            ]
          },
          {
            "name": "declared_a_known_confounder",
            "role": "gate",
            "field": "$.excluded_confounders[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "cf_signature_gut_contaminant",
            "role": "hard_fail",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "generic_signature_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd103_program_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "<"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "per_model_regulatory_state",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "poised_unfired"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "late_activation"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "closing_committed"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_tuples",
            "kind": "list",
            "matchKey": "regulator",
            "k": 1,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "role": {
                    "op": "equals",
                    "values": [
                      "repressor"
                    ]
                  },
                  "gating_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  },
                  "evidence_basis": {
                    "op": "equals",
                    "values": [
                      "target_program_behavior"
                    ]
                  },
                  "expr_vs_activity": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "locus_differential_gate",
            "role": "gate",
            "field": "locus_differential",
            "kind": "value",
            "expected": "uniform_across_models"
          },
          {
            "name": "formation_limited_by_gate",
            "role": "gate",
            "field": "formation_limited_by",
            "kind": "value",
            "expected": "absent_induction"
          },
          {
            "name": "locus_competent_in_naive_gate",
            "role": "gate",
            "field": "locus_competent_in_naive",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "ready_silent_fraction_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 3,
            "expected": [
              {
                "key": "B16_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_KPC_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "activator_chromatin_footprint_gate",
            "role": "gate",
            "field": "activator_chromatin_footprint",
            "kind": "value",
            "expected": "inconsistent_across_formers"
          },
          {
            "name": "exhaustion_comparable_gate",
            "role": "gate",
            "field": "exhaustion_comparable_across_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_nondeterminant_gate",
            "role": "gate",
            "field": "exhaustion_explains_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_separable_program_gate",
            "role": "gate",
            "field": "exhaustion_separable_program",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "exhaustion_counterexample_gate",
            "role": "gate",
            "field": "exhaustion_counterexample_model",
            "kind": "value",
            "expected": "KPC_GP"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "exhaustion_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "cf_inconsistent_present_without_activator",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "program",
            "entity": "present",
            "valueField": "activator_state",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_no_block_but_absent",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "none",
            "valueField": "program",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_late_block_without_poised_locus",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "late_activation",
            "valueField": "locus_state",
            "expected": [
              "closing_committed",
              "active"
            ]
          },
          {
            "name": "cf_inconsistent_early_block_with_brake_released",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "early_release",
            "valueField": "brake_state",
            "expected": [
              "released"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-terra-codex",
      "modelName": "gpt-5.6-terra",
      "harness": "openai-codex",
      "task": "green_2_tme_context_dependence",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 1039.617891,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "program_map",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "sig_includes_itgae",
            "role": "gate",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Itgae"
            ]
          },
          {
            "name": "declared_a_known_confounder",
            "role": "gate",
            "field": "$.excluded_confounders[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "cf_signature_gut_contaminant",
            "role": "hard_fail",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "generic_signature_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd103_program_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "<"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "per_model_regulatory_state",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "poised_unfired"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "late_activation"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "closing_committed"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_tuples",
            "kind": "list",
            "matchKey": "regulator",
            "k": 1,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "role": {
                    "op": "equals",
                    "values": [
                      "repressor"
                    ]
                  },
                  "gating_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  },
                  "evidence_basis": {
                    "op": "equals",
                    "values": [
                      "target_program_behavior"
                    ]
                  },
                  "expr_vs_activity": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "locus_differential_gate",
            "role": "gate",
            "field": "locus_differential",
            "kind": "value",
            "expected": "uniform_across_models"
          },
          {
            "name": "formation_limited_by_gate",
            "role": "gate",
            "field": "formation_limited_by",
            "kind": "value",
            "expected": "absent_induction"
          },
          {
            "name": "locus_competent_in_naive_gate",
            "role": "gate",
            "field": "locus_competent_in_naive",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "ready_silent_fraction_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 3,
            "expected": [
              {
                "key": "B16_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_KPC_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "activator_chromatin_footprint_gate",
            "role": "gate",
            "field": "activator_chromatin_footprint",
            "kind": "value",
            "expected": "inconsistent_across_formers"
          },
          {
            "name": "exhaustion_comparable_gate",
            "role": "gate",
            "field": "exhaustion_comparable_across_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_nondeterminant_gate",
            "role": "gate",
            "field": "exhaustion_explains_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_separable_program_gate",
            "role": "gate",
            "field": "exhaustion_separable_program",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "exhaustion_counterexample_gate",
            "role": "gate",
            "field": "exhaustion_counterexample_model",
            "kind": "value",
            "expected": "KPC_GP"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "exhaustion_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "cf_inconsistent_present_without_activator",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "program",
            "entity": "present",
            "valueField": "activator_state",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_no_block_but_absent",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "none",
            "valueField": "program",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_late_block_without_poised_locus",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "late_activation",
            "valueField": "locus_state",
            "expected": [
              "closing_committed",
              "active"
            ]
          },
          {
            "name": "cf_inconsistent_early_block_with_brake_released",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "early_release",
            "valueField": "brake_state",
            "expected": [
              "released"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-terra-codex",
      "modelName": "gpt-5.6-terra",
      "harness": "openai-codex",
      "task": "green_2_tme_context_dependence",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 721.648265,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "program_map",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "sig_includes_itgae",
            "role": "gate",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Itgae"
            ]
          },
          {
            "name": "declared_a_known_confounder",
            "role": "gate",
            "field": "$.excluded_confounders[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "cf_signature_gut_contaminant",
            "role": "hard_fail",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "generic_signature_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd103_program_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "<"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "per_model_regulatory_state",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "poised_unfired"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "late_activation"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "closing_committed"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_tuples",
            "kind": "list",
            "matchKey": "regulator",
            "k": 1,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "role": {
                    "op": "equals",
                    "values": [
                      "repressor"
                    ]
                  },
                  "gating_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  },
                  "evidence_basis": {
                    "op": "equals",
                    "values": [
                      "target_program_behavior"
                    ]
                  },
                  "expr_vs_activity": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "locus_differential_gate",
            "role": "gate",
            "field": "locus_differential",
            "kind": "value",
            "expected": "uniform_across_models"
          },
          {
            "name": "formation_limited_by_gate",
            "role": "gate",
            "field": "formation_limited_by",
            "kind": "value",
            "expected": "absent_induction"
          },
          {
            "name": "locus_competent_in_naive_gate",
            "role": "gate",
            "field": "locus_competent_in_naive",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "ready_silent_fraction_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 3,
            "expected": [
              {
                "key": "B16_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_KPC_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "activator_chromatin_footprint_gate",
            "role": "gate",
            "field": "activator_chromatin_footprint",
            "kind": "value",
            "expected": "inconsistent_across_formers"
          },
          {
            "name": "exhaustion_comparable_gate",
            "role": "gate",
            "field": "exhaustion_comparable_across_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_nondeterminant_gate",
            "role": "gate",
            "field": "exhaustion_explains_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_separable_program_gate",
            "role": "gate",
            "field": "exhaustion_separable_program",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "exhaustion_counterexample_gate",
            "role": "gate",
            "field": "exhaustion_counterexample_model",
            "kind": "value",
            "expected": "KPC_GP"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "exhaustion_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "cf_inconsistent_present_without_activator",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "program",
            "entity": "present",
            "valueField": "activator_state",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_no_block_but_absent",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "none",
            "valueField": "program",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_late_block_without_poised_locus",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "late_activation",
            "valueField": "locus_state",
            "expected": [
              "closing_committed",
              "active"
            ]
          },
          {
            "name": "cf_inconsistent_early_block_with_brake_released",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "early_release",
            "valueField": "brake_state",
            "expected": [
              "released"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-terra-codex",
      "modelName": "gpt-5.6-terra",
      "harness": "openai-codex",
      "task": "green_klf2_hidden_repressor_of_trm_like_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 577.371229,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "regulator_gate",
            "role": "gate",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "KLF2",
              "Klf2"
            ]
          },
          {
            "name": "controlling_node_character_gate",
            "role": "gate",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "high_differential_expression"
          },
          {
            "name": "expression_vs_activity_gate",
            "role": "gate",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "expression_exceeds_activity"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "target_program_partition",
            "kind": "list",
            "matchKey": "gene",
            "k": 7,
            "expected": [
              {
                "key": "S1pr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "S1pr4",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Klrg1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Cx3cr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Itgae",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Hic1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "P2rx7",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              }
            ]
          },
          {
            "name": "causal_status_gate",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "association_only_sufficiency_requires_perturbation"
          },
          {
            "name": "residency_control_logic_gate",
            "role": "gate",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_release_necessary_not_sufficient_multi_regulon"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "trm_like_program_by_model",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_absent"
                    ],
                    "canonical": "niche_absent"
                  }
                }
              }
            ]
          },
          {
            "name": "cf_regulator_activator_or_wrong_node",
            "role": "hard_fail",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "RUNX3",
              "Runx3",
              "PRDM1",
              "Prdm1",
              "BLIMP1",
              "Blimp1",
              "BHLHE40",
              "Bhlhe40",
              "ZNF683",
              "Zfp683",
              "HOBIT",
              "Hobit",
              "AHR",
              "Ahr",
              "BATF",
              "Batf",
              "TOX",
              "Tox",
              "NR4A1",
              "Nr4a1",
              "NR4A2",
              "Nr4a2",
              "NR4A3",
              "Nr4a3",
              "TCF7",
              "Tcf7",
              "TCF1",
              "Tcf1",
              "BACH2",
              "Bach2",
              "TBX21",
              "Tbx21",
              "TBET",
              "Tbet",
              "FLI1",
              "Fli1",
              "NR3C1",
              "Nr3c1",
              "EOMES",
              "Eomes",
              "ZEB2",
              "Zeb2",
              "ID2",
              "Id2",
              "ID3",
              "Id3",
              "IRF4",
              "Irf4"
            ]
          },
          {
            "name": "cf_node_character_hidden_driver_recall",
            "role": "hard_fail",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "hidden_driver_activity_not_expression"
          },
          {
            "name": "cf_effect_relation_hidden_pattern",
            "role": "hard_fail",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "activity_exceeds_expression"
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "causal_status",
            "kind": "value",
            "op": "in",
            "expected": [
              "sufficiency_established",
              "necessity_established",
              "causal_established_in_dataset",
              "sufficiency_via_perturbation_in_dataset"
            ]
          },
          {
            "name": "cf_residency_logic_activator_prior",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "residency_activator_induction_primary"
          },
          {
            "name": "cf_residency_logic_single_regulon",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_controls_entire_residency_program"
          },
          {
            "name": "cf_decoy_called_target",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_distractor_model_reported",
            "role": "hard_fail",
            "field": "$.trm_like_program_by_model[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_chromatin_only_decoy",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-terra-codex",
      "modelName": "gpt-5.6-terra",
      "harness": "openai-codex",
      "task": "green_klf2_hidden_repressor_of_trm_like_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 474.799945,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "regulator_gate",
            "role": "gate",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "KLF2",
              "Klf2"
            ]
          },
          {
            "name": "controlling_node_character_gate",
            "role": "gate",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "high_differential_expression"
          },
          {
            "name": "expression_vs_activity_gate",
            "role": "gate",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "expression_exceeds_activity"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "target_program_partition",
            "kind": "list",
            "matchKey": "gene",
            "k": 7,
            "expected": [
              {
                "key": "S1pr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "S1pr4",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Klrg1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Cx3cr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Itgae",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Hic1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "P2rx7",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              }
            ]
          },
          {
            "name": "causal_status_gate",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "association_only_sufficiency_requires_perturbation"
          },
          {
            "name": "residency_control_logic_gate",
            "role": "gate",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_release_necessary_not_sufficient_multi_regulon"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "trm_like_program_by_model",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_absent"
                    ],
                    "canonical": "niche_absent"
                  }
                }
              }
            ]
          },
          {
            "name": "cf_regulator_activator_or_wrong_node",
            "role": "hard_fail",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "RUNX3",
              "Runx3",
              "PRDM1",
              "Prdm1",
              "BLIMP1",
              "Blimp1",
              "BHLHE40",
              "Bhlhe40",
              "ZNF683",
              "Zfp683",
              "HOBIT",
              "Hobit",
              "AHR",
              "Ahr",
              "BATF",
              "Batf",
              "TOX",
              "Tox",
              "NR4A1",
              "Nr4a1",
              "NR4A2",
              "Nr4a2",
              "NR4A3",
              "Nr4a3",
              "TCF7",
              "Tcf7",
              "TCF1",
              "Tcf1",
              "BACH2",
              "Bach2",
              "TBX21",
              "Tbx21",
              "TBET",
              "Tbet",
              "FLI1",
              "Fli1",
              "NR3C1",
              "Nr3c1",
              "EOMES",
              "Eomes",
              "ZEB2",
              "Zeb2",
              "ID2",
              "Id2",
              "ID3",
              "Id3",
              "IRF4",
              "Irf4"
            ]
          },
          {
            "name": "cf_node_character_hidden_driver_recall",
            "role": "hard_fail",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "hidden_driver_activity_not_expression"
          },
          {
            "name": "cf_effect_relation_hidden_pattern",
            "role": "hard_fail",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "activity_exceeds_expression"
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "causal_status",
            "kind": "value",
            "op": "in",
            "expected": [
              "sufficiency_established",
              "necessity_established",
              "causal_established_in_dataset",
              "sufficiency_via_perturbation_in_dataset"
            ]
          },
          {
            "name": "cf_residency_logic_activator_prior",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "residency_activator_induction_primary"
          },
          {
            "name": "cf_residency_logic_single_regulon",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_controls_entire_residency_program"
          },
          {
            "name": "cf_decoy_called_target",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_distractor_model_reported",
            "role": "hard_fail",
            "field": "$.trm_like_program_by_model[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_chromatin_only_decoy",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-terra-codex",
      "modelName": "gpt-5.6-terra",
      "harness": "openai-codex",
      "task": "green_klf2_hidden_repressor_of_trm_like_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 867.754659,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "regulator_gate",
            "role": "gate",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "KLF2",
              "Klf2"
            ]
          },
          {
            "name": "controlling_node_character_gate",
            "role": "gate",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "high_differential_expression"
          },
          {
            "name": "expression_vs_activity_gate",
            "role": "gate",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "expression_exceeds_activity"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "target_program_partition",
            "kind": "list",
            "matchKey": "gene",
            "k": 7,
            "expected": [
              {
                "key": "S1pr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "S1pr4",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Klrg1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Cx3cr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Itgae",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Hic1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "P2rx7",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              }
            ]
          },
          {
            "name": "causal_status_gate",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "association_only_sufficiency_requires_perturbation"
          },
          {
            "name": "residency_control_logic_gate",
            "role": "gate",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_release_necessary_not_sufficient_multi_regulon"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "trm_like_program_by_model",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_absent"
                    ],
                    "canonical": "niche_absent"
                  }
                }
              }
            ]
          },
          {
            "name": "cf_regulator_activator_or_wrong_node",
            "role": "hard_fail",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "RUNX3",
              "Runx3",
              "PRDM1",
              "Prdm1",
              "BLIMP1",
              "Blimp1",
              "BHLHE40",
              "Bhlhe40",
              "ZNF683",
              "Zfp683",
              "HOBIT",
              "Hobit",
              "AHR",
              "Ahr",
              "BATF",
              "Batf",
              "TOX",
              "Tox",
              "NR4A1",
              "Nr4a1",
              "NR4A2",
              "Nr4a2",
              "NR4A3",
              "Nr4a3",
              "TCF7",
              "Tcf7",
              "TCF1",
              "Tcf1",
              "BACH2",
              "Bach2",
              "TBX21",
              "Tbx21",
              "TBET",
              "Tbet",
              "FLI1",
              "Fli1",
              "NR3C1",
              "Nr3c1",
              "EOMES",
              "Eomes",
              "ZEB2",
              "Zeb2",
              "ID2",
              "Id2",
              "ID3",
              "Id3",
              "IRF4",
              "Irf4"
            ]
          },
          {
            "name": "cf_node_character_hidden_driver_recall",
            "role": "hard_fail",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "hidden_driver_activity_not_expression"
          },
          {
            "name": "cf_effect_relation_hidden_pattern",
            "role": "hard_fail",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "activity_exceeds_expression"
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "causal_status",
            "kind": "value",
            "op": "in",
            "expected": [
              "sufficiency_established",
              "necessity_established",
              "causal_established_in_dataset",
              "sufficiency_via_perturbation_in_dataset"
            ]
          },
          {
            "name": "cf_residency_logic_activator_prior",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "residency_activator_induction_primary"
          },
          {
            "name": "cf_residency_logic_single_regulon",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_controls_entire_residency_program"
          },
          {
            "name": "cf_decoy_called_target",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_distractor_model_reported",
            "role": "hard_fail",
            "field": "$.trm_like_program_by_model[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_chromatin_only_decoy",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-terra-codex",
      "modelName": "gpt-5.6-terra",
      "harness": "openai-codex",
      "task": "green_shared_vs_distinct_regulators_trm_vs_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 1664.604074,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_program_map",
            "kind": "list",
            "matchKey": "regulator",
            "k": 3,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "both"
                    ],
                    "canonical": "both"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "represses"
                    ],
                    "canonical": "represses"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Batf",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "trm_like_til_specific"
                    ],
                    "canonical": "trm_like_til_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "chromatin_over_rna"
                    ],
                    "canonical": "chromatin_over_rna"
                  }
                }
              },
              {
                "key": "Ahr",
                "fields": {
                  "program_specificity": {
                    "op": "in",
                    "values": [
                      "trm_specific",
                      "both"
                    ],
                    "canonical": "trm_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "rna_over_chromatin"
                    ],
                    "canonical": "rna_over_chromatin"
                  }
                }
              }
            ]
          },
          {
            "name": "regulator_set_no_extras_gate",
            "role": "gate",
            "field": "$.regulator_program_map[*]",
            "kind": "other",
            "expected": null
          },
          {
            "name": "program_identity_conclusion_gate",
            "role": "gate",
            "field": "program_identity_conclusion",
            "kind": "value",
            "expected": "distinct_state_shared_brake_divergent_activators"
          },
          {
            "name": "residency_install_logic_gate",
            "role": "gate",
            "field": "residency_install_logic",
            "kind": "value",
            "expected": "brake_release_permits_context_matched_activator"
          },
          {
            "name": "activator_context_dependence_gate",
            "role": "gate",
            "field": "activator_context_dependence",
            "kind": "value",
            "expected": "context_matched_crossover"
          },
          {
            "name": "evidence_ceiling_gate",
            "role": "gate",
            "field": "evidence_ceiling",
            "kind": "value",
            "expected": "specificity_dissociations_require_perturbation"
          },
          {
            "name": "cf_install_logic_flat_or_single_step",
            "role": "hard_fail",
            "field": "residency_install_logic",
            "kind": "value",
            "op": "in",
            "expected": [
              "flat_shared_residency_toolkit",
              "activator_induction_sufficient",
              "brake_release_alone_sufficient"
            ]
          },
          {
            "name": "cf_program_identity_lumped_or_wrong",
            "role": "hard_fail",
            "field": "program_identity_conclusion",
            "kind": "value",
            "op": "in",
            "expected": [
              "trm_like_til_is_tumor_trm",
              "trm_like_til_is_exhausted_subset",
              "trm_like_til_is_effector_state",
              "trm_like_til_is_transitional_intermediate",
              "trm_like_til_is_shared_program_distinct_niche"
            ]
          },
          {
            "name": "cf_ahr_assigned_to_til",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Ahr",
            "valueField": "program_specificity",
            "expected": [
              "trm_like_til_specific"
            ]
          },
          {
            "name": "cf_batf_assigned_to_trm_or_shared",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Batf",
            "valueField": "program_specificity",
            "expected": [
              "trm_specific",
              "both"
            ]
          },
          {
            "name": "cf_klf2_called_activator",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Klf2",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          },
          {
            "name": "cf_evidence_ceiling_overclaim",
            "role": "hard_fail",
            "field": "evidence_ceiling",
            "kind": "value",
            "op": "in",
            "expected": [
              "specificity_established_from_regulon_activity",
              "specificity_established_in_dataset",
              "causation_established_by_cross_sectional_concordance",
              "necessity_and_sufficiency_established"
            ]
          },
          {
            "name": "cf_activator_context_shared_or_single",
            "role": "hard_fail",
            "field": "activator_context_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "shared_activators_both_contexts",
              "single_global_activator",
              "redundant_activators"
            ]
          },
          {
            "name": "cf_single_modality_rule",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_hic1_motif_brake_mislabel",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_runx3_anticanonical_as_controller",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Runx3",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-terra-codex",
      "modelName": "gpt-5.6-terra",
      "harness": "openai-codex",
      "task": "green_shared_vs_distinct_regulators_trm_vs_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 329.26975,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_program_map",
            "kind": "list",
            "matchKey": "regulator",
            "k": 3,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "both"
                    ],
                    "canonical": "both"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "represses"
                    ],
                    "canonical": "represses"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Batf",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "trm_like_til_specific"
                    ],
                    "canonical": "trm_like_til_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "chromatin_over_rna"
                    ],
                    "canonical": "chromatin_over_rna"
                  }
                }
              },
              {
                "key": "Ahr",
                "fields": {
                  "program_specificity": {
                    "op": "in",
                    "values": [
                      "trm_specific",
                      "both"
                    ],
                    "canonical": "trm_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "rna_over_chromatin"
                    ],
                    "canonical": "rna_over_chromatin"
                  }
                }
              }
            ]
          },
          {
            "name": "regulator_set_no_extras_gate",
            "role": "gate",
            "field": "$.regulator_program_map[*]",
            "kind": "other",
            "expected": null
          },
          {
            "name": "program_identity_conclusion_gate",
            "role": "gate",
            "field": "program_identity_conclusion",
            "kind": "value",
            "expected": "distinct_state_shared_brake_divergent_activators"
          },
          {
            "name": "residency_install_logic_gate",
            "role": "gate",
            "field": "residency_install_logic",
            "kind": "value",
            "expected": "brake_release_permits_context_matched_activator"
          },
          {
            "name": "activator_context_dependence_gate",
            "role": "gate",
            "field": "activator_context_dependence",
            "kind": "value",
            "expected": "context_matched_crossover"
          },
          {
            "name": "evidence_ceiling_gate",
            "role": "gate",
            "field": "evidence_ceiling",
            "kind": "value",
            "expected": "specificity_dissociations_require_perturbation"
          },
          {
            "name": "cf_install_logic_flat_or_single_step",
            "role": "hard_fail",
            "field": "residency_install_logic",
            "kind": "value",
            "op": "in",
            "expected": [
              "flat_shared_residency_toolkit",
              "activator_induction_sufficient",
              "brake_release_alone_sufficient"
            ]
          },
          {
            "name": "cf_program_identity_lumped_or_wrong",
            "role": "hard_fail",
            "field": "program_identity_conclusion",
            "kind": "value",
            "op": "in",
            "expected": [
              "trm_like_til_is_tumor_trm",
              "trm_like_til_is_exhausted_subset",
              "trm_like_til_is_effector_state",
              "trm_like_til_is_transitional_intermediate",
              "trm_like_til_is_shared_program_distinct_niche"
            ]
          },
          {
            "name": "cf_ahr_assigned_to_til",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Ahr",
            "valueField": "program_specificity",
            "expected": [
              "trm_like_til_specific"
            ]
          },
          {
            "name": "cf_batf_assigned_to_trm_or_shared",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Batf",
            "valueField": "program_specificity",
            "expected": [
              "trm_specific",
              "both"
            ]
          },
          {
            "name": "cf_klf2_called_activator",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Klf2",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          },
          {
            "name": "cf_evidence_ceiling_overclaim",
            "role": "hard_fail",
            "field": "evidence_ceiling",
            "kind": "value",
            "op": "in",
            "expected": [
              "specificity_established_from_regulon_activity",
              "specificity_established_in_dataset",
              "causation_established_by_cross_sectional_concordance",
              "necessity_and_sufficiency_established"
            ]
          },
          {
            "name": "cf_activator_context_shared_or_single",
            "role": "hard_fail",
            "field": "activator_context_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "shared_activators_both_contexts",
              "single_global_activator",
              "redundant_activators"
            ]
          },
          {
            "name": "cf_single_modality_rule",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_hic1_motif_brake_mislabel",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_runx3_anticanonical_as_controller",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Runx3",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-terra-codex",
      "modelName": "gpt-5.6-terra",
      "harness": "openai-codex",
      "task": "green_shared_vs_distinct_regulators_trm_vs_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1117.853414,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_program_map",
            "kind": "list",
            "matchKey": "regulator",
            "k": 3,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "both"
                    ],
                    "canonical": "both"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "represses"
                    ],
                    "canonical": "represses"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Batf",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "trm_like_til_specific"
                    ],
                    "canonical": "trm_like_til_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "chromatin_over_rna"
                    ],
                    "canonical": "chromatin_over_rna"
                  }
                }
              },
              {
                "key": "Ahr",
                "fields": {
                  "program_specificity": {
                    "op": "in",
                    "values": [
                      "trm_specific",
                      "both"
                    ],
                    "canonical": "trm_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "rna_over_chromatin"
                    ],
                    "canonical": "rna_over_chromatin"
                  }
                }
              }
            ]
          },
          {
            "name": "regulator_set_no_extras_gate",
            "role": "gate",
            "field": "$.regulator_program_map[*]",
            "kind": "other",
            "expected": null
          },
          {
            "name": "program_identity_conclusion_gate",
            "role": "gate",
            "field": "program_identity_conclusion",
            "kind": "value",
            "expected": "distinct_state_shared_brake_divergent_activators"
          },
          {
            "name": "residency_install_logic_gate",
            "role": "gate",
            "field": "residency_install_logic",
            "kind": "value",
            "expected": "brake_release_permits_context_matched_activator"
          },
          {
            "name": "activator_context_dependence_gate",
            "role": "gate",
            "field": "activator_context_dependence",
            "kind": "value",
            "expected": "context_matched_crossover"
          },
          {
            "name": "evidence_ceiling_gate",
            "role": "gate",
            "field": "evidence_ceiling",
            "kind": "value",
            "expected": "specificity_dissociations_require_perturbation"
          },
          {
            "name": "cf_install_logic_flat_or_single_step",
            "role": "hard_fail",
            "field": "residency_install_logic",
            "kind": "value",
            "op": "in",
            "expected": [
              "flat_shared_residency_toolkit",
              "activator_induction_sufficient",
              "brake_release_alone_sufficient"
            ]
          },
          {
            "name": "cf_program_identity_lumped_or_wrong",
            "role": "hard_fail",
            "field": "program_identity_conclusion",
            "kind": "value",
            "op": "in",
            "expected": [
              "trm_like_til_is_tumor_trm",
              "trm_like_til_is_exhausted_subset",
              "trm_like_til_is_effector_state",
              "trm_like_til_is_transitional_intermediate",
              "trm_like_til_is_shared_program_distinct_niche"
            ]
          },
          {
            "name": "cf_ahr_assigned_to_til",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Ahr",
            "valueField": "program_specificity",
            "expected": [
              "trm_like_til_specific"
            ]
          },
          {
            "name": "cf_batf_assigned_to_trm_or_shared",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Batf",
            "valueField": "program_specificity",
            "expected": [
              "trm_specific",
              "both"
            ]
          },
          {
            "name": "cf_klf2_called_activator",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Klf2",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          },
          {
            "name": "cf_evidence_ceiling_overclaim",
            "role": "hard_fail",
            "field": "evidence_ceiling",
            "kind": "value",
            "op": "in",
            "expected": [
              "specificity_established_from_regulon_activity",
              "specificity_established_in_dataset",
              "causation_established_by_cross_sectional_concordance",
              "necessity_and_sufficiency_established"
            ]
          },
          {
            "name": "cf_activator_context_shared_or_single",
            "role": "hard_fail",
            "field": "activator_context_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "shared_activators_both_contexts",
              "single_global_activator",
              "redundant_activators"
            ]
          },
          {
            "name": "cf_single_modality_rule",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_hic1_motif_brake_mislabel",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_runx3_anticanonical_as_controller",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Runx3",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-terra-codex",
      "modelName": "gpt-5.6-terra",
      "harness": "openai-codex",
      "task": "gse297023_q2_senescence_not_mechanism",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 1,
      "passed": true,
      "score": null,
      "durationS": 505.612162,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "candidate_is_senescence",
            "role": "gate",
            "field": "$.candidate_mechanism",
            "kind": "value",
            "op": "in",
            "expected": [
              "cellular_senescence",
              "oncogene_induced_senescence",
              "replicative_senescence"
            ]
          },
          {
            "name": "arrest_core_not_engaged",
            "role": "gate",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_not_engaged"
            ]
          },
          {
            "name": "secretory_arm_is_inflammaging",
            "role": "gate",
            "field": "$.secretory_arm_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "acute_phase_inflammaging_response"
            ]
          },
          {
            "name": "secretory_arm_broadly_elevated",
            "role": "gate",
            "field": "$.secretory_arm_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "broadly_elevated"
            ]
          },
          {
            "name": "carries_aging_program",
            "role": "gate",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "carries_bona_fide_aging_program_distinct_from_nominated"
            ]
          },
          {
            "name": "conclusion_senescence_unlikely",
            "role": "gate",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "unlikely_major_contributor"
            ]
          },
          {
            "name": "cf_arrest_core_engaged_recall",
            "role": "hard_fail",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_engaged"
            ]
          },
          {
            "name": "cf_aging_program_conflation_or_explain_away",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "aging_program_indistinguishable_from_nominated_program",
              "aging_program_is_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_no_aging_program_sensitivity_failure",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_detectable_aging_program"
            ]
          },
          {
            "name": "cf_senescence_driver_conclusion",
            "role": "hard_fail",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "likely_primary_driver",
              "plausible_supported_contributor"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-terra-codex",
      "modelName": "gpt-5.6-terra",
      "harness": "openai-codex",
      "task": "gse297023_q2_senescence_not_mechanism",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 385.705134,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "candidate_is_senescence",
            "role": "gate",
            "field": "$.candidate_mechanism",
            "kind": "value",
            "op": "in",
            "expected": [
              "cellular_senescence",
              "oncogene_induced_senescence",
              "replicative_senescence"
            ]
          },
          {
            "name": "arrest_core_not_engaged",
            "role": "gate",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_not_engaged"
            ]
          },
          {
            "name": "secretory_arm_is_inflammaging",
            "role": "gate",
            "field": "$.secretory_arm_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "acute_phase_inflammaging_response"
            ]
          },
          {
            "name": "secretory_arm_broadly_elevated",
            "role": "gate",
            "field": "$.secretory_arm_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "broadly_elevated"
            ]
          },
          {
            "name": "carries_aging_program",
            "role": "gate",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "carries_bona_fide_aging_program_distinct_from_nominated"
            ]
          },
          {
            "name": "conclusion_senescence_unlikely",
            "role": "gate",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "unlikely_major_contributor"
            ]
          },
          {
            "name": "cf_arrest_core_engaged_recall",
            "role": "hard_fail",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_engaged"
            ]
          },
          {
            "name": "cf_aging_program_conflation_or_explain_away",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "aging_program_indistinguishable_from_nominated_program",
              "aging_program_is_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_no_aging_program_sensitivity_failure",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_detectable_aging_program"
            ]
          },
          {
            "name": "cf_senescence_driver_conclusion",
            "role": "hard_fail",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "likely_primary_driver",
              "plausible_supported_contributor"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-terra-codex",
      "modelName": "gpt-5.6-terra",
      "harness": "openai-codex",
      "task": "gse297023_q2_senescence_not_mechanism",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 925.785048,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "candidate_is_senescence",
            "role": "gate",
            "field": "$.candidate_mechanism",
            "kind": "value",
            "op": "in",
            "expected": [
              "cellular_senescence",
              "oncogene_induced_senescence",
              "replicative_senescence"
            ]
          },
          {
            "name": "arrest_core_not_engaged",
            "role": "gate",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_not_engaged"
            ]
          },
          {
            "name": "secretory_arm_is_inflammaging",
            "role": "gate",
            "field": "$.secretory_arm_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "acute_phase_inflammaging_response"
            ]
          },
          {
            "name": "secretory_arm_broadly_elevated",
            "role": "gate",
            "field": "$.secretory_arm_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "broadly_elevated"
            ]
          },
          {
            "name": "carries_aging_program",
            "role": "gate",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "carries_bona_fide_aging_program_distinct_from_nominated"
            ]
          },
          {
            "name": "conclusion_senescence_unlikely",
            "role": "gate",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "unlikely_major_contributor"
            ]
          },
          {
            "name": "cf_arrest_core_engaged_recall",
            "role": "hard_fail",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_engaged"
            ]
          },
          {
            "name": "cf_aging_program_conflation_or_explain_away",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "aging_program_indistinguishable_from_nominated_program",
              "aging_program_is_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_no_aging_program_sensitivity_failure",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_detectable_aging_program"
            ]
          },
          {
            "name": "cf_senescence_driver_conclusion",
            "role": "hard_fail",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "likely_primary_driver",
              "plausible_supported_contributor"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-terra-codex",
      "modelName": "gpt-5.6-terra",
      "harness": "openai-codex",
      "task": "gse297023_q3_age_retained_mapk_reduced",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 977.47396,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "age_retained",
            "role": "gate",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_retained"
            ]
          },
          {
            "name": "identity_is_aging",
            "role": "gate",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_aging_signatures"
            ]
          },
          {
            "name": "nature_state_shift",
            "role": "gate",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_state_expression_shift"
            ]
          },
          {
            "name": "pathway_is_mapk",
            "role": "gate",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "mapk"
            ]
          },
          {
            "name": "pathway_reduced_in_aged",
            "role": "gate",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "reduced_in_aged"
            ]
          },
          {
            "name": "cf_mapk_increased_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_in_aged"
            ]
          },
          {
            "name": "cf_age_erased",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_erased"
            ]
          },
          {
            "name": "cf_pervasive_overcall",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_pervasive_artifactual"
            ]
          },
          {
            "name": "cf_explain_away_aging",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_compositional_or_identity",
            "role": "hard_fail",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_proportion_shift",
              "cell_identity_shift",
              "composition_dominant_state_secondary",
              "both_state_and_composition_equally"
            ]
          },
          {
            "name": "cf_biggest_inflammatory_signal_not_mapk",
            "role": "hard_fail",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "jak_stat",
              "nfkb",
              "tnfa"
            ]
          },
          {
            "name": "cf_senescence_misattribution",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_senescence_signature_specifically"
            ]
          },
          {
            "name": "cf_dismiss_as_batch_artifact",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_batch_or_technical_artifact"
            ]
          },
          {
            "name": "cf_mapk_comparable_constitutive_kras_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "comparable_across_ages"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-terra-codex",
      "modelName": "gpt-5.6-terra",
      "harness": "openai-codex",
      "task": "gse297023_q3_age_retained_mapk_reduced",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 923.808963,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "age_retained",
            "role": "gate",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_retained"
            ]
          },
          {
            "name": "identity_is_aging",
            "role": "gate",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_aging_signatures"
            ]
          },
          {
            "name": "nature_state_shift",
            "role": "gate",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_state_expression_shift"
            ]
          },
          {
            "name": "pathway_is_mapk",
            "role": "gate",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "mapk"
            ]
          },
          {
            "name": "pathway_reduced_in_aged",
            "role": "gate",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "reduced_in_aged"
            ]
          },
          {
            "name": "cf_mapk_increased_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_in_aged"
            ]
          },
          {
            "name": "cf_age_erased",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_erased"
            ]
          },
          {
            "name": "cf_pervasive_overcall",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_pervasive_artifactual"
            ]
          },
          {
            "name": "cf_explain_away_aging",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_compositional_or_identity",
            "role": "hard_fail",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_proportion_shift",
              "cell_identity_shift",
              "composition_dominant_state_secondary",
              "both_state_and_composition_equally"
            ]
          },
          {
            "name": "cf_biggest_inflammatory_signal_not_mapk",
            "role": "hard_fail",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "jak_stat",
              "nfkb",
              "tnfa"
            ]
          },
          {
            "name": "cf_senescence_misattribution",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_senescence_signature_specifically"
            ]
          },
          {
            "name": "cf_dismiss_as_batch_artifact",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_batch_or_technical_artifact"
            ]
          },
          {
            "name": "cf_mapk_comparable_constitutive_kras_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "comparable_across_ages"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-terra-codex",
      "modelName": "gpt-5.6-terra",
      "harness": "openai-codex",
      "task": "gse297023_q3_age_retained_mapk_reduced",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 852.938823,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "age_retained",
            "role": "gate",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_retained"
            ]
          },
          {
            "name": "identity_is_aging",
            "role": "gate",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_aging_signatures"
            ]
          },
          {
            "name": "nature_state_shift",
            "role": "gate",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_state_expression_shift"
            ]
          },
          {
            "name": "pathway_is_mapk",
            "role": "gate",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "mapk"
            ]
          },
          {
            "name": "pathway_reduced_in_aged",
            "role": "gate",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "reduced_in_aged"
            ]
          },
          {
            "name": "cf_mapk_increased_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_in_aged"
            ]
          },
          {
            "name": "cf_age_erased",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_erased"
            ]
          },
          {
            "name": "cf_pervasive_overcall",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_pervasive_artifactual"
            ]
          },
          {
            "name": "cf_explain_away_aging",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_compositional_or_identity",
            "role": "hard_fail",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_proportion_shift",
              "cell_identity_shift",
              "composition_dominant_state_secondary",
              "both_state_and_composition_equally"
            ]
          },
          {
            "name": "cf_biggest_inflammatory_signal_not_mapk",
            "role": "hard_fail",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "jak_stat",
              "nfkb",
              "tnfa"
            ]
          },
          {
            "name": "cf_senescence_misattribution",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_senescence_signature_specifically"
            ]
          },
          {
            "name": "cf_dismiss_as_batch_artifact",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_batch_or_technical_artifact"
            ]
          },
          {
            "name": "cf_mapk_comparable_constitutive_kras_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "comparable_across_ages"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-terra-codex",
      "modelName": "gpt-5.6-terra",
      "harness": "openai-codex",
      "task": "gse297023_q5_tme_age_dependent_remodeling",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 1,
      "passed": true,
      "score": null,
      "durationS": 1028.446685,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "perturbation_is_pten",
            "role": "gate",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "pten"
            ]
          },
          {
            "name": "tme_non_cell_autonomous",
            "role": "gate",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "substantial_non_cell_autonomous_remodeling"
            ]
          },
          {
            "name": "tme_state_not_composition",
            "role": "gate",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "tme_age_dependent",
            "role": "gate",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_dependent"
            ]
          },
          {
            "name": "tme_response_broader_in_aged",
            "role": "gate",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_aged"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "most_responsive_noncancer_cell_types",
            "kind": "list",
            "matchKey": "cell_type",
            "k": 2,
            "expected": [
              {
                "key": "alveolar_macrophage",
                "fields": {}
              },
              {
                "key": "endothelial",
                "fields": {}
              }
            ]
          },
          {
            "name": "cf_cell_intrinsic_only",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "negligible_cell_intrinsic_only_effect",
              "remodeling_unresolved",
              "modest_remodeling_few_cell_types"
            ]
          },
          {
            "name": "cf_remodeling_is_contamination",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "apparent_remodeling_reflects_cancer_contamination"
            ]
          },
          {
            "name": "cf_perturbation_design_misread",
            "role": "hard_fail",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "multiple_tumor_suppressors_pooled",
              "no_additional_gene_difference_kras_only",
              "rb1_cdkn2a_double",
              "pcna_essential_gene_control",
              "difference_is_oncogene_not_suppressor"
            ]
          },
          {
            "name": "cf_age_independent_tme",
            "role": "hard_fail",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_independent"
            ]
          },
          {
            "name": "cf_compositional_shift",
            "role": "hard_fail",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_frequency_shift",
              "compositional_immune_infiltration",
              "composition_dominant_state_secondary",
              "both_state_and_composition"
            ]
          },
          {
            "name": "cf_epithelial_or_cancer_substitution",
            "role": "hard_fail",
            "field": "$.most_responsive_noncancer_cell_types[*].cell_type",
            "kind": "avoid",
            "expected": [
              "AT2_cell",
              "AT1_cell",
              "cancer_cell",
              "ciliated",
              "secretory",
              "club_cell",
              "basal_cell",
              "goblet_cell"
            ]
          },
          {
            "name": "cf_tme_breadth_broader_in_young",
            "role": "hard_fail",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_young"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-terra-codex",
      "modelName": "gpt-5.6-terra",
      "harness": "openai-codex",
      "task": "gse297023_q5_tme_age_dependent_remodeling",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 906.291878,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "perturbation_is_pten",
            "role": "gate",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "pten"
            ]
          },
          {
            "name": "tme_non_cell_autonomous",
            "role": "gate",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "substantial_non_cell_autonomous_remodeling"
            ]
          },
          {
            "name": "tme_state_not_composition",
            "role": "gate",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "tme_age_dependent",
            "role": "gate",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_dependent"
            ]
          },
          {
            "name": "tme_response_broader_in_aged",
            "role": "gate",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_aged"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "most_responsive_noncancer_cell_types",
            "kind": "list",
            "matchKey": "cell_type",
            "k": 2,
            "expected": [
              {
                "key": "alveolar_macrophage",
                "fields": {}
              },
              {
                "key": "endothelial",
                "fields": {}
              }
            ]
          },
          {
            "name": "cf_cell_intrinsic_only",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "negligible_cell_intrinsic_only_effect",
              "remodeling_unresolved",
              "modest_remodeling_few_cell_types"
            ]
          },
          {
            "name": "cf_remodeling_is_contamination",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "apparent_remodeling_reflects_cancer_contamination"
            ]
          },
          {
            "name": "cf_perturbation_design_misread",
            "role": "hard_fail",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "multiple_tumor_suppressors_pooled",
              "no_additional_gene_difference_kras_only",
              "rb1_cdkn2a_double",
              "pcna_essential_gene_control",
              "difference_is_oncogene_not_suppressor"
            ]
          },
          {
            "name": "cf_age_independent_tme",
            "role": "hard_fail",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_independent"
            ]
          },
          {
            "name": "cf_compositional_shift",
            "role": "hard_fail",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_frequency_shift",
              "compositional_immune_infiltration",
              "composition_dominant_state_secondary",
              "both_state_and_composition"
            ]
          },
          {
            "name": "cf_epithelial_or_cancer_substitution",
            "role": "hard_fail",
            "field": "$.most_responsive_noncancer_cell_types[*].cell_type",
            "kind": "avoid",
            "expected": [
              "AT2_cell",
              "AT1_cell",
              "cancer_cell",
              "ciliated",
              "secretory",
              "club_cell",
              "basal_cell",
              "goblet_cell"
            ]
          },
          {
            "name": "cf_tme_breadth_broader_in_young",
            "role": "hard_fail",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_young"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-terra-codex",
      "modelName": "gpt-5.6-terra",
      "harness": "openai-codex",
      "task": "gse297023_q5_tme_age_dependent_remodeling",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1154.104342,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "perturbation_is_pten",
            "role": "gate",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "pten"
            ]
          },
          {
            "name": "tme_non_cell_autonomous",
            "role": "gate",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "substantial_non_cell_autonomous_remodeling"
            ]
          },
          {
            "name": "tme_state_not_composition",
            "role": "gate",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "tme_age_dependent",
            "role": "gate",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_dependent"
            ]
          },
          {
            "name": "tme_response_broader_in_aged",
            "role": "gate",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_aged"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "most_responsive_noncancer_cell_types",
            "kind": "list",
            "matchKey": "cell_type",
            "k": 2,
            "expected": [
              {
                "key": "alveolar_macrophage",
                "fields": {}
              },
              {
                "key": "endothelial",
                "fields": {}
              }
            ]
          },
          {
            "name": "cf_cell_intrinsic_only",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "negligible_cell_intrinsic_only_effect",
              "remodeling_unresolved",
              "modest_remodeling_few_cell_types"
            ]
          },
          {
            "name": "cf_remodeling_is_contamination",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "apparent_remodeling_reflects_cancer_contamination"
            ]
          },
          {
            "name": "cf_perturbation_design_misread",
            "role": "hard_fail",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "multiple_tumor_suppressors_pooled",
              "no_additional_gene_difference_kras_only",
              "rb1_cdkn2a_double",
              "pcna_essential_gene_control",
              "difference_is_oncogene_not_suppressor"
            ]
          },
          {
            "name": "cf_age_independent_tme",
            "role": "hard_fail",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_independent"
            ]
          },
          {
            "name": "cf_compositional_shift",
            "role": "hard_fail",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_frequency_shift",
              "compositional_immune_infiltration",
              "composition_dominant_state_secondary",
              "both_state_and_composition"
            ]
          },
          {
            "name": "cf_epithelial_or_cancer_substitution",
            "role": "hard_fail",
            "field": "$.most_responsive_noncancer_cell_types[*].cell_type",
            "kind": "avoid",
            "expected": [
              "AT2_cell",
              "AT1_cell",
              "cancer_cell",
              "ciliated",
              "secretory",
              "club_cell",
              "basal_cell",
              "goblet_cell"
            ]
          },
          {
            "name": "cf_tme_breadth_broader_in_young",
            "role": "hard_fail",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_young"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-terra-codex",
      "modelName": "gpt-5.6-terra",
      "harness": "openai-codex",
      "task": "gse297023_q6_age_immune_infiltration",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 1,
      "passed": true,
      "score": null,
      "durationS": 353.447889,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "composition_stable_with_age",
            "role": "gate",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "frequencies_largely_stable"
            ]
          },
          {
            "name": "age_signal_is_state_not_composition",
            "role": "gate",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "frequencies_uninformative_for_infiltration",
            "role": "gate",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_cannot_establish_infiltration"
            ]
          },
          {
            "name": "infiltration_increases_with_age",
            "role": "gate",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_with_age"
            ]
          },
          {
            "name": "infiltration_strength_calibrated",
            "role": "gate",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "significant_increase_strongest_markers_only",
              "consistent_increase_not_statistically_robust"
            ]
          },
          {
            "name": "genotype_marker_fraction_lower_in_pten",
            "role": "gate",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "lower_in_pten_deficient"
            ]
          },
          {
            "name": "cf_composition_shift_reading",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "major_compositional_shift_with_age",
              "increased_immune_fraction_with_age"
            ]
          },
          {
            "name": "cf_lineage_specific_composition_claim",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "myeloid_expansion_with_age",
              "lymphoid_expansion_with_age",
              "minor_shifts_in_some_cell_types"
            ]
          },
          {
            "name": "cf_immunosenescence_depletion_prior",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "immune_depletion_with_age"
            ]
          },
          {
            "name": "cf_age_modality_compositional",
            "role": "hard_fail",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_type_composition_shift",
              "both_equally",
              "composition_dominant_state_secondary"
            ]
          },
          {
            "name": "cf_overread_frequencies_as_infiltration",
            "role": "hard_fail",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_shows_no_infiltration_change",
              "single_cell_quantifies_infiltration",
              "frequencies_show_increased_infiltration",
              "frequencies_show_decreased_infiltration"
            ]
          },
          {
            "name": "cf_infiltration_decreases_immunosenescence",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "decreased_with_age"
            ]
          },
          {
            "name": "cf_infiltration_unchanged_scrna_overread",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "unchanged_with_age"
            ]
          },
          {
            "name": "cf_infiltration_indeterminate_unused_node",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "direction_indeterminate_from_data"
            ]
          },
          {
            "name": "cf_infiltration_mixed_direction",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "mixed_by_immune_cell_type"
            ]
          },
          {
            "name": "cf_infiltration_genotype_confusion",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_only_in_pten_deficient",
              "decreased_only_in_pten_deficient"
            ]
          },
          {
            "name": "cf_pten_infiltration_paper_text_recall",
            "role": "hard_fail",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "higher_in_pten_deficient",
              "higher_in_pten_deficient_only_in_young",
              "higher_in_pten_deficient_only_in_aged"
            ]
          },
          {
            "name": "cf_pseudoreplication_overclaim",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_increase_across_markers"
            ]
          },
          {
            "name": "cf_infiltration_strength_decrease_direction",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_decrease_across_markers",
              "significant_decrease_strongest_markers_only",
              "consistent_decrease_not_statistically_robust"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-terra-codex",
      "modelName": "gpt-5.6-terra",
      "harness": "openai-codex",
      "task": "gse297023_q6_age_immune_infiltration",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 646.23444,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "composition_stable_with_age",
            "role": "gate",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "frequencies_largely_stable"
            ]
          },
          {
            "name": "age_signal_is_state_not_composition",
            "role": "gate",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "frequencies_uninformative_for_infiltration",
            "role": "gate",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_cannot_establish_infiltration"
            ]
          },
          {
            "name": "infiltration_increases_with_age",
            "role": "gate",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_with_age"
            ]
          },
          {
            "name": "infiltration_strength_calibrated",
            "role": "gate",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "significant_increase_strongest_markers_only",
              "consistent_increase_not_statistically_robust"
            ]
          },
          {
            "name": "genotype_marker_fraction_lower_in_pten",
            "role": "gate",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "lower_in_pten_deficient"
            ]
          },
          {
            "name": "cf_composition_shift_reading",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "major_compositional_shift_with_age",
              "increased_immune_fraction_with_age"
            ]
          },
          {
            "name": "cf_lineage_specific_composition_claim",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "myeloid_expansion_with_age",
              "lymphoid_expansion_with_age",
              "minor_shifts_in_some_cell_types"
            ]
          },
          {
            "name": "cf_immunosenescence_depletion_prior",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "immune_depletion_with_age"
            ]
          },
          {
            "name": "cf_age_modality_compositional",
            "role": "hard_fail",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_type_composition_shift",
              "both_equally",
              "composition_dominant_state_secondary"
            ]
          },
          {
            "name": "cf_overread_frequencies_as_infiltration",
            "role": "hard_fail",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_shows_no_infiltration_change",
              "single_cell_quantifies_infiltration",
              "frequencies_show_increased_infiltration",
              "frequencies_show_decreased_infiltration"
            ]
          },
          {
            "name": "cf_infiltration_decreases_immunosenescence",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "decreased_with_age"
            ]
          },
          {
            "name": "cf_infiltration_unchanged_scrna_overread",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "unchanged_with_age"
            ]
          },
          {
            "name": "cf_infiltration_indeterminate_unused_node",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "direction_indeterminate_from_data"
            ]
          },
          {
            "name": "cf_infiltration_mixed_direction",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "mixed_by_immune_cell_type"
            ]
          },
          {
            "name": "cf_infiltration_genotype_confusion",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_only_in_pten_deficient",
              "decreased_only_in_pten_deficient"
            ]
          },
          {
            "name": "cf_pten_infiltration_paper_text_recall",
            "role": "hard_fail",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "higher_in_pten_deficient",
              "higher_in_pten_deficient_only_in_young",
              "higher_in_pten_deficient_only_in_aged"
            ]
          },
          {
            "name": "cf_pseudoreplication_overclaim",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_increase_across_markers"
            ]
          },
          {
            "name": "cf_infiltration_strength_decrease_direction",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_decrease_across_markers",
              "significant_decrease_strongest_markers_only",
              "consistent_decrease_not_statistically_robust"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-terra-codex",
      "modelName": "gpt-5.6-terra",
      "harness": "openai-codex",
      "task": "gse297023_q6_age_immune_infiltration",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 470.956798,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "composition_stable_with_age",
            "role": "gate",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "frequencies_largely_stable"
            ]
          },
          {
            "name": "age_signal_is_state_not_composition",
            "role": "gate",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "frequencies_uninformative_for_infiltration",
            "role": "gate",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_cannot_establish_infiltration"
            ]
          },
          {
            "name": "infiltration_increases_with_age",
            "role": "gate",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_with_age"
            ]
          },
          {
            "name": "infiltration_strength_calibrated",
            "role": "gate",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "significant_increase_strongest_markers_only",
              "consistent_increase_not_statistically_robust"
            ]
          },
          {
            "name": "genotype_marker_fraction_lower_in_pten",
            "role": "gate",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "lower_in_pten_deficient"
            ]
          },
          {
            "name": "cf_composition_shift_reading",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "major_compositional_shift_with_age",
              "increased_immune_fraction_with_age"
            ]
          },
          {
            "name": "cf_lineage_specific_composition_claim",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "myeloid_expansion_with_age",
              "lymphoid_expansion_with_age",
              "minor_shifts_in_some_cell_types"
            ]
          },
          {
            "name": "cf_immunosenescence_depletion_prior",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "immune_depletion_with_age"
            ]
          },
          {
            "name": "cf_age_modality_compositional",
            "role": "hard_fail",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_type_composition_shift",
              "both_equally",
              "composition_dominant_state_secondary"
            ]
          },
          {
            "name": "cf_overread_frequencies_as_infiltration",
            "role": "hard_fail",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_shows_no_infiltration_change",
              "single_cell_quantifies_infiltration",
              "frequencies_show_increased_infiltration",
              "frequencies_show_decreased_infiltration"
            ]
          },
          {
            "name": "cf_infiltration_decreases_immunosenescence",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "decreased_with_age"
            ]
          },
          {
            "name": "cf_infiltration_unchanged_scrna_overread",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "unchanged_with_age"
            ]
          },
          {
            "name": "cf_infiltration_indeterminate_unused_node",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "direction_indeterminate_from_data"
            ]
          },
          {
            "name": "cf_infiltration_mixed_direction",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "mixed_by_immune_cell_type"
            ]
          },
          {
            "name": "cf_infiltration_genotype_confusion",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_only_in_pten_deficient",
              "decreased_only_in_pten_deficient"
            ]
          },
          {
            "name": "cf_pten_infiltration_paper_text_recall",
            "role": "hard_fail",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "higher_in_pten_deficient",
              "higher_in_pten_deficient_only_in_young",
              "higher_in_pten_deficient_only_in_aged"
            ]
          },
          {
            "name": "cf_pseudoreplication_overclaim",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_increase_across_markers"
            ]
          },
          {
            "name": "cf_infiltration_strength_decrease_direction",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_decrease_across_markers",
              "significant_decrease_strongest_markers_only",
              "consistent_decrease_not_statistically_robust"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-5-pi",
      "modelName": "claude-sonnet-5",
      "harness": "pi",
      "task": "gse155381_q4_pseudotime_delay",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 1,
      "passed": true,
      "score": null,
      "durationS": 1496.546746,
      "cost": 2.5302379,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "pseudotime_direction_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "E17_chronological_match_weighted",
            "role": "additive",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pluripotency_exit_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pseudotime_direction_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_synchronous_with_monkey_EPI",
              "chimera_human_EPI_developmentally_advanced",
              "chimera_human_EPI_delayed_at_pre_implantation_only_resolved_post_implantation"
            ]
          },
          {
            "name": "E17_chronological_match_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "resembles_monkey_E17_Gast_EPI",
              "resembles_chimera_monkey_E17_EPI_specifically",
              "chimera_human_E17_EPI_more_advanced_than_monkey_E17_Gast"
            ]
          },
          {
            "name": "pluripotency_exit_paper_inverted_or_prior_driven",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_advances_pluripotency_exit_prematurely",
              "naive_pluripotency_program_specifically_engaged",
              "pluripotency_core_retained_via_KLF17_naive_marker_specifically"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-5-pi",
      "modelName": "claude-sonnet-5",
      "harness": "pi",
      "task": "gse155381_q4_pseudotime_delay",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 1176.315386,
      "cost": 1.7728485000000003,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "pseudotime_direction_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "E17_chronological_match_weighted",
            "role": "additive",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pluripotency_exit_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pseudotime_direction_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_synchronous_with_monkey_EPI",
              "chimera_human_EPI_developmentally_advanced",
              "chimera_human_EPI_delayed_at_pre_implantation_only_resolved_post_implantation"
            ]
          },
          {
            "name": "E17_chronological_match_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "resembles_monkey_E17_Gast_EPI",
              "resembles_chimera_monkey_E17_EPI_specifically",
              "chimera_human_E17_EPI_more_advanced_than_monkey_E17_Gast"
            ]
          },
          {
            "name": "pluripotency_exit_paper_inverted_or_prior_driven",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_advances_pluripotency_exit_prematurely",
              "naive_pluripotency_program_specifically_engaged",
              "pluripotency_core_retained_via_KLF17_naive_marker_specifically"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-5-pi",
      "modelName": "claude-sonnet-5",
      "harness": "pi",
      "task": "gse155381_q4_pseudotime_delay",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1393.024414,
      "cost": 1.3992676000000004,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "pseudotime_direction_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "E17_chronological_match_weighted",
            "role": "additive",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pluripotency_exit_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pseudotime_direction_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_synchronous_with_monkey_EPI",
              "chimera_human_EPI_developmentally_advanced",
              "chimera_human_EPI_delayed_at_pre_implantation_only_resolved_post_implantation"
            ]
          },
          {
            "name": "E17_chronological_match_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "resembles_monkey_E17_Gast_EPI",
              "resembles_chimera_monkey_E17_EPI_specifically",
              "chimera_human_E17_EPI_more_advanced_than_monkey_E17_Gast"
            ]
          },
          {
            "name": "pluripotency_exit_paper_inverted_or_prior_driven",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_advances_pluripotency_exit_prematurely",
              "naive_pluripotency_program_specifically_engaged",
              "pluripotency_core_retained_via_KLF17_naive_marker_specifically"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-5-pi",
      "modelName": "claude-sonnet-5",
      "harness": "pi",
      "task": "gse155381_q5_chimera_monkey_perturbation",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 812.737541,
      "cost": 0.9790688,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "deg_direction_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "comparison_subset_choice_weighted",
            "role": "additive",
            "field": "$.comparison_subset_choice",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "panel_resolved_strong_target_ccn1_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN1",
              "CYR61"
            ]
          },
          {
            "name": "panel_resolved_strong_target_ccn2_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN2",
              "CTGF"
            ]
          },
          {
            "name": "hippo_pathway_weighted",
            "role": "additive",
            "field": "$.hippo_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "tgf_beta_pathway_weighted",
            "role": "additive",
            "field": "$.tgf_beta_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "deg_direction_batch_effect_uncorrected",
            "role": "hard_fail",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_dominated_global",
              "up_dominated_in_chosen_subset"
            ]
          },
          {
            "name": "silent_panel_dropout_paired_with_directional_pathway_claim",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "hippo_pathway_direction": {
                "op": "in",
                "values": [
                  "hippo_pathway_upregulated",
                  "hippo_pathway_unchanged",
                  "wnt_pathway_specifically_perturbed",
                  "stress_response_pathway_dominantly_engaged",
                  "no_pathway_signal_above_confound",
                  "yap_tead_direct_target_panel_downregulated",
                  "yap_tead_panel_dependent_direction",
                  "hippo_pathway_downregulated"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-5-pi",
      "modelName": "claude-sonnet-5",
      "harness": "pi",
      "task": "gse155381_q5_chimera_monkey_perturbation",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 710.675043,
      "cost": 0.6625527,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "deg_direction_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "comparison_subset_choice_weighted",
            "role": "additive",
            "field": "$.comparison_subset_choice",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "panel_resolved_strong_target_ccn1_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN1",
              "CYR61"
            ]
          },
          {
            "name": "panel_resolved_strong_target_ccn2_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN2",
              "CTGF"
            ]
          },
          {
            "name": "hippo_pathway_weighted",
            "role": "additive",
            "field": "$.hippo_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "tgf_beta_pathway_weighted",
            "role": "additive",
            "field": "$.tgf_beta_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "deg_direction_batch_effect_uncorrected",
            "role": "hard_fail",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_dominated_global",
              "up_dominated_in_chosen_subset"
            ]
          },
          {
            "name": "silent_panel_dropout_paired_with_directional_pathway_claim",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "hippo_pathway_direction": {
                "op": "in",
                "values": [
                  "hippo_pathway_upregulated",
                  "hippo_pathway_unchanged",
                  "wnt_pathway_specifically_perturbed",
                  "stress_response_pathway_dominantly_engaged",
                  "no_pathway_signal_above_confound",
                  "yap_tead_direct_target_panel_downregulated",
                  "yap_tead_panel_dependent_direction",
                  "hippo_pathway_downregulated"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-5-pi",
      "modelName": "claude-sonnet-5",
      "harness": "pi",
      "task": "gse155381_q5_chimera_monkey_perturbation",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 775.492115,
      "cost": 0.7819462,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "deg_direction_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "comparison_subset_choice_weighted",
            "role": "additive",
            "field": "$.comparison_subset_choice",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "panel_resolved_strong_target_ccn1_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN1",
              "CYR61"
            ]
          },
          {
            "name": "panel_resolved_strong_target_ccn2_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN2",
              "CTGF"
            ]
          },
          {
            "name": "hippo_pathway_weighted",
            "role": "additive",
            "field": "$.hippo_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "tgf_beta_pathway_weighted",
            "role": "additive",
            "field": "$.tgf_beta_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "deg_direction_batch_effect_uncorrected",
            "role": "hard_fail",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_dominated_global",
              "up_dominated_in_chosen_subset"
            ]
          },
          {
            "name": "silent_panel_dropout_paired_with_directional_pathway_claim",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "hippo_pathway_direction": {
                "op": "in",
                "values": [
                  "hippo_pathway_upregulated",
                  "hippo_pathway_unchanged",
                  "wnt_pathway_specifically_perturbed",
                  "stress_response_pathway_dominantly_engaged",
                  "no_pathway_signal_above_confound",
                  "yap_tead_direct_target_panel_downregulated",
                  "yap_tead_panel_dependent_direction",
                  "hippo_pathway_downregulated"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-5-pi",
      "modelName": "claude-sonnet-5",
      "harness": "pi",
      "task": "gse155381_q7_xspecies_epi_communication",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 980.752275,
      "cost": 1.2836413000000002,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "richness_weighted",
            "role": "additive",
            "field": "$.xspecies_signaling_richness",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_weighted",
            "role": "additive",
            "field": "$.dominant_signaling_axis",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_wnt_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "WNT_FZD_LRP_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_fgf_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "FGF_FGFR_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_notch_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "NOTCH_DLL_JAG_family"
                ]
              }
            }
          },
          {
            "name": "signaling_directionality_weighted",
            "role": "additive",
            "field": "$.signaling_directionality",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "top_signal_interaction_class_weighted",
            "role": "additive",
            "field": "$.top_signal_interaction_class",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-5-pi",
      "modelName": "claude-sonnet-5",
      "harness": "pi",
      "task": "gse155381_q7_xspecies_epi_communication",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 4625.95493,
      "cost": 2.8737712000000006,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "richness_weighted",
            "role": "additive",
            "field": "$.xspecies_signaling_richness",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_weighted",
            "role": "additive",
            "field": "$.dominant_signaling_axis",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_wnt_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "WNT_FZD_LRP_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_fgf_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "FGF_FGFR_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_notch_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "NOTCH_DLL_JAG_family"
                ]
              }
            }
          },
          {
            "name": "signaling_directionality_weighted",
            "role": "additive",
            "field": "$.signaling_directionality",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "top_signal_interaction_class_weighted",
            "role": "additive",
            "field": "$.top_signal_interaction_class",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-5-pi",
      "modelName": "claude-sonnet-5",
      "harness": "pi",
      "task": "gse155381_q7_xspecies_epi_communication",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 958.746929,
      "cost": 0.8477139,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "richness_weighted",
            "role": "additive",
            "field": "$.xspecies_signaling_richness",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_weighted",
            "role": "additive",
            "field": "$.dominant_signaling_axis",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_wnt_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "WNT_FZD_LRP_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_fgf_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "FGF_FGFR_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_notch_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "NOTCH_DLL_JAG_family"
                ]
              }
            }
          },
          {
            "name": "signaling_directionality_weighted",
            "role": "additive",
            "field": "$.signaling_directionality",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "top_signal_interaction_class_weighted",
            "role": "additive",
            "field": "$.top_signal_interaction_class",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-5-pi",
      "modelName": "claude-sonnet-5",
      "harness": "pi",
      "task": "gse155381_q8_bidirectional_convergence",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 2071.058662,
      "cost": 3.9135114000000004,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "chimera_human_identity_shift_target_weighted",
            "role": "additive",
            "field": "$.chimera_human_identity_shift_target",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_in_vivo_state_assignment_weighted",
            "role": "additive",
            "field": "$.chimera_human_in_vivo_state_assignment",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_shift_magnitude_weighted",
            "role": "additive",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_monkey_identity_shift_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "both_arms_null_contradicted_by_calibration",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "chimera_human_identity_shift_target": {
                "op": "equals",
                "values": [
                  "chimera_human_maintains_origin_identity_no_measurable_shift"
                ]
              },
              "chimera_monkey_identity_shift": {
                "op": "equals",
                "values": [
                  "chimera_monkey_unchanged_relative_to_control"
                ]
              }
            }
          },
          {
            "name": "chimera_monkey_paper_claim_on_confounded_input",
            "role": "hard_fail",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "value",
            "expected": "chimera_monkey_shifts_toward_chimera_human_identity"
          },
          {
            "name": "chimera_human_magnitude_wrong_attribution_signature",
            "role": "hard_fail",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "value",
            "op": "in",
            "expected": [
              "dramatic_complete_replacement_of_origin_identity",
              "magnitude_dominated_by_culture_or_metabolic_axis",
              "magnitude_dominated_by_developmental_stage_axis",
              "magnitude_dominated_by_cell_cycle_axis",
              "magnitude_dominated_by_batch_effect_axis"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-5-pi",
      "modelName": "claude-sonnet-5",
      "harness": "pi",
      "task": "gse155381_q8_bidirectional_convergence",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 1294.890534,
      "cost": 2.0157955000000003,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "chimera_human_identity_shift_target_weighted",
            "role": "additive",
            "field": "$.chimera_human_identity_shift_target",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_in_vivo_state_assignment_weighted",
            "role": "additive",
            "field": "$.chimera_human_in_vivo_state_assignment",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_shift_magnitude_weighted",
            "role": "additive",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_monkey_identity_shift_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "both_arms_null_contradicted_by_calibration",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "chimera_human_identity_shift_target": {
                "op": "equals",
                "values": [
                  "chimera_human_maintains_origin_identity_no_measurable_shift"
                ]
              },
              "chimera_monkey_identity_shift": {
                "op": "equals",
                "values": [
                  "chimera_monkey_unchanged_relative_to_control"
                ]
              }
            }
          },
          {
            "name": "chimera_monkey_paper_claim_on_confounded_input",
            "role": "hard_fail",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "value",
            "expected": "chimera_monkey_shifts_toward_chimera_human_identity"
          },
          {
            "name": "chimera_human_magnitude_wrong_attribution_signature",
            "role": "hard_fail",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "value",
            "op": "in",
            "expected": [
              "dramatic_complete_replacement_of_origin_identity",
              "magnitude_dominated_by_culture_or_metabolic_axis",
              "magnitude_dominated_by_developmental_stage_axis",
              "magnitude_dominated_by_cell_cycle_axis",
              "magnitude_dominated_by_batch_effect_axis"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-5-pi",
      "modelName": "claude-sonnet-5",
      "harness": "pi",
      "task": "gse155381_q8_bidirectional_convergence",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1731.513754,
      "cost": 3.1116109999999995,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "chimera_human_identity_shift_target_weighted",
            "role": "additive",
            "field": "$.chimera_human_identity_shift_target",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_in_vivo_state_assignment_weighted",
            "role": "additive",
            "field": "$.chimera_human_in_vivo_state_assignment",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_shift_magnitude_weighted",
            "role": "additive",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_monkey_identity_shift_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "both_arms_null_contradicted_by_calibration",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "chimera_human_identity_shift_target": {
                "op": "equals",
                "values": [
                  "chimera_human_maintains_origin_identity_no_measurable_shift"
                ]
              },
              "chimera_monkey_identity_shift": {
                "op": "equals",
                "values": [
                  "chimera_monkey_unchanged_relative_to_control"
                ]
              }
            }
          },
          {
            "name": "chimera_monkey_paper_claim_on_confounded_input",
            "role": "hard_fail",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "value",
            "expected": "chimera_monkey_shifts_toward_chimera_human_identity"
          },
          {
            "name": "chimera_human_magnitude_wrong_attribution_signature",
            "role": "hard_fail",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "value",
            "op": "in",
            "expected": [
              "dramatic_complete_replacement_of_origin_identity",
              "magnitude_dominated_by_culture_or_metabolic_axis",
              "magnitude_dominated_by_developmental_stage_axis",
              "magnitude_dominated_by_cell_cycle_axis",
              "magnitude_dominated_by_batch_effect_axis"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-5-pi",
      "modelName": "claude-sonnet-5",
      "harness": "pi",
      "task": "gse171524_ectopic_airway_rare_cell",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 5861.65275,
      "cost": 1.0305154000000003,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "focus_population_identity_gate",
            "role": "gate",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "chemosensory_marker_expressing_population_alpha"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "focus_population_marker_calls",
            "kind": "list",
            "matchKey": "marker",
            "k": 3,
            "expected": [
              {
                "key": "POU2F3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "CHAT",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "TRPM5",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "LRMP",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNAT3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "AVIL",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNG13",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              }
            ]
          },
          {
            "name": "focus_population_condition_distribution_gate",
            "role": "gate",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_enriched_with_substantial_control_presence",
              "covid_predominant",
              "covid_only",
              "covid_enriched_trend_not_significant"
            ]
          },
          {
            "name": "focus_population_location_gate",
            "role": "gate",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "ectopic_parenchyma_in_covid",
              "parenchyma_only",
              "bronchiolized_or_peribronchial_parenchyma",
              "airway_and_parenchyma"
            ]
          },
          {
            "name": "focus_population_recovery_mode_gate",
            "role": "gate",
            "field": "$.focus_population_recovery_mode",
            "kind": "value",
            "op": "in",
            "expected": [
              "continuum_not_discrete_cluster"
            ]
          },
          {
            "name": "focus_population_identity_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "ionocyte_like_beta",
              "neuroendocrine_like_gamma",
              "basal_stem_progenitor_like_delta",
              "aberrant_basaloid_transitional_like_epsilon",
              "neuronal_or_neural_gene_minor_population_zeta",
              "deuterosomal_ciliated_progenitor_like_eta",
              "hillock_transitional_like_theta",
              "low_quality_doublet_or_debris",
              "no_rare_cell_recovered",
              "not_assessed"
            ]
          },
          {
            "name": "cf_wrong_lineage_marker_called_positive",
            "role": "hard_fail",
            "field": "$.focus_population_marker_calls[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "focus_population_condition_distribution_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "equally_distributed",
              "control_predominant",
              "not_assessed"
            ]
          },
          {
            "name": "focus_population_location_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "airway_only",
              "not_assessed"
            ]
          },
          {
            "name": "cross_axis_canonical_identity_with_airway_only_location_incoherence",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "focus_population_identity": {
                "op": "in",
                "values": [
                  "chemosensory_marker_expressing_population_alpha"
                ]
              },
              "focus_population_location": {
                "op": "in",
                "values": [
                  "airway_only"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-5-pi",
      "modelName": "claude-sonnet-5",
      "harness": "pi",
      "task": "gse171524_ectopic_airway_rare_cell",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 3690.498636,
      "cost": 0.9267967000000001,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "focus_population_identity_gate",
            "role": "gate",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "chemosensory_marker_expressing_population_alpha"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "focus_population_marker_calls",
            "kind": "list",
            "matchKey": "marker",
            "k": 3,
            "expected": [
              {
                "key": "POU2F3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "CHAT",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "TRPM5",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "LRMP",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNAT3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "AVIL",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNG13",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              }
            ]
          },
          {
            "name": "focus_population_condition_distribution_gate",
            "role": "gate",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_enriched_with_substantial_control_presence",
              "covid_predominant",
              "covid_only",
              "covid_enriched_trend_not_significant"
            ]
          },
          {
            "name": "focus_population_location_gate",
            "role": "gate",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "ectopic_parenchyma_in_covid",
              "parenchyma_only",
              "bronchiolized_or_peribronchial_parenchyma",
              "airway_and_parenchyma"
            ]
          },
          {
            "name": "focus_population_recovery_mode_gate",
            "role": "gate",
            "field": "$.focus_population_recovery_mode",
            "kind": "value",
            "op": "in",
            "expected": [
              "continuum_not_discrete_cluster"
            ]
          },
          {
            "name": "focus_population_identity_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "ionocyte_like_beta",
              "neuroendocrine_like_gamma",
              "basal_stem_progenitor_like_delta",
              "aberrant_basaloid_transitional_like_epsilon",
              "neuronal_or_neural_gene_minor_population_zeta",
              "deuterosomal_ciliated_progenitor_like_eta",
              "hillock_transitional_like_theta",
              "low_quality_doublet_or_debris",
              "no_rare_cell_recovered",
              "not_assessed"
            ]
          },
          {
            "name": "cf_wrong_lineage_marker_called_positive",
            "role": "hard_fail",
            "field": "$.focus_population_marker_calls[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "focus_population_condition_distribution_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "equally_distributed",
              "control_predominant",
              "not_assessed"
            ]
          },
          {
            "name": "focus_population_location_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "airway_only",
              "not_assessed"
            ]
          },
          {
            "name": "cross_axis_canonical_identity_with_airway_only_location_incoherence",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "focus_population_identity": {
                "op": "in",
                "values": [
                  "chemosensory_marker_expressing_population_alpha"
                ]
              },
              "focus_population_location": {
                "op": "in",
                "values": [
                  "airway_only"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-5-pi",
      "modelName": "claude-sonnet-5",
      "harness": "pi",
      "task": "gse171524_ectopic_airway_rare_cell",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 3119.211805,
      "cost": 1.1409607000000002,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "focus_population_identity_gate",
            "role": "gate",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "chemosensory_marker_expressing_population_alpha"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "focus_population_marker_calls",
            "kind": "list",
            "matchKey": "marker",
            "k": 3,
            "expected": [
              {
                "key": "POU2F3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "CHAT",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "TRPM5",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "LRMP",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNAT3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "AVIL",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNG13",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              }
            ]
          },
          {
            "name": "focus_population_condition_distribution_gate",
            "role": "gate",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_enriched_with_substantial_control_presence",
              "covid_predominant",
              "covid_only",
              "covid_enriched_trend_not_significant"
            ]
          },
          {
            "name": "focus_population_location_gate",
            "role": "gate",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "ectopic_parenchyma_in_covid",
              "parenchyma_only",
              "bronchiolized_or_peribronchial_parenchyma",
              "airway_and_parenchyma"
            ]
          },
          {
            "name": "focus_population_recovery_mode_gate",
            "role": "gate",
            "field": "$.focus_population_recovery_mode",
            "kind": "value",
            "op": "in",
            "expected": [
              "continuum_not_discrete_cluster"
            ]
          },
          {
            "name": "focus_population_identity_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "ionocyte_like_beta",
              "neuroendocrine_like_gamma",
              "basal_stem_progenitor_like_delta",
              "aberrant_basaloid_transitional_like_epsilon",
              "neuronal_or_neural_gene_minor_population_zeta",
              "deuterosomal_ciliated_progenitor_like_eta",
              "hillock_transitional_like_theta",
              "low_quality_doublet_or_debris",
              "no_rare_cell_recovered",
              "not_assessed"
            ]
          },
          {
            "name": "cf_wrong_lineage_marker_called_positive",
            "role": "hard_fail",
            "field": "$.focus_population_marker_calls[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "focus_population_condition_distribution_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "equally_distributed",
              "control_predominant",
              "not_assessed"
            ]
          },
          {
            "name": "focus_population_location_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "airway_only",
              "not_assessed"
            ]
          },
          {
            "name": "cross_axis_canonical_identity_with_airway_only_location_incoherence",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "focus_population_identity": {
                "op": "in",
                "values": [
                  "chemosensory_marker_expressing_population_alpha"
                ]
              },
              "focus_population_location": {
                "op": "in",
                "values": [
                  "airway_only"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-5-pi",
      "modelName": "claude-sonnet-5",
      "harness": "pi",
      "task": "gse171524_pathological_fibroblast_master_regulator_druggable_targets",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 1498.053978,
      "cost": 1.1938134000000002,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "disease_associated_fibroblast_subtype_weighted",
            "role": "additive",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "mr_tf_junb_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_alpha"
            ]
          },
          {
            "name": "mr_tf_jund_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_beta"
            ]
          },
          {
            "name": "drug_mmp14_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_alpha"
            ]
          },
          {
            "name": "drug_stat3_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_beta"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_weighted",
            "role": "additive",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "disease_associated_fibroblast_subtype_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "value",
            "op": "in",
            "expected": [
              "subtype_lambda",
              "subtype_epsilon",
              "subtype_iota",
              "subtype_beta",
              "subtype_eta",
              "subtype_zeta",
              "subtype_not_assessed"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "simple_de_ranking_by_log_fc",
              "marker_based_de_only",
              "not_computed"
            ]
          },
          {
            "name": "top_master_regulator_transcription_factors_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_not_computed"
            ]
          },
          {
            "name": "top_druggable_proteins_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_not_assessed"
            ]
          },
          {
            "name": "cross_axis_circular_marker_in_druggable_proteins",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_gamma",
              "drug_target_delta",
              "drug_target_eta"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-5-pi",
      "modelName": "claude-sonnet-5",
      "harness": "pi",
      "task": "gse171524_pathological_fibroblast_master_regulator_druggable_targets",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 1390.11893,
      "cost": 1.1711143,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "disease_associated_fibroblast_subtype_weighted",
            "role": "additive",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "mr_tf_junb_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_alpha"
            ]
          },
          {
            "name": "mr_tf_jund_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_beta"
            ]
          },
          {
            "name": "drug_mmp14_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_alpha"
            ]
          },
          {
            "name": "drug_stat3_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_beta"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_weighted",
            "role": "additive",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "disease_associated_fibroblast_subtype_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "value",
            "op": "in",
            "expected": [
              "subtype_lambda",
              "subtype_epsilon",
              "subtype_iota",
              "subtype_beta",
              "subtype_eta",
              "subtype_zeta",
              "subtype_not_assessed"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "simple_de_ranking_by_log_fc",
              "marker_based_de_only",
              "not_computed"
            ]
          },
          {
            "name": "top_master_regulator_transcription_factors_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_not_computed"
            ]
          },
          {
            "name": "top_druggable_proteins_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_not_assessed"
            ]
          },
          {
            "name": "cross_axis_circular_marker_in_druggable_proteins",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_gamma",
              "drug_target_delta",
              "drug_target_eta"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-5-pi",
      "modelName": "claude-sonnet-5",
      "harness": "pi",
      "task": "gse171524_pathological_fibroblast_master_regulator_druggable_targets",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 3742.529453,
      "cost": 2.135499,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "disease_associated_fibroblast_subtype_weighted",
            "role": "additive",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "mr_tf_junb_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_alpha"
            ]
          },
          {
            "name": "mr_tf_jund_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_beta"
            ]
          },
          {
            "name": "drug_mmp14_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_alpha"
            ]
          },
          {
            "name": "drug_stat3_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_beta"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_weighted",
            "role": "additive",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "disease_associated_fibroblast_subtype_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "value",
            "op": "in",
            "expected": [
              "subtype_lambda",
              "subtype_epsilon",
              "subtype_iota",
              "subtype_beta",
              "subtype_eta",
              "subtype_zeta",
              "subtype_not_assessed"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "simple_de_ranking_by_log_fc",
              "marker_based_de_only",
              "not_computed"
            ]
          },
          {
            "name": "top_master_regulator_transcription_factors_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_not_computed"
            ]
          },
          {
            "name": "top_druggable_proteins_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_not_assessed"
            ]
          },
          {
            "name": "cross_axis_circular_marker_in_druggable_proteins",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_gamma",
              "drug_target_delta",
              "drug_target_eta"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-5-pi",
      "modelName": "claude-sonnet-5",
      "harness": "pi",
      "task": "gse171524_plasma_bcr_neutralizing_clonotype",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 582.366066,
      "cost": 0.4758828,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dominant_pairing_heavy_gate",
            "role": "gate",
            "field": "$.dominant_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV4-34"
            ]
          },
          {
            "name": "dominant_pairing_light_gate",
            "role": "gate",
            "field": "$.dominant_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGLV3-19"
            ]
          },
          {
            "name": "dominant_neutralizer_match_gate",
            "role": "gate",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_known_neutralizer_match"
            ]
          },
          {
            "name": "neutralizer_pairing_heavy_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV1-18"
            ]
          },
          {
            "name": "neutralizer_pairing_light_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGKV3-20"
            ]
          },
          {
            "name": "neutralizer_recurrence_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "observed_in_2_to_5_donors"
            ]
          },
          {
            "name": "neutralizer_rank_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "top_5_pairing",
              "top_6_to_20_pairing"
            ]
          },
          {
            "name": "dominant_clone_isotype_gate",
            "role": "gate",
            "field": "$.dominant_clone_isotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "IgG"
            ]
          },
          {
            "name": "blineage_class_switch_contrast_gate",
            "role": "gate",
            "field": "$.blineage_class_switch_contrast",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_class_switched_control_higher_unswitched",
              "class_switched_in_both_cohorts"
            ]
          },
          {
            "name": "dominant_match_narrative_trap_or_null",
            "role": "hard_fail",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_rbd_neutralizer",
              "matches_known_autoreactive_clone",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_heavy",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_light",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_recurrence_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_rank_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "cross_axis_found_dominant_but_failed_to_scan_for_embedded_neutralizer",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_pairing_neutralizer_match": {
                "op": "in",
                "values": [
                  "no_known_neutralizer_match"
                ]
              },
              "neutralizer_pairing_heavy": {
                "op": "in",
                "values": [
                  "none_detected"
                ]
              }
            }
          },
          {
            "name": "neutralizer_pairing_not_exactly_the_s309_pair",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "op": "not",
            "expected": {
              "op": "and",
              "args": [
                {
                  "op": "field",
                  "name": "neutralizer_pairing_heavy",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGHV1-18"
                    ]
                  }
                },
                {
                  "op": "field",
                  "name": "neutralizer_pairing_light",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGKV3-20"
                    ]
                  }
                }
              ]
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-5-pi",
      "modelName": "claude-sonnet-5",
      "harness": "pi",
      "task": "gse171524_plasma_bcr_neutralizing_clonotype",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 1018.584463,
      "cost": 0.5478063,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dominant_pairing_heavy_gate",
            "role": "gate",
            "field": "$.dominant_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV4-34"
            ]
          },
          {
            "name": "dominant_pairing_light_gate",
            "role": "gate",
            "field": "$.dominant_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGLV3-19"
            ]
          },
          {
            "name": "dominant_neutralizer_match_gate",
            "role": "gate",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_known_neutralizer_match"
            ]
          },
          {
            "name": "neutralizer_pairing_heavy_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV1-18"
            ]
          },
          {
            "name": "neutralizer_pairing_light_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGKV3-20"
            ]
          },
          {
            "name": "neutralizer_recurrence_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "observed_in_2_to_5_donors"
            ]
          },
          {
            "name": "neutralizer_rank_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "top_5_pairing",
              "top_6_to_20_pairing"
            ]
          },
          {
            "name": "dominant_clone_isotype_gate",
            "role": "gate",
            "field": "$.dominant_clone_isotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "IgG"
            ]
          },
          {
            "name": "blineage_class_switch_contrast_gate",
            "role": "gate",
            "field": "$.blineage_class_switch_contrast",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_class_switched_control_higher_unswitched",
              "class_switched_in_both_cohorts"
            ]
          },
          {
            "name": "dominant_match_narrative_trap_or_null",
            "role": "hard_fail",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_rbd_neutralizer",
              "matches_known_autoreactive_clone",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_heavy",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_light",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_recurrence_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_rank_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "cross_axis_found_dominant_but_failed_to_scan_for_embedded_neutralizer",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_pairing_neutralizer_match": {
                "op": "in",
                "values": [
                  "no_known_neutralizer_match"
                ]
              },
              "neutralizer_pairing_heavy": {
                "op": "in",
                "values": [
                  "none_detected"
                ]
              }
            }
          },
          {
            "name": "neutralizer_pairing_not_exactly_the_s309_pair",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "op": "not",
            "expected": {
              "op": "and",
              "args": [
                {
                  "op": "field",
                  "name": "neutralizer_pairing_heavy",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGHV1-18"
                    ]
                  }
                },
                {
                  "op": "field",
                  "name": "neutralizer_pairing_light",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGKV3-20"
                    ]
                  }
                }
              ]
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-5-pi",
      "modelName": "claude-sonnet-5",
      "harness": "pi",
      "task": "gse171524_plasma_bcr_neutralizing_clonotype",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 791.515589,
      "cost": 0.462258,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dominant_pairing_heavy_gate",
            "role": "gate",
            "field": "$.dominant_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV4-34"
            ]
          },
          {
            "name": "dominant_pairing_light_gate",
            "role": "gate",
            "field": "$.dominant_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGLV3-19"
            ]
          },
          {
            "name": "dominant_neutralizer_match_gate",
            "role": "gate",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_known_neutralizer_match"
            ]
          },
          {
            "name": "neutralizer_pairing_heavy_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV1-18"
            ]
          },
          {
            "name": "neutralizer_pairing_light_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGKV3-20"
            ]
          },
          {
            "name": "neutralizer_recurrence_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "observed_in_2_to_5_donors"
            ]
          },
          {
            "name": "neutralizer_rank_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "top_5_pairing",
              "top_6_to_20_pairing"
            ]
          },
          {
            "name": "dominant_clone_isotype_gate",
            "role": "gate",
            "field": "$.dominant_clone_isotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "IgG"
            ]
          },
          {
            "name": "blineage_class_switch_contrast_gate",
            "role": "gate",
            "field": "$.blineage_class_switch_contrast",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_class_switched_control_higher_unswitched",
              "class_switched_in_both_cohorts"
            ]
          },
          {
            "name": "dominant_match_narrative_trap_or_null",
            "role": "hard_fail",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_rbd_neutralizer",
              "matches_known_autoreactive_clone",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_heavy",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_light",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_recurrence_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_rank_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "cross_axis_found_dominant_but_failed_to_scan_for_embedded_neutralizer",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_pairing_neutralizer_match": {
                "op": "in",
                "values": [
                  "no_known_neutralizer_match"
                ]
              },
              "neutralizer_pairing_heavy": {
                "op": "in",
                "values": [
                  "none_detected"
                ]
              }
            }
          },
          {
            "name": "neutralizer_pairing_not_exactly_the_s309_pair",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "op": "not",
            "expected": {
              "op": "and",
              "args": [
                {
                  "op": "field",
                  "name": "neutralizer_pairing_heavy",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGHV1-18"
                    ]
                  }
                },
                {
                  "op": "field",
                  "name": "neutralizer_pairing_light",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGKV3-20"
                    ]
                  }
                }
              ]
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-5-pi",
      "modelName": "claude-sonnet-5",
      "harness": "pi",
      "task": "q6_myeloid_subset_restricted_de",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 1,
      "passed": true,
      "score": null,
      "durationS": 2434.879909,
      "cost": 1.4235857000000007,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "myeloid_subset_with_strongest_change_gate",
            "role": "gate",
            "field": "$.myeloid_subset_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "subset_gamma"
            ]
          },
          {
            "name": "gene_with_strongest_change_gate",
            "role": "gate",
            "field": "$.gene_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "AXL",
              "PROS1"
            ]
          },
          {
            "name": "direction_of_change_gate",
            "role": "gate",
            "field": "$.direction_of_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "down_in_covid"
            ]
          },
          {
            "name": "biological_function_interpretation_gate",
            "role": "gate",
            "field": "$.biological_function_interpretation",
            "kind": "value",
            "op": "in",
            "expected": [
              "efferocytosis",
              "phagocytosis"
            ]
          },
          {
            "name": "myeloid_compartment_abundance_gate",
            "role": "gate",
            "field": "$.myeloid_compartment_abundance",
            "kind": "value",
            "op": "in",
            "expected": [
              "expands_in_covid"
            ]
          },
          {
            "name": "myeloid_expansion_localization_gate",
            "role": "gate",
            "field": "$.myeloid_expansion_localization",
            "kind": "value",
            "op": "in",
            "expected": [
              "recruited_driven_resident_dysfunction"
            ]
          },
          {
            "name": "dominant_signature_gate",
            "role": "gate",
            "field": "$.dominant_signature",
            "kind": "value",
            "op": "in",
            "expected": [
              "technical_or_agonal_artifact"
            ]
          },
          {
            "name": "dominant_signature_direction_gate",
            "role": "gate",
            "field": "$.dominant_signature_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_in_covid"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-5-pi",
      "modelName": "claude-sonnet-5",
      "harness": "pi",
      "task": "q6_myeloid_subset_restricted_de",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 2512.742524,
      "cost": 1.2163105,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "myeloid_subset_with_strongest_change_gate",
            "role": "gate",
            "field": "$.myeloid_subset_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "subset_gamma"
            ]
          },
          {
            "name": "gene_with_strongest_change_gate",
            "role": "gate",
            "field": "$.gene_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "AXL",
              "PROS1"
            ]
          },
          {
            "name": "direction_of_change_gate",
            "role": "gate",
            "field": "$.direction_of_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "down_in_covid"
            ]
          },
          {
            "name": "biological_function_interpretation_gate",
            "role": "gate",
            "field": "$.biological_function_interpretation",
            "kind": "value",
            "op": "in",
            "expected": [
              "efferocytosis",
              "phagocytosis"
            ]
          },
          {
            "name": "myeloid_compartment_abundance_gate",
            "role": "gate",
            "field": "$.myeloid_compartment_abundance",
            "kind": "value",
            "op": "in",
            "expected": [
              "expands_in_covid"
            ]
          },
          {
            "name": "myeloid_expansion_localization_gate",
            "role": "gate",
            "field": "$.myeloid_expansion_localization",
            "kind": "value",
            "op": "in",
            "expected": [
              "recruited_driven_resident_dysfunction"
            ]
          },
          {
            "name": "dominant_signature_gate",
            "role": "gate",
            "field": "$.dominant_signature",
            "kind": "value",
            "op": "in",
            "expected": [
              "technical_or_agonal_artifact"
            ]
          },
          {
            "name": "dominant_signature_direction_gate",
            "role": "gate",
            "field": "$.dominant_signature_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_in_covid"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-5-pi",
      "modelName": "claude-sonnet-5",
      "harness": "pi",
      "task": "q6_myeloid_subset_restricted_de",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 2089.297674,
      "cost": 1.000319,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "myeloid_subset_with_strongest_change_gate",
            "role": "gate",
            "field": "$.myeloid_subset_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "subset_gamma"
            ]
          },
          {
            "name": "gene_with_strongest_change_gate",
            "role": "gate",
            "field": "$.gene_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "AXL",
              "PROS1"
            ]
          },
          {
            "name": "direction_of_change_gate",
            "role": "gate",
            "field": "$.direction_of_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "down_in_covid"
            ]
          },
          {
            "name": "biological_function_interpretation_gate",
            "role": "gate",
            "field": "$.biological_function_interpretation",
            "kind": "value",
            "op": "in",
            "expected": [
              "efferocytosis",
              "phagocytosis"
            ]
          },
          {
            "name": "myeloid_compartment_abundance_gate",
            "role": "gate",
            "field": "$.myeloid_compartment_abundance",
            "kind": "value",
            "op": "in",
            "expected": [
              "expands_in_covid"
            ]
          },
          {
            "name": "myeloid_expansion_localization_gate",
            "role": "gate",
            "field": "$.myeloid_expansion_localization",
            "kind": "value",
            "op": "in",
            "expected": [
              "recruited_driven_resident_dysfunction"
            ]
          },
          {
            "name": "dominant_signature_gate",
            "role": "gate",
            "field": "$.dominant_signature",
            "kind": "value",
            "op": "in",
            "expected": [
              "technical_or_agonal_artifact"
            ]
          },
          {
            "name": "dominant_signature_direction_gate",
            "role": "gate",
            "field": "$.dominant_signature_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_in_covid"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-5-pi",
      "modelName": "claude-sonnet-5",
      "harness": "pi",
      "task": "snrnaseq_per_donor_composition_shifts_lethal_covid_lung",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 2825.290209,
      "cost": 1.0520039,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dict_match",
            "role": "gate",
            "field": "cell_type_direction_map",
            "kind": "dict",
            "expected": {
              "Myeloid": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "Epithelia": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Fibroblasts": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "T_NK": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "B_plasma": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Endothelial": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Mast": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "APC": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Neuronal": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              }
            }
          },
          {
            "name": "over_merge_immune_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Immune",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_leukocyte_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Leukocyte",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_lymphoid_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Lymphoid",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_stromal_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Stromal",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-5-pi",
      "modelName": "claude-sonnet-5",
      "harness": "pi",
      "task": "snrnaseq_per_donor_composition_shifts_lethal_covid_lung",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 2911.023259,
      "cost": 0.9697897,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dict_match",
            "role": "gate",
            "field": "cell_type_direction_map",
            "kind": "dict",
            "expected": {
              "Myeloid": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "Epithelia": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Fibroblasts": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "T_NK": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "B_plasma": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Endothelial": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Mast": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "APC": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Neuronal": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              }
            }
          },
          {
            "name": "over_merge_immune_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Immune",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_leukocyte_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Leukocyte",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_lymphoid_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Lymphoid",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_stromal_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Stromal",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-5-pi",
      "modelName": "claude-sonnet-5",
      "harness": "pi",
      "task": "snrnaseq_per_donor_composition_shifts_lethal_covid_lung",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1489.885309,
      "cost": 0.9366699,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dict_match",
            "role": "gate",
            "field": "cell_type_direction_map",
            "kind": "dict",
            "expected": {
              "Myeloid": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "Epithelia": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Fibroblasts": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "T_NK": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "B_plasma": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Endothelial": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Mast": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "APC": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Neuronal": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              }
            }
          },
          {
            "name": "over_merge_immune_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Immune",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_leukocyte_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Leukocyte",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_lymphoid_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Lymphoid",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_stromal_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Stromal",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-5-pi",
      "modelName": "claude-sonnet-5",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_enriched_apc_states_v2",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 1,
      "passed": true,
      "score": null,
      "durationS": 3184.194995,
      "cost": 1.9037764,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "top_3_cluster_enriched_apc_states",
            "kind": "list",
            "matchKey": "apc_state",
            "k": 3,
            "expected": [
              {
                "key": "plasmacytoid_DC",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "T_cell_stimulatory_role_supported",
                      "role_direction_unresolved"
                    ],
                    "canonical": "T_cell_stimulatory_role_supported"
                  }
                }
              },
              {
                "key": "C1qhigh_lipid_associated_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "C1qhigh_inflammatory_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "plasma_cell",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "cohort_significant_per_patient_inconsistent"
                    ],
                    "canonical": "cohort_significant_per_patient_inconsistent"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CD80_CD86": {
                "op": "in",
                "value": [
                  "mreg_DC",
                  "shared_across_APC_states"
                ]
              },
              "PD_L1_in_APC_clusters": {
                "op": "in",
                "value": [
                  "C1qhigh_macrophage_either_subtype",
                  "shared_across_APC_states",
                  "mreg_DC"
                ]
              },
              "type_I_IFN": {
                "op": "in",
                "value": [
                  "plasmacytoid_DC",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "cf_mreg_dc_claimed_enriched",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "mreg_DC",
              "mature_DC_enriched_in_immunoregulatory_molecules"
            ]
          },
          {
            "name": "cf_framework_offcompartment_or_generic_substitution",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "cDC1",
              "Maier_mregDC",
              "Cheng_C1QC_TAM",
              "Cheng_SPP1_TAM",
              "Cheng_FOLR2_TAM",
              "Villani_pDC",
              "Zhang_inflammatory_monocyte",
              "LAMP3_pos_DC",
              "myeloid_derived_suppressor_cell",
              "MDSC",
              "tumor_associated_neutrophil",
              "mast_cell",
              "APC",
              "antigen_presenting_cell",
              "macrophage",
              "dendritic_cell",
              "tumor_associated_macrophage",
              "TAM"
            ]
          },
          {
            "name": "cf_direction_commit_on_unresolvable_slot",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_not_resolvable_hedge",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].functional_attribution_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_type_I_IFN_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.type_I_IFN",
            "kind": "value",
            "op": "in",
            "expected": [
              "mreg_DC",
              "C1qhigh_macrophage_either_subtype"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-5-pi",
      "modelName": "claude-sonnet-5",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_enriched_apc_states_v2",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 3201.668049,
      "cost": 1.6323134,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "top_3_cluster_enriched_apc_states",
            "kind": "list",
            "matchKey": "apc_state",
            "k": 3,
            "expected": [
              {
                "key": "plasmacytoid_DC",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "T_cell_stimulatory_role_supported",
                      "role_direction_unresolved"
                    ],
                    "canonical": "T_cell_stimulatory_role_supported"
                  }
                }
              },
              {
                "key": "C1qhigh_lipid_associated_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "C1qhigh_inflammatory_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "plasma_cell",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "cohort_significant_per_patient_inconsistent"
                    ],
                    "canonical": "cohort_significant_per_patient_inconsistent"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CD80_CD86": {
                "op": "in",
                "value": [
                  "mreg_DC",
                  "shared_across_APC_states"
                ]
              },
              "PD_L1_in_APC_clusters": {
                "op": "in",
                "value": [
                  "C1qhigh_macrophage_either_subtype",
                  "shared_across_APC_states",
                  "mreg_DC"
                ]
              },
              "type_I_IFN": {
                "op": "in",
                "value": [
                  "plasmacytoid_DC",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "cf_mreg_dc_claimed_enriched",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "mreg_DC",
              "mature_DC_enriched_in_immunoregulatory_molecules"
            ]
          },
          {
            "name": "cf_framework_offcompartment_or_generic_substitution",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "cDC1",
              "Maier_mregDC",
              "Cheng_C1QC_TAM",
              "Cheng_SPP1_TAM",
              "Cheng_FOLR2_TAM",
              "Villani_pDC",
              "Zhang_inflammatory_monocyte",
              "LAMP3_pos_DC",
              "myeloid_derived_suppressor_cell",
              "MDSC",
              "tumor_associated_neutrophil",
              "mast_cell",
              "APC",
              "antigen_presenting_cell",
              "macrophage",
              "dendritic_cell",
              "tumor_associated_macrophage",
              "TAM"
            ]
          },
          {
            "name": "cf_direction_commit_on_unresolvable_slot",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_not_resolvable_hedge",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].functional_attribution_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_type_I_IFN_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.type_I_IFN",
            "kind": "value",
            "op": "in",
            "expected": [
              "mreg_DC",
              "C1qhigh_macrophage_either_subtype"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-5-pi",
      "modelName": "claude-sonnet-5",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_enriched_apc_states_v2",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 3499.968941,
      "cost": 2.114586,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "top_3_cluster_enriched_apc_states",
            "kind": "list",
            "matchKey": "apc_state",
            "k": 3,
            "expected": [
              {
                "key": "plasmacytoid_DC",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "T_cell_stimulatory_role_supported",
                      "role_direction_unresolved"
                    ],
                    "canonical": "T_cell_stimulatory_role_supported"
                  }
                }
              },
              {
                "key": "C1qhigh_lipid_associated_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "C1qhigh_inflammatory_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "plasma_cell",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "cohort_significant_per_patient_inconsistent"
                    ],
                    "canonical": "cohort_significant_per_patient_inconsistent"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CD80_CD86": {
                "op": "in",
                "value": [
                  "mreg_DC",
                  "shared_across_APC_states"
                ]
              },
              "PD_L1_in_APC_clusters": {
                "op": "in",
                "value": [
                  "C1qhigh_macrophage_either_subtype",
                  "shared_across_APC_states",
                  "mreg_DC"
                ]
              },
              "type_I_IFN": {
                "op": "in",
                "value": [
                  "plasmacytoid_DC",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "cf_mreg_dc_claimed_enriched",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "mreg_DC",
              "mature_DC_enriched_in_immunoregulatory_molecules"
            ]
          },
          {
            "name": "cf_framework_offcompartment_or_generic_substitution",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "cDC1",
              "Maier_mregDC",
              "Cheng_C1QC_TAM",
              "Cheng_SPP1_TAM",
              "Cheng_FOLR2_TAM",
              "Villani_pDC",
              "Zhang_inflammatory_monocyte",
              "LAMP3_pos_DC",
              "myeloid_derived_suppressor_cell",
              "MDSC",
              "tumor_associated_neutrophil",
              "mast_cell",
              "APC",
              "antigen_presenting_cell",
              "macrophage",
              "dendritic_cell",
              "tumor_associated_macrophage",
              "TAM"
            ]
          },
          {
            "name": "cf_direction_commit_on_unresolvable_slot",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_not_resolvable_hedge",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].functional_attribution_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_type_I_IFN_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.type_I_IFN",
            "kind": "value",
            "op": "in",
            "expected": [
              "mreg_DC",
              "C1qhigh_macrophage_either_subtype"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-5-pi",
      "modelName": "claude-sonnet-5",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_tumor_state_causation_v5",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 2976.195271,
      "cost": 1.9722978000000004,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "enriched_tumor_states",
            "kind": "list",
            "matchKey": "tumor_state",
            "k": 3,
            "expected": [
              {
                "key": "immune_response",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "direction_unresolved_cross_sectional"
                    ],
                    "canonical": "direction_unresolved_cross_sectional"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "IFN_response_axis",
                      "broad_antigen_presentation_MHC"
                    ],
                    "canonical": "IFN_response_axis"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "enrichment_diminished_by_decontamination",
                      "threshold_sensitive"
                    ],
                    "canonical": "enrichment_diminished_by_decontamination"
                  }
                }
              },
              {
                "key": "stress_hypoxia",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "data_supports_cell_intrinsic_cause"
                    ],
                    "canonical": "data_supports_cell_intrinsic_cause"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "HIF_hypoxia_program_specific"
                    ],
                    "canonical": "HIF_hypoxia_program_specific"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "robust_across_methods",
                      "robust_after_decontamination"
                    ],
                    "canonical": "robust_across_methods"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CXCL9_CXCL10": {
                "op": "in",
                "value": [
                  "immune_response_tumor_state"
                ]
              },
              "CCL5": {
                "op": "in",
                "value": [
                  "shared_across_tumor_states",
                  "partner_lymphocyte_derived",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "gate_conjugate_gate_confounder",
            "role": "gate",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "partner_lymphocyte_capture"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "antigen_presentation_decomposition",
            "kind": "dict",
            "expected": {
              "MHC_class_I": {
                "op": "in",
                "value": [
                  "co_regulated_with_dominant_program",
                  "tumor_intrinsic_enriched_independent"
                ]
              },
              "MHC_class_II": {
                "op": "in",
                "value": [
                  "partner_cell_attributable",
                  "flat_not_enriched",
                  "method_sensitive_no_robust_call",
                  "co_regulated_with_dominant_program"
                ]
              }
            }
          },
          {
            "name": "cf_non_robust_state_claimed_enriched",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "transitory_melanocytic",
              "intermediate_melanocytic",
              "melanocytic",
              "MITF_high",
              "neural_crest_like",
              "mitotic",
              "stress_p53",
              "undifferentiated",
              "dedifferentiated",
              "mesenchymal_tumor",
              "AXL_high_tumor",
              "invasive_tumor",
              "senescence_like",
              "EMT_transition",
              "cancer_stem_cell_like",
              "oxidative_stress_tumor"
            ]
          },
          {
            "name": "cf_framework_generic_or_offtissue_substitution",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "Tsoi_undifferentiated",
              "Tsoi_neural_crest_like",
              "Tsoi_transitory",
              "Tsoi_melanocytic",
              "Rambow_NCSC",
              "Wouters_mesenchymal",
              "Pozniak_TCF4_dependent",
              "tumor_cell",
              "melanoma_cell",
              "malignant_cell",
              "tumor_subpopulation",
              "keratinocyte",
              "hepatocyte",
              "lung_adenocarcinoma_cell",
              "colon_adenocarcinoma_cell",
              "Tirosh_cell_cycle_program",
              "Jerby_Arnon_program",
              "glioblastoma_cell",
              "breast_carcinoma_cell"
            ]
          },
          {
            "name": "cf_immune_response_direction_committed",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_immune_robustness_recall",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_ambiguous_hedge",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].causation_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_stress_generic_stress_response_evidence",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_conjugate_confounder_contamination_denied",
            "role": "hard_fail",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected"
            ]
          },
          {
            "name": "cf_ccl5_recall_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CCL5",
            "kind": "avoid",
            "expected": [
              "immune_response_tumor_state"
            ]
          },
          {
            "name": "cf_cxcl9_cxcl10_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CXCL9_CXCL10",
            "kind": "avoid",
            "expected": [
              "stress_hypoxia_tumor_state"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-5-pi",
      "modelName": "claude-sonnet-5",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_tumor_state_causation_v5",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 2373.924046,
      "cost": 1.8349852,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "enriched_tumor_states",
            "kind": "list",
            "matchKey": "tumor_state",
            "k": 3,
            "expected": [
              {
                "key": "immune_response",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "direction_unresolved_cross_sectional"
                    ],
                    "canonical": "direction_unresolved_cross_sectional"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "IFN_response_axis",
                      "broad_antigen_presentation_MHC"
                    ],
                    "canonical": "IFN_response_axis"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "enrichment_diminished_by_decontamination",
                      "threshold_sensitive"
                    ],
                    "canonical": "enrichment_diminished_by_decontamination"
                  }
                }
              },
              {
                "key": "stress_hypoxia",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "data_supports_cell_intrinsic_cause"
                    ],
                    "canonical": "data_supports_cell_intrinsic_cause"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "HIF_hypoxia_program_specific"
                    ],
                    "canonical": "HIF_hypoxia_program_specific"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "robust_across_methods",
                      "robust_after_decontamination"
                    ],
                    "canonical": "robust_across_methods"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CXCL9_CXCL10": {
                "op": "in",
                "value": [
                  "immune_response_tumor_state"
                ]
              },
              "CCL5": {
                "op": "in",
                "value": [
                  "shared_across_tumor_states",
                  "partner_lymphocyte_derived",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "gate_conjugate_gate_confounder",
            "role": "gate",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "partner_lymphocyte_capture"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "antigen_presentation_decomposition",
            "kind": "dict",
            "expected": {
              "MHC_class_I": {
                "op": "in",
                "value": [
                  "co_regulated_with_dominant_program",
                  "tumor_intrinsic_enriched_independent"
                ]
              },
              "MHC_class_II": {
                "op": "in",
                "value": [
                  "partner_cell_attributable",
                  "flat_not_enriched",
                  "method_sensitive_no_robust_call",
                  "co_regulated_with_dominant_program"
                ]
              }
            }
          },
          {
            "name": "cf_non_robust_state_claimed_enriched",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "transitory_melanocytic",
              "intermediate_melanocytic",
              "melanocytic",
              "MITF_high",
              "neural_crest_like",
              "mitotic",
              "stress_p53",
              "undifferentiated",
              "dedifferentiated",
              "mesenchymal_tumor",
              "AXL_high_tumor",
              "invasive_tumor",
              "senescence_like",
              "EMT_transition",
              "cancer_stem_cell_like",
              "oxidative_stress_tumor"
            ]
          },
          {
            "name": "cf_framework_generic_or_offtissue_substitution",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "Tsoi_undifferentiated",
              "Tsoi_neural_crest_like",
              "Tsoi_transitory",
              "Tsoi_melanocytic",
              "Rambow_NCSC",
              "Wouters_mesenchymal",
              "Pozniak_TCF4_dependent",
              "tumor_cell",
              "melanoma_cell",
              "malignant_cell",
              "tumor_subpopulation",
              "keratinocyte",
              "hepatocyte",
              "lung_adenocarcinoma_cell",
              "colon_adenocarcinoma_cell",
              "Tirosh_cell_cycle_program",
              "Jerby_Arnon_program",
              "glioblastoma_cell",
              "breast_carcinoma_cell"
            ]
          },
          {
            "name": "cf_immune_response_direction_committed",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_immune_robustness_recall",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_ambiguous_hedge",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].causation_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_stress_generic_stress_response_evidence",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_conjugate_confounder_contamination_denied",
            "role": "hard_fail",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected"
            ]
          },
          {
            "name": "cf_ccl5_recall_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CCL5",
            "kind": "avoid",
            "expected": [
              "immune_response_tumor_state"
            ]
          },
          {
            "name": "cf_cxcl9_cxcl10_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CXCL9_CXCL10",
            "kind": "avoid",
            "expected": [
              "stress_hypoxia_tumor_state"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-5-pi",
      "modelName": "claude-sonnet-5",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_tumor_state_causation_v5",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 2960.516369,
      "cost": 1.9836748000000002,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "enriched_tumor_states",
            "kind": "list",
            "matchKey": "tumor_state",
            "k": 3,
            "expected": [
              {
                "key": "immune_response",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "direction_unresolved_cross_sectional"
                    ],
                    "canonical": "direction_unresolved_cross_sectional"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "IFN_response_axis",
                      "broad_antigen_presentation_MHC"
                    ],
                    "canonical": "IFN_response_axis"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "enrichment_diminished_by_decontamination",
                      "threshold_sensitive"
                    ],
                    "canonical": "enrichment_diminished_by_decontamination"
                  }
                }
              },
              {
                "key": "stress_hypoxia",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "data_supports_cell_intrinsic_cause"
                    ],
                    "canonical": "data_supports_cell_intrinsic_cause"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "HIF_hypoxia_program_specific"
                    ],
                    "canonical": "HIF_hypoxia_program_specific"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "robust_across_methods",
                      "robust_after_decontamination"
                    ],
                    "canonical": "robust_across_methods"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CXCL9_CXCL10": {
                "op": "in",
                "value": [
                  "immune_response_tumor_state"
                ]
              },
              "CCL5": {
                "op": "in",
                "value": [
                  "shared_across_tumor_states",
                  "partner_lymphocyte_derived",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "gate_conjugate_gate_confounder",
            "role": "gate",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "partner_lymphocyte_capture"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "antigen_presentation_decomposition",
            "kind": "dict",
            "expected": {
              "MHC_class_I": {
                "op": "in",
                "value": [
                  "co_regulated_with_dominant_program",
                  "tumor_intrinsic_enriched_independent"
                ]
              },
              "MHC_class_II": {
                "op": "in",
                "value": [
                  "partner_cell_attributable",
                  "flat_not_enriched",
                  "method_sensitive_no_robust_call",
                  "co_regulated_with_dominant_program"
                ]
              }
            }
          },
          {
            "name": "cf_non_robust_state_claimed_enriched",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "transitory_melanocytic",
              "intermediate_melanocytic",
              "melanocytic",
              "MITF_high",
              "neural_crest_like",
              "mitotic",
              "stress_p53",
              "undifferentiated",
              "dedifferentiated",
              "mesenchymal_tumor",
              "AXL_high_tumor",
              "invasive_tumor",
              "senescence_like",
              "EMT_transition",
              "cancer_stem_cell_like",
              "oxidative_stress_tumor"
            ]
          },
          {
            "name": "cf_framework_generic_or_offtissue_substitution",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "Tsoi_undifferentiated",
              "Tsoi_neural_crest_like",
              "Tsoi_transitory",
              "Tsoi_melanocytic",
              "Rambow_NCSC",
              "Wouters_mesenchymal",
              "Pozniak_TCF4_dependent",
              "tumor_cell",
              "melanoma_cell",
              "malignant_cell",
              "tumor_subpopulation",
              "keratinocyte",
              "hepatocyte",
              "lung_adenocarcinoma_cell",
              "colon_adenocarcinoma_cell",
              "Tirosh_cell_cycle_program",
              "Jerby_Arnon_program",
              "glioblastoma_cell",
              "breast_carcinoma_cell"
            ]
          },
          {
            "name": "cf_immune_response_direction_committed",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_immune_robustness_recall",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_ambiguous_hedge",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].causation_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_stress_generic_stress_response_evidence",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_conjugate_confounder_contamination_denied",
            "role": "hard_fail",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected"
            ]
          },
          {
            "name": "cf_ccl5_recall_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CCL5",
            "kind": "avoid",
            "expected": [
              "immune_response_tumor_state"
            ]
          },
          {
            "name": "cf_cxcl9_cxcl10_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CXCL9_CXCL10",
            "kind": "avoid",
            "expected": [
              "stress_hypoxia_tumor_state"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-5-pi",
      "modelName": "claude-sonnet-5",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_vs_cd39_tumor_reactivity_dominance",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 2339.899925,
      "cost": 1.5757964,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "population_findings",
            "kind": "list",
            "matchKey": "capture_gate",
            "k": 4,
            "expected": [
              {
                "key": "tumor_cell_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "apc_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "combined_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "unselected_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "naive",
                      "central_memory",
                      "effector_memory"
                    ],
                    "canonical": "naive"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "low"
                    ],
                    "canonical": "low"
                  },
                  "purity_caveat": {
                    "op": "equals",
                    "values": [
                      "pure_T_cell"
                    ],
                    "canonical": "pure_T_cell"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "no_reactivity_evidence"
                    ],
                    "canonical": "no_reactivity_evidence"
                  }
                }
              },
              {
                "key": "marker_sorted_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "equals",
                    "values": [
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "terminal_exhausted_LAG3_high"
                  },
                  "clonal_expansion_level": {
                    "op": "in",
                    "values": [
                      "high",
                      "intermediate"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "marker_on_non_T_lineages",
                      "single_cell_gating_loses_target"
                    ],
                    "canonical": "marker_on_non_T_lineages"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "marker_or_exhaustion_defined_only"
                    ],
                    "canonical": "marker_or_exhaustion_defined_only"
                  }
                }
              }
            ]
          },
          {
            "name": "best_capture_approach_gate",
            "role": "gate",
            "field": "best_capture_approach",
            "kind": "value",
            "expected": "heterotypic_cell_conjugate"
          },
          {
            "name": "best_selection_marker_gate",
            "role": "gate",
            "field": "best_selection_marker",
            "kind": "value",
            "expected": "none"
          },
          {
            "name": "reactivity_criterion_gate",
            "role": "gate",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "clonal_expansion_and_physical_engagement"
          },
          {
            "name": "cross_patient_robustness_gate",
            "role": "gate",
            "field": "cross_patient_robustness",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_across_patients",
              "marginal_in_some_patients"
            ]
          },
          {
            "name": "marker_selection_consequence_gate",
            "role": "gate",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "discards_clonally_expanded_reactive_cells"
          },
          {
            "name": "superiority_claim_basis_gate",
            "role": "gate",
            "field": "superiority_claim_basis",
            "kind": "value",
            "expected": "requires_functional_reactivity_assay"
          },
          {
            "name": "conjugate_enriched_state_gate",
            "role": "gate",
            "field": "conjugate_enriched_state",
            "kind": "value",
            "expected": "progenitor_exhausted_TCF7_pos_stem_like"
          },
          {
            "name": "cf_circular_exhaustion_criterion",
            "role": "hard_fail",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "exhaustion_signature_intensity"
          },
          {
            "name": "cf_marker_sort_prior",
            "role": "hard_fail",
            "field": "best_selection_marker",
            "kind": "value",
            "op": "in",
            "expected": [
              "CD39",
              "PD1",
              "PD1_high",
              "TIM3",
              "PD1_CD39",
              "CD39_CD103"
            ]
          },
          {
            "name": "cf_single_cell_capture_prior",
            "role": "hard_fail",
            "field": "best_capture_approach",
            "kind": "value",
            "op": "in",
            "expected": [
              "dissociated_single_cell",
              "dissociated_bulk_population"
            ]
          },
          {
            "name": "cf_off_data_capture_gate",
            "role": "hard_fail",
            "field": "$.population_findings[*].capture_gate",
            "kind": "avoid",
            "expected": [
              "dissociated_bulk",
              "matched_peripheral_blood",
              "matched_lymph_node",
              "post_hoc_expanded_clonotype_group"
            ]
          },
          {
            "name": "cf_conjugate_mischaracterized",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_marker_singlet_called_engaged",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "contains",
            "matchKey": "capture_gate",
            "entity": "marker_sorted_singlet",
            "valueField": "reactivity_evidence",
            "expected": [
              "engagement_and_clonal_expansion"
            ]
          },
          {
            "name": "cf_no_engaged_reactive_population",
            "role": "hard_fail",
            "field": "$.population_findings[*].reactivity_evidence",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_marker_captures_all_reactive",
            "role": "hard_fail",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "captures_full_reactive_repertoire"
          },
          {
            "name": "cf_superiority_overclaimed_from_phenotype",
            "role": "hard_fail",
            "field": "superiority_claim_basis",
            "kind": "value",
            "op": "in",
            "expected": [
              "established_by_scrna_phenotype",
              "established_by_reactivity_proxy_scores"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-5-pi",
      "modelName": "claude-sonnet-5",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_vs_cd39_tumor_reactivity_dominance",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 1514.006812,
      "cost": 1.0105802,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "population_findings",
            "kind": "list",
            "matchKey": "capture_gate",
            "k": 4,
            "expected": [
              {
                "key": "tumor_cell_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "apc_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "combined_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "unselected_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "naive",
                      "central_memory",
                      "effector_memory"
                    ],
                    "canonical": "naive"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "low"
                    ],
                    "canonical": "low"
                  },
                  "purity_caveat": {
                    "op": "equals",
                    "values": [
                      "pure_T_cell"
                    ],
                    "canonical": "pure_T_cell"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "no_reactivity_evidence"
                    ],
                    "canonical": "no_reactivity_evidence"
                  }
                }
              },
              {
                "key": "marker_sorted_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "equals",
                    "values": [
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "terminal_exhausted_LAG3_high"
                  },
                  "clonal_expansion_level": {
                    "op": "in",
                    "values": [
                      "high",
                      "intermediate"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "marker_on_non_T_lineages",
                      "single_cell_gating_loses_target"
                    ],
                    "canonical": "marker_on_non_T_lineages"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "marker_or_exhaustion_defined_only"
                    ],
                    "canonical": "marker_or_exhaustion_defined_only"
                  }
                }
              }
            ]
          },
          {
            "name": "best_capture_approach_gate",
            "role": "gate",
            "field": "best_capture_approach",
            "kind": "value",
            "expected": "heterotypic_cell_conjugate"
          },
          {
            "name": "best_selection_marker_gate",
            "role": "gate",
            "field": "best_selection_marker",
            "kind": "value",
            "expected": "none"
          },
          {
            "name": "reactivity_criterion_gate",
            "role": "gate",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "clonal_expansion_and_physical_engagement"
          },
          {
            "name": "cross_patient_robustness_gate",
            "role": "gate",
            "field": "cross_patient_robustness",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_across_patients",
              "marginal_in_some_patients"
            ]
          },
          {
            "name": "marker_selection_consequence_gate",
            "role": "gate",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "discards_clonally_expanded_reactive_cells"
          },
          {
            "name": "superiority_claim_basis_gate",
            "role": "gate",
            "field": "superiority_claim_basis",
            "kind": "value",
            "expected": "requires_functional_reactivity_assay"
          },
          {
            "name": "conjugate_enriched_state_gate",
            "role": "gate",
            "field": "conjugate_enriched_state",
            "kind": "value",
            "expected": "progenitor_exhausted_TCF7_pos_stem_like"
          },
          {
            "name": "cf_circular_exhaustion_criterion",
            "role": "hard_fail",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "exhaustion_signature_intensity"
          },
          {
            "name": "cf_marker_sort_prior",
            "role": "hard_fail",
            "field": "best_selection_marker",
            "kind": "value",
            "op": "in",
            "expected": [
              "CD39",
              "PD1",
              "PD1_high",
              "TIM3",
              "PD1_CD39",
              "CD39_CD103"
            ]
          },
          {
            "name": "cf_single_cell_capture_prior",
            "role": "hard_fail",
            "field": "best_capture_approach",
            "kind": "value",
            "op": "in",
            "expected": [
              "dissociated_single_cell",
              "dissociated_bulk_population"
            ]
          },
          {
            "name": "cf_off_data_capture_gate",
            "role": "hard_fail",
            "field": "$.population_findings[*].capture_gate",
            "kind": "avoid",
            "expected": [
              "dissociated_bulk",
              "matched_peripheral_blood",
              "matched_lymph_node",
              "post_hoc_expanded_clonotype_group"
            ]
          },
          {
            "name": "cf_conjugate_mischaracterized",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_marker_singlet_called_engaged",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "contains",
            "matchKey": "capture_gate",
            "entity": "marker_sorted_singlet",
            "valueField": "reactivity_evidence",
            "expected": [
              "engagement_and_clonal_expansion"
            ]
          },
          {
            "name": "cf_no_engaged_reactive_population",
            "role": "hard_fail",
            "field": "$.population_findings[*].reactivity_evidence",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_marker_captures_all_reactive",
            "role": "hard_fail",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "captures_full_reactive_repertoire"
          },
          {
            "name": "cf_superiority_overclaimed_from_phenotype",
            "role": "hard_fail",
            "field": "superiority_claim_basis",
            "kind": "value",
            "op": "in",
            "expected": [
              "established_by_scrna_phenotype",
              "established_by_reactivity_proxy_scores"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-5-pi",
      "modelName": "claude-sonnet-5",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_vs_cd39_tumor_reactivity_dominance",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1518.131281,
      "cost": 1.4462866000000003,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "population_findings",
            "kind": "list",
            "matchKey": "capture_gate",
            "k": 4,
            "expected": [
              {
                "key": "tumor_cell_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "apc_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "combined_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "unselected_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "naive",
                      "central_memory",
                      "effector_memory"
                    ],
                    "canonical": "naive"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "low"
                    ],
                    "canonical": "low"
                  },
                  "purity_caveat": {
                    "op": "equals",
                    "values": [
                      "pure_T_cell"
                    ],
                    "canonical": "pure_T_cell"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "no_reactivity_evidence"
                    ],
                    "canonical": "no_reactivity_evidence"
                  }
                }
              },
              {
                "key": "marker_sorted_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "equals",
                    "values": [
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "terminal_exhausted_LAG3_high"
                  },
                  "clonal_expansion_level": {
                    "op": "in",
                    "values": [
                      "high",
                      "intermediate"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "marker_on_non_T_lineages",
                      "single_cell_gating_loses_target"
                    ],
                    "canonical": "marker_on_non_T_lineages"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "marker_or_exhaustion_defined_only"
                    ],
                    "canonical": "marker_or_exhaustion_defined_only"
                  }
                }
              }
            ]
          },
          {
            "name": "best_capture_approach_gate",
            "role": "gate",
            "field": "best_capture_approach",
            "kind": "value",
            "expected": "heterotypic_cell_conjugate"
          },
          {
            "name": "best_selection_marker_gate",
            "role": "gate",
            "field": "best_selection_marker",
            "kind": "value",
            "expected": "none"
          },
          {
            "name": "reactivity_criterion_gate",
            "role": "gate",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "clonal_expansion_and_physical_engagement"
          },
          {
            "name": "cross_patient_robustness_gate",
            "role": "gate",
            "field": "cross_patient_robustness",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_across_patients",
              "marginal_in_some_patients"
            ]
          },
          {
            "name": "marker_selection_consequence_gate",
            "role": "gate",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "discards_clonally_expanded_reactive_cells"
          },
          {
            "name": "superiority_claim_basis_gate",
            "role": "gate",
            "field": "superiority_claim_basis",
            "kind": "value",
            "expected": "requires_functional_reactivity_assay"
          },
          {
            "name": "conjugate_enriched_state_gate",
            "role": "gate",
            "field": "conjugate_enriched_state",
            "kind": "value",
            "expected": "progenitor_exhausted_TCF7_pos_stem_like"
          },
          {
            "name": "cf_circular_exhaustion_criterion",
            "role": "hard_fail",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "exhaustion_signature_intensity"
          },
          {
            "name": "cf_marker_sort_prior",
            "role": "hard_fail",
            "field": "best_selection_marker",
            "kind": "value",
            "op": "in",
            "expected": [
              "CD39",
              "PD1",
              "PD1_high",
              "TIM3",
              "PD1_CD39",
              "CD39_CD103"
            ]
          },
          {
            "name": "cf_single_cell_capture_prior",
            "role": "hard_fail",
            "field": "best_capture_approach",
            "kind": "value",
            "op": "in",
            "expected": [
              "dissociated_single_cell",
              "dissociated_bulk_population"
            ]
          },
          {
            "name": "cf_off_data_capture_gate",
            "role": "hard_fail",
            "field": "$.population_findings[*].capture_gate",
            "kind": "avoid",
            "expected": [
              "dissociated_bulk",
              "matched_peripheral_blood",
              "matched_lymph_node",
              "post_hoc_expanded_clonotype_group"
            ]
          },
          {
            "name": "cf_conjugate_mischaracterized",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_marker_singlet_called_engaged",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "contains",
            "matchKey": "capture_gate",
            "entity": "marker_sorted_singlet",
            "valueField": "reactivity_evidence",
            "expected": [
              "engagement_and_clonal_expansion"
            ]
          },
          {
            "name": "cf_no_engaged_reactive_population",
            "role": "hard_fail",
            "field": "$.population_findings[*].reactivity_evidence",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_marker_captures_all_reactive",
            "role": "hard_fail",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "captures_full_reactive_repertoire"
          },
          {
            "name": "cf_superiority_overclaimed_from_phenotype",
            "role": "hard_fail",
            "field": "superiority_claim_basis",
            "kind": "value",
            "op": "in",
            "expected": [
              "established_by_scrna_phenotype",
              "established_by_reactivity_proxy_scores"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-5-pi",
      "modelName": "claude-sonnet-5",
      "harness": "pi",
      "task": "10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 1,
      "passed": true,
      "score": null,
      "durationS": 2103.670747,
      "cost": 1.9865155000000008,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_singlet_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "TOXhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "LAG3high_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "GZMKhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "intermediate_exhausted_gzmk"
                    ],
                    "canonical": "intermediate_exhausted_gzmk"
                  }
                }
              },
              {
                "key": "TCF7_stem_like_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "stem_like_exhausted"
                    ],
                    "canonical": "stem_like_exhausted"
                  }
                }
              },
              {
                "key": "MKI67_pos_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              },
              {
                "key": "MKI67high_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_tumor_cluster_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "Tn",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "Tn_Tmem",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "early_Tem_GZMK",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              },
              {
                "key": "Tem_GZMH",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              }
            ]
          },
          {
            "name": "clonotype_expansion_direction_data_true",
            "role": "gate",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_substantially_more_expanded_than_cluster",
              "singlet_modestly_more_expanded_than_cluster"
            ]
          },
          {
            "name": "biological_inference_data_true",
            "role": "gate",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_antigen_experienced_cluster_bystander"
            ]
          },
          {
            "name": "singlet_top3_contains_naive_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Tn",
              "Tn_Tmem",
              "early_Tem_GZMK",
              "Tem_GZMH"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "TOXhigh_Tex",
              "LAG3high_Tex",
              "GZMKhigh_Tex",
              "TCF7_stem_like_Tex",
              "MKI67_pos_Tex_Tprol",
              "MKI67high_Tex_Tprol"
            ]
          },
          {
            "name": "singlet_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "TOXhigh_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "TCF7_stem_like_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "MKI67_state_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "Tn_or_Tn_Tmem_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_stemlike_precursor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_reported_canonical_expansion_direction",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_substantially_more_expanded_than_singlet",
              "cluster_modestly_more_expanded_than_singlet"
            ]
          },
          {
            "name": "cf_expansion_lazy_defer",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "expansion_pattern_not_detectable"
            ]
          },
          {
            "name": "cf_reported_canonical_expected_biology",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_antigen_experienced_singlet_bystander"
            ]
          },
          {
            "name": "cf_biological_inference_null_artifact_or_lineage",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "differences_reflect_dissociation_or_sorting_artifact",
              "cluster_and_singlet_capture_same_biology",
              "cluster_and_singlet_capture_distinct_cd8_lineages"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "biological_relations",
            "kind": "dict",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_greater_than",
                  "greater_than"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "less_than",
                  "much_less_than"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than",
                  "approximately_equal"
                ]
              },
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal"
                ]
              },
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal",
                  "greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_exhaustion_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_naive_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_proliferation_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_expansion_richness_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_multichain_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_intermediate_gzmk_overgeneralized",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_melanocytic_conjugate_overcall",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "values": [
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_relations_mass_hedge",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-5-pi",
      "modelName": "claude-sonnet-5",
      "harness": "pi",
      "task": "10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 3162.157123,
      "cost": 2.2529227999999994,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_singlet_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "TOXhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "LAG3high_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "GZMKhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "intermediate_exhausted_gzmk"
                    ],
                    "canonical": "intermediate_exhausted_gzmk"
                  }
                }
              },
              {
                "key": "TCF7_stem_like_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "stem_like_exhausted"
                    ],
                    "canonical": "stem_like_exhausted"
                  }
                }
              },
              {
                "key": "MKI67_pos_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              },
              {
                "key": "MKI67high_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_tumor_cluster_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "Tn",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "Tn_Tmem",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "early_Tem_GZMK",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              },
              {
                "key": "Tem_GZMH",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              }
            ]
          },
          {
            "name": "clonotype_expansion_direction_data_true",
            "role": "gate",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_substantially_more_expanded_than_cluster",
              "singlet_modestly_more_expanded_than_cluster"
            ]
          },
          {
            "name": "biological_inference_data_true",
            "role": "gate",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_antigen_experienced_cluster_bystander"
            ]
          },
          {
            "name": "singlet_top3_contains_naive_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Tn",
              "Tn_Tmem",
              "early_Tem_GZMK",
              "Tem_GZMH"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "TOXhigh_Tex",
              "LAG3high_Tex",
              "GZMKhigh_Tex",
              "TCF7_stem_like_Tex",
              "MKI67_pos_Tex_Tprol",
              "MKI67high_Tex_Tprol"
            ]
          },
          {
            "name": "singlet_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "TOXhigh_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "TCF7_stem_like_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "MKI67_state_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "Tn_or_Tn_Tmem_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_stemlike_precursor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_reported_canonical_expansion_direction",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_substantially_more_expanded_than_singlet",
              "cluster_modestly_more_expanded_than_singlet"
            ]
          },
          {
            "name": "cf_expansion_lazy_defer",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "expansion_pattern_not_detectable"
            ]
          },
          {
            "name": "cf_reported_canonical_expected_biology",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_antigen_experienced_singlet_bystander"
            ]
          },
          {
            "name": "cf_biological_inference_null_artifact_or_lineage",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "differences_reflect_dissociation_or_sorting_artifact",
              "cluster_and_singlet_capture_same_biology",
              "cluster_and_singlet_capture_distinct_cd8_lineages"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "biological_relations",
            "kind": "dict",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_greater_than",
                  "greater_than"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "less_than",
                  "much_less_than"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than",
                  "approximately_equal"
                ]
              },
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal"
                ]
              },
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal",
                  "greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_exhaustion_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_naive_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_proliferation_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_expansion_richness_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_multichain_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_intermediate_gzmk_overgeneralized",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_melanocytic_conjugate_overcall",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "values": [
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_relations_mass_hedge",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-5-pi",
      "modelName": "claude-sonnet-5",
      "harness": "pi",
      "task": "10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 3039.359825,
      "cost": 1.7236320000000003,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_singlet_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "TOXhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "LAG3high_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "GZMKhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "intermediate_exhausted_gzmk"
                    ],
                    "canonical": "intermediate_exhausted_gzmk"
                  }
                }
              },
              {
                "key": "TCF7_stem_like_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "stem_like_exhausted"
                    ],
                    "canonical": "stem_like_exhausted"
                  }
                }
              },
              {
                "key": "MKI67_pos_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              },
              {
                "key": "MKI67high_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_tumor_cluster_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "Tn",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "Tn_Tmem",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "early_Tem_GZMK",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              },
              {
                "key": "Tem_GZMH",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              }
            ]
          },
          {
            "name": "clonotype_expansion_direction_data_true",
            "role": "gate",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_substantially_more_expanded_than_cluster",
              "singlet_modestly_more_expanded_than_cluster"
            ]
          },
          {
            "name": "biological_inference_data_true",
            "role": "gate",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_antigen_experienced_cluster_bystander"
            ]
          },
          {
            "name": "singlet_top3_contains_naive_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Tn",
              "Tn_Tmem",
              "early_Tem_GZMK",
              "Tem_GZMH"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "TOXhigh_Tex",
              "LAG3high_Tex",
              "GZMKhigh_Tex",
              "TCF7_stem_like_Tex",
              "MKI67_pos_Tex_Tprol",
              "MKI67high_Tex_Tprol"
            ]
          },
          {
            "name": "singlet_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "TOXhigh_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "TCF7_stem_like_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "MKI67_state_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "Tn_or_Tn_Tmem_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_stemlike_precursor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_reported_canonical_expansion_direction",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_substantially_more_expanded_than_singlet",
              "cluster_modestly_more_expanded_than_singlet"
            ]
          },
          {
            "name": "cf_expansion_lazy_defer",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "expansion_pattern_not_detectable"
            ]
          },
          {
            "name": "cf_reported_canonical_expected_biology",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_antigen_experienced_singlet_bystander"
            ]
          },
          {
            "name": "cf_biological_inference_null_artifact_or_lineage",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "differences_reflect_dissociation_or_sorting_artifact",
              "cluster_and_singlet_capture_same_biology",
              "cluster_and_singlet_capture_distinct_cd8_lineages"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "biological_relations",
            "kind": "dict",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_greater_than",
                  "greater_than"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "less_than",
                  "much_less_than"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than",
                  "approximately_equal"
                ]
              },
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal"
                ]
              },
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal",
                  "greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_exhaustion_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_naive_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_proliferation_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_expansion_richness_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_multichain_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_intermediate_gzmk_overgeneralized",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_melanocytic_conjugate_overcall",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "values": [
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_relations_mass_hedge",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-5-pi",
      "modelName": "claude-sonnet-5",
      "harness": "pi",
      "task": "10x_5p_melanoma_tcr_matched_apc_vs_tumor_exhaustion",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 1,
      "passed": true,
      "score": null,
      "durationS": 1620.579539,
      "cost": 1.0883039,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "state_discriminating_genes",
            "kind": "list",
            "matchKey": "gene",
            "k": 3,
            "expected": [
              {
                "key": "CTLA4",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TOX",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "ENTPD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "HAVCR2",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TIGIT",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "PDCD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "difference_attributable_to",
            "role": "gate",
            "field": "difference_attributable_to",
            "kind": "value",
            "expected": "covaries_with_apc_conjugate_state"
          },
          {
            "name": "causal_status",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "direction_unresolved_selection_confound"
          },
          {
            "name": "functional_interpretation",
            "role": "gate",
            "field": "functional_interpretation",
            "kind": "value",
            "expected": "costimulation_signature_function_unresolved"
          },
          {
            "name": "cf_contamination_gene_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "MLANA",
              "PMEL",
              "DCT",
              "TYR",
              "TYRP1",
              "SOX10",
              "MITF",
              "MAGEA4"
            ]
          },
          {
            "name": "cf_apc_partner_contamination_gene",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "HLA-DRA",
              "CD68",
              "LYZ",
              "CD14",
              "C1QA",
              "ITGAX",
              "CD1C"
            ]
          },
          {
            "name": "cf_ligand_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "CD274",
              "PDCD1LG2",
              "CD80",
              "CD86"
            ]
          },
          {
            "name": "cf_attribution_causal_or_baseline",
            "role": "hard_fail",
            "field": "$.difference_attributable_to",
            "kind": "avoid",
            "expected": [
              "apc_engagement",
              "tumor_cell_engagement",
              "clonotype_intrinsic_baseline"
            ]
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "engagement_induces_exhausted_state",
              "pre_exhausted_cells_select_into_apc_conjugates"
            ]
          },
          {
            "name": "cf_causal_fabrication",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "resolved_by_pseudotemporal_ordering",
              "resolved_by_perturbation_experiment"
            ]
          },
          {
            "name": "cf_functional_overclaim",
            "role": "hard_fail",
            "field": "$.functional_interpretation",
            "kind": "avoid",
            "expected": [
              "costimulation_licensed_reinvigoration_competent",
              "costimulation_licensed_checkpoint_responsive",
              "tumor_driven_terminal_exhaustion"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-5-pi",
      "modelName": "claude-sonnet-5",
      "harness": "pi",
      "task": "10x_5p_melanoma_tcr_matched_apc_vs_tumor_exhaustion",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 1371.243305,
      "cost": 0.698569,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "state_discriminating_genes",
            "kind": "list",
            "matchKey": "gene",
            "k": 3,
            "expected": [
              {
                "key": "CTLA4",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TOX",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "ENTPD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "HAVCR2",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TIGIT",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "PDCD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "difference_attributable_to",
            "role": "gate",
            "field": "difference_attributable_to",
            "kind": "value",
            "expected": "covaries_with_apc_conjugate_state"
          },
          {
            "name": "causal_status",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "direction_unresolved_selection_confound"
          },
          {
            "name": "functional_interpretation",
            "role": "gate",
            "field": "functional_interpretation",
            "kind": "value",
            "expected": "costimulation_signature_function_unresolved"
          },
          {
            "name": "cf_contamination_gene_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "MLANA",
              "PMEL",
              "DCT",
              "TYR",
              "TYRP1",
              "SOX10",
              "MITF",
              "MAGEA4"
            ]
          },
          {
            "name": "cf_apc_partner_contamination_gene",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "HLA-DRA",
              "CD68",
              "LYZ",
              "CD14",
              "C1QA",
              "ITGAX",
              "CD1C"
            ]
          },
          {
            "name": "cf_ligand_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "CD274",
              "PDCD1LG2",
              "CD80",
              "CD86"
            ]
          },
          {
            "name": "cf_attribution_causal_or_baseline",
            "role": "hard_fail",
            "field": "$.difference_attributable_to",
            "kind": "avoid",
            "expected": [
              "apc_engagement",
              "tumor_cell_engagement",
              "clonotype_intrinsic_baseline"
            ]
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "engagement_induces_exhausted_state",
              "pre_exhausted_cells_select_into_apc_conjugates"
            ]
          },
          {
            "name": "cf_causal_fabrication",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "resolved_by_pseudotemporal_ordering",
              "resolved_by_perturbation_experiment"
            ]
          },
          {
            "name": "cf_functional_overclaim",
            "role": "hard_fail",
            "field": "$.functional_interpretation",
            "kind": "avoid",
            "expected": [
              "costimulation_licensed_reinvigoration_competent",
              "costimulation_licensed_checkpoint_responsive",
              "tumor_driven_terminal_exhaustion"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-5-pi",
      "modelName": "claude-sonnet-5",
      "harness": "pi",
      "task": "10x_5p_melanoma_tcr_matched_apc_vs_tumor_exhaustion",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1457.760841,
      "cost": 1.0778886000000003,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "state_discriminating_genes",
            "kind": "list",
            "matchKey": "gene",
            "k": 3,
            "expected": [
              {
                "key": "CTLA4",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TOX",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "ENTPD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "HAVCR2",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TIGIT",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "PDCD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "difference_attributable_to",
            "role": "gate",
            "field": "difference_attributable_to",
            "kind": "value",
            "expected": "covaries_with_apc_conjugate_state"
          },
          {
            "name": "causal_status",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "direction_unresolved_selection_confound"
          },
          {
            "name": "functional_interpretation",
            "role": "gate",
            "field": "functional_interpretation",
            "kind": "value",
            "expected": "costimulation_signature_function_unresolved"
          },
          {
            "name": "cf_contamination_gene_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "MLANA",
              "PMEL",
              "DCT",
              "TYR",
              "TYRP1",
              "SOX10",
              "MITF",
              "MAGEA4"
            ]
          },
          {
            "name": "cf_apc_partner_contamination_gene",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "HLA-DRA",
              "CD68",
              "LYZ",
              "CD14",
              "C1QA",
              "ITGAX",
              "CD1C"
            ]
          },
          {
            "name": "cf_ligand_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "CD274",
              "PDCD1LG2",
              "CD80",
              "CD86"
            ]
          },
          {
            "name": "cf_attribution_causal_or_baseline",
            "role": "hard_fail",
            "field": "$.difference_attributable_to",
            "kind": "avoid",
            "expected": [
              "apc_engagement",
              "tumor_cell_engagement",
              "clonotype_intrinsic_baseline"
            ]
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "engagement_induces_exhausted_state",
              "pre_exhausted_cells_select_into_apc_conjugates"
            ]
          },
          {
            "name": "cf_causal_fabrication",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "resolved_by_pseudotemporal_ordering",
              "resolved_by_perturbation_experiment"
            ]
          },
          {
            "name": "cf_functional_overclaim",
            "role": "hard_fail",
            "field": "$.functional_interpretation",
            "kind": "avoid",
            "expected": [
              "costimulation_licensed_reinvigoration_competent",
              "costimulation_licensed_checkpoint_responsive",
              "tumor_driven_terminal_exhaustion"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-5-pi",
      "modelName": "claude-sonnet-5",
      "harness": "pi",
      "task": "green_2_tme_context_dependence",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 2106.389441,
      "cost": 1.9019097000000003,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "program_map",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "sig_includes_itgae",
            "role": "gate",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Itgae"
            ]
          },
          {
            "name": "declared_a_known_confounder",
            "role": "gate",
            "field": "$.excluded_confounders[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "cf_signature_gut_contaminant",
            "role": "hard_fail",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "generic_signature_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd103_program_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "<"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "per_model_regulatory_state",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "poised_unfired"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "late_activation"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "closing_committed"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_tuples",
            "kind": "list",
            "matchKey": "regulator",
            "k": 1,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "role": {
                    "op": "equals",
                    "values": [
                      "repressor"
                    ]
                  },
                  "gating_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  },
                  "evidence_basis": {
                    "op": "equals",
                    "values": [
                      "target_program_behavior"
                    ]
                  },
                  "expr_vs_activity": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "locus_differential_gate",
            "role": "gate",
            "field": "locus_differential",
            "kind": "value",
            "expected": "uniform_across_models"
          },
          {
            "name": "formation_limited_by_gate",
            "role": "gate",
            "field": "formation_limited_by",
            "kind": "value",
            "expected": "absent_induction"
          },
          {
            "name": "locus_competent_in_naive_gate",
            "role": "gate",
            "field": "locus_competent_in_naive",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "ready_silent_fraction_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 3,
            "expected": [
              {
                "key": "B16_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_KPC_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "activator_chromatin_footprint_gate",
            "role": "gate",
            "field": "activator_chromatin_footprint",
            "kind": "value",
            "expected": "inconsistent_across_formers"
          },
          {
            "name": "exhaustion_comparable_gate",
            "role": "gate",
            "field": "exhaustion_comparable_across_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_nondeterminant_gate",
            "role": "gate",
            "field": "exhaustion_explains_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_separable_program_gate",
            "role": "gate",
            "field": "exhaustion_separable_program",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "exhaustion_counterexample_gate",
            "role": "gate",
            "field": "exhaustion_counterexample_model",
            "kind": "value",
            "expected": "KPC_GP"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "exhaustion_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "cf_inconsistent_present_without_activator",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "program",
            "entity": "present",
            "valueField": "activator_state",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_no_block_but_absent",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "none",
            "valueField": "program",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_late_block_without_poised_locus",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "late_activation",
            "valueField": "locus_state",
            "expected": [
              "closing_committed",
              "active"
            ]
          },
          {
            "name": "cf_inconsistent_early_block_with_brake_released",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "early_release",
            "valueField": "brake_state",
            "expected": [
              "released"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-5-pi",
      "modelName": "claude-sonnet-5",
      "harness": "pi",
      "task": "green_2_tme_context_dependence",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 2600.440209,
      "cost": 2.0111771000000003,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "program_map",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "sig_includes_itgae",
            "role": "gate",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Itgae"
            ]
          },
          {
            "name": "declared_a_known_confounder",
            "role": "gate",
            "field": "$.excluded_confounders[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "cf_signature_gut_contaminant",
            "role": "hard_fail",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "generic_signature_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd103_program_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "<"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "per_model_regulatory_state",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "poised_unfired"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "late_activation"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "closing_committed"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_tuples",
            "kind": "list",
            "matchKey": "regulator",
            "k": 1,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "role": {
                    "op": "equals",
                    "values": [
                      "repressor"
                    ]
                  },
                  "gating_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  },
                  "evidence_basis": {
                    "op": "equals",
                    "values": [
                      "target_program_behavior"
                    ]
                  },
                  "expr_vs_activity": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "locus_differential_gate",
            "role": "gate",
            "field": "locus_differential",
            "kind": "value",
            "expected": "uniform_across_models"
          },
          {
            "name": "formation_limited_by_gate",
            "role": "gate",
            "field": "formation_limited_by",
            "kind": "value",
            "expected": "absent_induction"
          },
          {
            "name": "locus_competent_in_naive_gate",
            "role": "gate",
            "field": "locus_competent_in_naive",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "ready_silent_fraction_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 3,
            "expected": [
              {
                "key": "B16_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_KPC_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "activator_chromatin_footprint_gate",
            "role": "gate",
            "field": "activator_chromatin_footprint",
            "kind": "value",
            "expected": "inconsistent_across_formers"
          },
          {
            "name": "exhaustion_comparable_gate",
            "role": "gate",
            "field": "exhaustion_comparable_across_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_nondeterminant_gate",
            "role": "gate",
            "field": "exhaustion_explains_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_separable_program_gate",
            "role": "gate",
            "field": "exhaustion_separable_program",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "exhaustion_counterexample_gate",
            "role": "gate",
            "field": "exhaustion_counterexample_model",
            "kind": "value",
            "expected": "KPC_GP"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "exhaustion_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "cf_inconsistent_present_without_activator",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "program",
            "entity": "present",
            "valueField": "activator_state",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_no_block_but_absent",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "none",
            "valueField": "program",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_late_block_without_poised_locus",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "late_activation",
            "valueField": "locus_state",
            "expected": [
              "closing_committed",
              "active"
            ]
          },
          {
            "name": "cf_inconsistent_early_block_with_brake_released",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "early_release",
            "valueField": "brake_state",
            "expected": [
              "released"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-5-pi",
      "modelName": "claude-sonnet-5",
      "harness": "pi",
      "task": "green_2_tme_context_dependence",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 2541.636789,
      "cost": 1.6349331999999999,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "program_map",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "sig_includes_itgae",
            "role": "gate",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Itgae"
            ]
          },
          {
            "name": "declared_a_known_confounder",
            "role": "gate",
            "field": "$.excluded_confounders[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "cf_signature_gut_contaminant",
            "role": "hard_fail",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "generic_signature_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd103_program_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "<"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "per_model_regulatory_state",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "poised_unfired"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "late_activation"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "closing_committed"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_tuples",
            "kind": "list",
            "matchKey": "regulator",
            "k": 1,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "role": {
                    "op": "equals",
                    "values": [
                      "repressor"
                    ]
                  },
                  "gating_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  },
                  "evidence_basis": {
                    "op": "equals",
                    "values": [
                      "target_program_behavior"
                    ]
                  },
                  "expr_vs_activity": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "locus_differential_gate",
            "role": "gate",
            "field": "locus_differential",
            "kind": "value",
            "expected": "uniform_across_models"
          },
          {
            "name": "formation_limited_by_gate",
            "role": "gate",
            "field": "formation_limited_by",
            "kind": "value",
            "expected": "absent_induction"
          },
          {
            "name": "locus_competent_in_naive_gate",
            "role": "gate",
            "field": "locus_competent_in_naive",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "ready_silent_fraction_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 3,
            "expected": [
              {
                "key": "B16_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_KPC_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "activator_chromatin_footprint_gate",
            "role": "gate",
            "field": "activator_chromatin_footprint",
            "kind": "value",
            "expected": "inconsistent_across_formers"
          },
          {
            "name": "exhaustion_comparable_gate",
            "role": "gate",
            "field": "exhaustion_comparable_across_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_nondeterminant_gate",
            "role": "gate",
            "field": "exhaustion_explains_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_separable_program_gate",
            "role": "gate",
            "field": "exhaustion_separable_program",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "exhaustion_counterexample_gate",
            "role": "gate",
            "field": "exhaustion_counterexample_model",
            "kind": "value",
            "expected": "KPC_GP"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "exhaustion_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "cf_inconsistent_present_without_activator",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "program",
            "entity": "present",
            "valueField": "activator_state",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_no_block_but_absent",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "none",
            "valueField": "program",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_late_block_without_poised_locus",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "late_activation",
            "valueField": "locus_state",
            "expected": [
              "closing_committed",
              "active"
            ]
          },
          {
            "name": "cf_inconsistent_early_block_with_brake_released",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "early_release",
            "valueField": "brake_state",
            "expected": [
              "released"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-5-pi",
      "modelName": "claude-sonnet-5",
      "harness": "pi",
      "task": "green_klf2_hidden_repressor_of_trm_like_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 2260.863669,
      "cost": 1.4048592,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "regulator_gate",
            "role": "gate",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "KLF2",
              "Klf2"
            ]
          },
          {
            "name": "controlling_node_character_gate",
            "role": "gate",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "high_differential_expression"
          },
          {
            "name": "expression_vs_activity_gate",
            "role": "gate",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "expression_exceeds_activity"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "target_program_partition",
            "kind": "list",
            "matchKey": "gene",
            "k": 7,
            "expected": [
              {
                "key": "S1pr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "S1pr4",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Klrg1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Cx3cr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Itgae",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Hic1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "P2rx7",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              }
            ]
          },
          {
            "name": "causal_status_gate",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "association_only_sufficiency_requires_perturbation"
          },
          {
            "name": "residency_control_logic_gate",
            "role": "gate",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_release_necessary_not_sufficient_multi_regulon"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "trm_like_program_by_model",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_absent"
                    ],
                    "canonical": "niche_absent"
                  }
                }
              }
            ]
          },
          {
            "name": "cf_regulator_activator_or_wrong_node",
            "role": "hard_fail",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "RUNX3",
              "Runx3",
              "PRDM1",
              "Prdm1",
              "BLIMP1",
              "Blimp1",
              "BHLHE40",
              "Bhlhe40",
              "ZNF683",
              "Zfp683",
              "HOBIT",
              "Hobit",
              "AHR",
              "Ahr",
              "BATF",
              "Batf",
              "TOX",
              "Tox",
              "NR4A1",
              "Nr4a1",
              "NR4A2",
              "Nr4a2",
              "NR4A3",
              "Nr4a3",
              "TCF7",
              "Tcf7",
              "TCF1",
              "Tcf1",
              "BACH2",
              "Bach2",
              "TBX21",
              "Tbx21",
              "TBET",
              "Tbet",
              "FLI1",
              "Fli1",
              "NR3C1",
              "Nr3c1",
              "EOMES",
              "Eomes",
              "ZEB2",
              "Zeb2",
              "ID2",
              "Id2",
              "ID3",
              "Id3",
              "IRF4",
              "Irf4"
            ]
          },
          {
            "name": "cf_node_character_hidden_driver_recall",
            "role": "hard_fail",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "hidden_driver_activity_not_expression"
          },
          {
            "name": "cf_effect_relation_hidden_pattern",
            "role": "hard_fail",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "activity_exceeds_expression"
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "causal_status",
            "kind": "value",
            "op": "in",
            "expected": [
              "sufficiency_established",
              "necessity_established",
              "causal_established_in_dataset",
              "sufficiency_via_perturbation_in_dataset"
            ]
          },
          {
            "name": "cf_residency_logic_activator_prior",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "residency_activator_induction_primary"
          },
          {
            "name": "cf_residency_logic_single_regulon",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_controls_entire_residency_program"
          },
          {
            "name": "cf_decoy_called_target",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_distractor_model_reported",
            "role": "hard_fail",
            "field": "$.trm_like_program_by_model[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_chromatin_only_decoy",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-5-pi",
      "modelName": "claude-sonnet-5",
      "harness": "pi",
      "task": "green_klf2_hidden_repressor_of_trm_like_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 1483.255149,
      "cost": 1.3905061,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "regulator_gate",
            "role": "gate",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "KLF2",
              "Klf2"
            ]
          },
          {
            "name": "controlling_node_character_gate",
            "role": "gate",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "high_differential_expression"
          },
          {
            "name": "expression_vs_activity_gate",
            "role": "gate",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "expression_exceeds_activity"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "target_program_partition",
            "kind": "list",
            "matchKey": "gene",
            "k": 7,
            "expected": [
              {
                "key": "S1pr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "S1pr4",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Klrg1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Cx3cr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Itgae",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Hic1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "P2rx7",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              }
            ]
          },
          {
            "name": "causal_status_gate",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "association_only_sufficiency_requires_perturbation"
          },
          {
            "name": "residency_control_logic_gate",
            "role": "gate",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_release_necessary_not_sufficient_multi_regulon"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "trm_like_program_by_model",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_absent"
                    ],
                    "canonical": "niche_absent"
                  }
                }
              }
            ]
          },
          {
            "name": "cf_regulator_activator_or_wrong_node",
            "role": "hard_fail",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "RUNX3",
              "Runx3",
              "PRDM1",
              "Prdm1",
              "BLIMP1",
              "Blimp1",
              "BHLHE40",
              "Bhlhe40",
              "ZNF683",
              "Zfp683",
              "HOBIT",
              "Hobit",
              "AHR",
              "Ahr",
              "BATF",
              "Batf",
              "TOX",
              "Tox",
              "NR4A1",
              "Nr4a1",
              "NR4A2",
              "Nr4a2",
              "NR4A3",
              "Nr4a3",
              "TCF7",
              "Tcf7",
              "TCF1",
              "Tcf1",
              "BACH2",
              "Bach2",
              "TBX21",
              "Tbx21",
              "TBET",
              "Tbet",
              "FLI1",
              "Fli1",
              "NR3C1",
              "Nr3c1",
              "EOMES",
              "Eomes",
              "ZEB2",
              "Zeb2",
              "ID2",
              "Id2",
              "ID3",
              "Id3",
              "IRF4",
              "Irf4"
            ]
          },
          {
            "name": "cf_node_character_hidden_driver_recall",
            "role": "hard_fail",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "hidden_driver_activity_not_expression"
          },
          {
            "name": "cf_effect_relation_hidden_pattern",
            "role": "hard_fail",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "activity_exceeds_expression"
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "causal_status",
            "kind": "value",
            "op": "in",
            "expected": [
              "sufficiency_established",
              "necessity_established",
              "causal_established_in_dataset",
              "sufficiency_via_perturbation_in_dataset"
            ]
          },
          {
            "name": "cf_residency_logic_activator_prior",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "residency_activator_induction_primary"
          },
          {
            "name": "cf_residency_logic_single_regulon",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_controls_entire_residency_program"
          },
          {
            "name": "cf_decoy_called_target",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_distractor_model_reported",
            "role": "hard_fail",
            "field": "$.trm_like_program_by_model[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_chromatin_only_decoy",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-5-pi",
      "modelName": "claude-sonnet-5",
      "harness": "pi",
      "task": "green_klf2_hidden_repressor_of_trm_like_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 6158.260028,
      "cost": 1.8017731,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "regulator_gate",
            "role": "gate",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "KLF2",
              "Klf2"
            ]
          },
          {
            "name": "controlling_node_character_gate",
            "role": "gate",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "high_differential_expression"
          },
          {
            "name": "expression_vs_activity_gate",
            "role": "gate",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "expression_exceeds_activity"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "target_program_partition",
            "kind": "list",
            "matchKey": "gene",
            "k": 7,
            "expected": [
              {
                "key": "S1pr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "S1pr4",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Klrg1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Cx3cr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Itgae",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Hic1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "P2rx7",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              }
            ]
          },
          {
            "name": "causal_status_gate",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "association_only_sufficiency_requires_perturbation"
          },
          {
            "name": "residency_control_logic_gate",
            "role": "gate",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_release_necessary_not_sufficient_multi_regulon"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "trm_like_program_by_model",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_absent"
                    ],
                    "canonical": "niche_absent"
                  }
                }
              }
            ]
          },
          {
            "name": "cf_regulator_activator_or_wrong_node",
            "role": "hard_fail",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "RUNX3",
              "Runx3",
              "PRDM1",
              "Prdm1",
              "BLIMP1",
              "Blimp1",
              "BHLHE40",
              "Bhlhe40",
              "ZNF683",
              "Zfp683",
              "HOBIT",
              "Hobit",
              "AHR",
              "Ahr",
              "BATF",
              "Batf",
              "TOX",
              "Tox",
              "NR4A1",
              "Nr4a1",
              "NR4A2",
              "Nr4a2",
              "NR4A3",
              "Nr4a3",
              "TCF7",
              "Tcf7",
              "TCF1",
              "Tcf1",
              "BACH2",
              "Bach2",
              "TBX21",
              "Tbx21",
              "TBET",
              "Tbet",
              "FLI1",
              "Fli1",
              "NR3C1",
              "Nr3c1",
              "EOMES",
              "Eomes",
              "ZEB2",
              "Zeb2",
              "ID2",
              "Id2",
              "ID3",
              "Id3",
              "IRF4",
              "Irf4"
            ]
          },
          {
            "name": "cf_node_character_hidden_driver_recall",
            "role": "hard_fail",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "hidden_driver_activity_not_expression"
          },
          {
            "name": "cf_effect_relation_hidden_pattern",
            "role": "hard_fail",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "activity_exceeds_expression"
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "causal_status",
            "kind": "value",
            "op": "in",
            "expected": [
              "sufficiency_established",
              "necessity_established",
              "causal_established_in_dataset",
              "sufficiency_via_perturbation_in_dataset"
            ]
          },
          {
            "name": "cf_residency_logic_activator_prior",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "residency_activator_induction_primary"
          },
          {
            "name": "cf_residency_logic_single_regulon",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_controls_entire_residency_program"
          },
          {
            "name": "cf_decoy_called_target",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_distractor_model_reported",
            "role": "hard_fail",
            "field": "$.trm_like_program_by_model[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_chromatin_only_decoy",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-5-pi",
      "modelName": "claude-sonnet-5",
      "harness": "pi",
      "task": "green_shared_vs_distinct_regulators_trm_vs_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 1529.12914,
      "cost": 1.4421044999999995,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_program_map",
            "kind": "list",
            "matchKey": "regulator",
            "k": 3,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "both"
                    ],
                    "canonical": "both"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "represses"
                    ],
                    "canonical": "represses"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Batf",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "trm_like_til_specific"
                    ],
                    "canonical": "trm_like_til_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "chromatin_over_rna"
                    ],
                    "canonical": "chromatin_over_rna"
                  }
                }
              },
              {
                "key": "Ahr",
                "fields": {
                  "program_specificity": {
                    "op": "in",
                    "values": [
                      "trm_specific",
                      "both"
                    ],
                    "canonical": "trm_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "rna_over_chromatin"
                    ],
                    "canonical": "rna_over_chromatin"
                  }
                }
              }
            ]
          },
          {
            "name": "regulator_set_no_extras_gate",
            "role": "gate",
            "field": "$.regulator_program_map[*]",
            "kind": "other",
            "expected": null
          },
          {
            "name": "program_identity_conclusion_gate",
            "role": "gate",
            "field": "program_identity_conclusion",
            "kind": "value",
            "expected": "distinct_state_shared_brake_divergent_activators"
          },
          {
            "name": "residency_install_logic_gate",
            "role": "gate",
            "field": "residency_install_logic",
            "kind": "value",
            "expected": "brake_release_permits_context_matched_activator"
          },
          {
            "name": "activator_context_dependence_gate",
            "role": "gate",
            "field": "activator_context_dependence",
            "kind": "value",
            "expected": "context_matched_crossover"
          },
          {
            "name": "evidence_ceiling_gate",
            "role": "gate",
            "field": "evidence_ceiling",
            "kind": "value",
            "expected": "specificity_dissociations_require_perturbation"
          },
          {
            "name": "cf_install_logic_flat_or_single_step",
            "role": "hard_fail",
            "field": "residency_install_logic",
            "kind": "value",
            "op": "in",
            "expected": [
              "flat_shared_residency_toolkit",
              "activator_induction_sufficient",
              "brake_release_alone_sufficient"
            ]
          },
          {
            "name": "cf_program_identity_lumped_or_wrong",
            "role": "hard_fail",
            "field": "program_identity_conclusion",
            "kind": "value",
            "op": "in",
            "expected": [
              "trm_like_til_is_tumor_trm",
              "trm_like_til_is_exhausted_subset",
              "trm_like_til_is_effector_state",
              "trm_like_til_is_transitional_intermediate",
              "trm_like_til_is_shared_program_distinct_niche"
            ]
          },
          {
            "name": "cf_ahr_assigned_to_til",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Ahr",
            "valueField": "program_specificity",
            "expected": [
              "trm_like_til_specific"
            ]
          },
          {
            "name": "cf_batf_assigned_to_trm_or_shared",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Batf",
            "valueField": "program_specificity",
            "expected": [
              "trm_specific",
              "both"
            ]
          },
          {
            "name": "cf_klf2_called_activator",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Klf2",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          },
          {
            "name": "cf_evidence_ceiling_overclaim",
            "role": "hard_fail",
            "field": "evidence_ceiling",
            "kind": "value",
            "op": "in",
            "expected": [
              "specificity_established_from_regulon_activity",
              "specificity_established_in_dataset",
              "causation_established_by_cross_sectional_concordance",
              "necessity_and_sufficiency_established"
            ]
          },
          {
            "name": "cf_activator_context_shared_or_single",
            "role": "hard_fail",
            "field": "activator_context_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "shared_activators_both_contexts",
              "single_global_activator",
              "redundant_activators"
            ]
          },
          {
            "name": "cf_single_modality_rule",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_hic1_motif_brake_mislabel",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_runx3_anticanonical_as_controller",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Runx3",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-5-pi",
      "modelName": "claude-sonnet-5",
      "harness": "pi",
      "task": "green_shared_vs_distinct_regulators_trm_vs_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 3460.457623,
      "cost": 1.5626434,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_program_map",
            "kind": "list",
            "matchKey": "regulator",
            "k": 3,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "both"
                    ],
                    "canonical": "both"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "represses"
                    ],
                    "canonical": "represses"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Batf",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "trm_like_til_specific"
                    ],
                    "canonical": "trm_like_til_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "chromatin_over_rna"
                    ],
                    "canonical": "chromatin_over_rna"
                  }
                }
              },
              {
                "key": "Ahr",
                "fields": {
                  "program_specificity": {
                    "op": "in",
                    "values": [
                      "trm_specific",
                      "both"
                    ],
                    "canonical": "trm_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "rna_over_chromatin"
                    ],
                    "canonical": "rna_over_chromatin"
                  }
                }
              }
            ]
          },
          {
            "name": "regulator_set_no_extras_gate",
            "role": "gate",
            "field": "$.regulator_program_map[*]",
            "kind": "other",
            "expected": null
          },
          {
            "name": "program_identity_conclusion_gate",
            "role": "gate",
            "field": "program_identity_conclusion",
            "kind": "value",
            "expected": "distinct_state_shared_brake_divergent_activators"
          },
          {
            "name": "residency_install_logic_gate",
            "role": "gate",
            "field": "residency_install_logic",
            "kind": "value",
            "expected": "brake_release_permits_context_matched_activator"
          },
          {
            "name": "activator_context_dependence_gate",
            "role": "gate",
            "field": "activator_context_dependence",
            "kind": "value",
            "expected": "context_matched_crossover"
          },
          {
            "name": "evidence_ceiling_gate",
            "role": "gate",
            "field": "evidence_ceiling",
            "kind": "value",
            "expected": "specificity_dissociations_require_perturbation"
          },
          {
            "name": "cf_install_logic_flat_or_single_step",
            "role": "hard_fail",
            "field": "residency_install_logic",
            "kind": "value",
            "op": "in",
            "expected": [
              "flat_shared_residency_toolkit",
              "activator_induction_sufficient",
              "brake_release_alone_sufficient"
            ]
          },
          {
            "name": "cf_program_identity_lumped_or_wrong",
            "role": "hard_fail",
            "field": "program_identity_conclusion",
            "kind": "value",
            "op": "in",
            "expected": [
              "trm_like_til_is_tumor_trm",
              "trm_like_til_is_exhausted_subset",
              "trm_like_til_is_effector_state",
              "trm_like_til_is_transitional_intermediate",
              "trm_like_til_is_shared_program_distinct_niche"
            ]
          },
          {
            "name": "cf_ahr_assigned_to_til",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Ahr",
            "valueField": "program_specificity",
            "expected": [
              "trm_like_til_specific"
            ]
          },
          {
            "name": "cf_batf_assigned_to_trm_or_shared",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Batf",
            "valueField": "program_specificity",
            "expected": [
              "trm_specific",
              "both"
            ]
          },
          {
            "name": "cf_klf2_called_activator",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Klf2",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          },
          {
            "name": "cf_evidence_ceiling_overclaim",
            "role": "hard_fail",
            "field": "evidence_ceiling",
            "kind": "value",
            "op": "in",
            "expected": [
              "specificity_established_from_regulon_activity",
              "specificity_established_in_dataset",
              "causation_established_by_cross_sectional_concordance",
              "necessity_and_sufficiency_established"
            ]
          },
          {
            "name": "cf_activator_context_shared_or_single",
            "role": "hard_fail",
            "field": "activator_context_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "shared_activators_both_contexts",
              "single_global_activator",
              "redundant_activators"
            ]
          },
          {
            "name": "cf_single_modality_rule",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_hic1_motif_brake_mislabel",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_runx3_anticanonical_as_controller",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Runx3",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-5-pi",
      "modelName": "claude-sonnet-5",
      "harness": "pi",
      "task": "green_shared_vs_distinct_regulators_trm_vs_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 2011.254534,
      "cost": 1.6573260999999995,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_program_map",
            "kind": "list",
            "matchKey": "regulator",
            "k": 3,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "both"
                    ],
                    "canonical": "both"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "represses"
                    ],
                    "canonical": "represses"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Batf",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "trm_like_til_specific"
                    ],
                    "canonical": "trm_like_til_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "chromatin_over_rna"
                    ],
                    "canonical": "chromatin_over_rna"
                  }
                }
              },
              {
                "key": "Ahr",
                "fields": {
                  "program_specificity": {
                    "op": "in",
                    "values": [
                      "trm_specific",
                      "both"
                    ],
                    "canonical": "trm_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "rna_over_chromatin"
                    ],
                    "canonical": "rna_over_chromatin"
                  }
                }
              }
            ]
          },
          {
            "name": "regulator_set_no_extras_gate",
            "role": "gate",
            "field": "$.regulator_program_map[*]",
            "kind": "other",
            "expected": null
          },
          {
            "name": "program_identity_conclusion_gate",
            "role": "gate",
            "field": "program_identity_conclusion",
            "kind": "value",
            "expected": "distinct_state_shared_brake_divergent_activators"
          },
          {
            "name": "residency_install_logic_gate",
            "role": "gate",
            "field": "residency_install_logic",
            "kind": "value",
            "expected": "brake_release_permits_context_matched_activator"
          },
          {
            "name": "activator_context_dependence_gate",
            "role": "gate",
            "field": "activator_context_dependence",
            "kind": "value",
            "expected": "context_matched_crossover"
          },
          {
            "name": "evidence_ceiling_gate",
            "role": "gate",
            "field": "evidence_ceiling",
            "kind": "value",
            "expected": "specificity_dissociations_require_perturbation"
          },
          {
            "name": "cf_install_logic_flat_or_single_step",
            "role": "hard_fail",
            "field": "residency_install_logic",
            "kind": "value",
            "op": "in",
            "expected": [
              "flat_shared_residency_toolkit",
              "activator_induction_sufficient",
              "brake_release_alone_sufficient"
            ]
          },
          {
            "name": "cf_program_identity_lumped_or_wrong",
            "role": "hard_fail",
            "field": "program_identity_conclusion",
            "kind": "value",
            "op": "in",
            "expected": [
              "trm_like_til_is_tumor_trm",
              "trm_like_til_is_exhausted_subset",
              "trm_like_til_is_effector_state",
              "trm_like_til_is_transitional_intermediate",
              "trm_like_til_is_shared_program_distinct_niche"
            ]
          },
          {
            "name": "cf_ahr_assigned_to_til",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Ahr",
            "valueField": "program_specificity",
            "expected": [
              "trm_like_til_specific"
            ]
          },
          {
            "name": "cf_batf_assigned_to_trm_or_shared",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Batf",
            "valueField": "program_specificity",
            "expected": [
              "trm_specific",
              "both"
            ]
          },
          {
            "name": "cf_klf2_called_activator",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Klf2",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          },
          {
            "name": "cf_evidence_ceiling_overclaim",
            "role": "hard_fail",
            "field": "evidence_ceiling",
            "kind": "value",
            "op": "in",
            "expected": [
              "specificity_established_from_regulon_activity",
              "specificity_established_in_dataset",
              "causation_established_by_cross_sectional_concordance",
              "necessity_and_sufficiency_established"
            ]
          },
          {
            "name": "cf_activator_context_shared_or_single",
            "role": "hard_fail",
            "field": "activator_context_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "shared_activators_both_contexts",
              "single_global_activator",
              "redundant_activators"
            ]
          },
          {
            "name": "cf_single_modality_rule",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_hic1_motif_brake_mislabel",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_runx3_anticanonical_as_controller",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Runx3",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-5-pi",
      "modelName": "claude-sonnet-5",
      "harness": "pi",
      "task": "gse297023_q2_senescence_not_mechanism",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 1,
      "passed": true,
      "score": null,
      "durationS": 1563.529886,
      "cost": 0.9202874000000001,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "candidate_is_senescence",
            "role": "gate",
            "field": "$.candidate_mechanism",
            "kind": "value",
            "op": "in",
            "expected": [
              "cellular_senescence",
              "oncogene_induced_senescence",
              "replicative_senescence"
            ]
          },
          {
            "name": "arrest_core_not_engaged",
            "role": "gate",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_not_engaged"
            ]
          },
          {
            "name": "secretory_arm_is_inflammaging",
            "role": "gate",
            "field": "$.secretory_arm_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "acute_phase_inflammaging_response"
            ]
          },
          {
            "name": "secretory_arm_broadly_elevated",
            "role": "gate",
            "field": "$.secretory_arm_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "broadly_elevated"
            ]
          },
          {
            "name": "carries_aging_program",
            "role": "gate",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "carries_bona_fide_aging_program_distinct_from_nominated"
            ]
          },
          {
            "name": "conclusion_senescence_unlikely",
            "role": "gate",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "unlikely_major_contributor"
            ]
          },
          {
            "name": "cf_arrest_core_engaged_recall",
            "role": "hard_fail",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_engaged"
            ]
          },
          {
            "name": "cf_aging_program_conflation_or_explain_away",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "aging_program_indistinguishable_from_nominated_program",
              "aging_program_is_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_no_aging_program_sensitivity_failure",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_detectable_aging_program"
            ]
          },
          {
            "name": "cf_senescence_driver_conclusion",
            "role": "hard_fail",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "likely_primary_driver",
              "plausible_supported_contributor"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-5-pi",
      "modelName": "claude-sonnet-5",
      "harness": "pi",
      "task": "gse297023_q2_senescence_not_mechanism",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 2,
      "passed": true,
      "score": null,
      "durationS": 1106.898861,
      "cost": 0.4766123,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "candidate_is_senescence",
            "role": "gate",
            "field": "$.candidate_mechanism",
            "kind": "value",
            "op": "in",
            "expected": [
              "cellular_senescence",
              "oncogene_induced_senescence",
              "replicative_senescence"
            ]
          },
          {
            "name": "arrest_core_not_engaged",
            "role": "gate",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_not_engaged"
            ]
          },
          {
            "name": "secretory_arm_is_inflammaging",
            "role": "gate",
            "field": "$.secretory_arm_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "acute_phase_inflammaging_response"
            ]
          },
          {
            "name": "secretory_arm_broadly_elevated",
            "role": "gate",
            "field": "$.secretory_arm_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "broadly_elevated"
            ]
          },
          {
            "name": "carries_aging_program",
            "role": "gate",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "carries_bona_fide_aging_program_distinct_from_nominated"
            ]
          },
          {
            "name": "conclusion_senescence_unlikely",
            "role": "gate",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "unlikely_major_contributor"
            ]
          },
          {
            "name": "cf_arrest_core_engaged_recall",
            "role": "hard_fail",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_engaged"
            ]
          },
          {
            "name": "cf_aging_program_conflation_or_explain_away",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "aging_program_indistinguishable_from_nominated_program",
              "aging_program_is_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_no_aging_program_sensitivity_failure",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_detectable_aging_program"
            ]
          },
          {
            "name": "cf_senescence_driver_conclusion",
            "role": "hard_fail",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "likely_primary_driver",
              "plausible_supported_contributor"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-5-pi",
      "modelName": "claude-sonnet-5",
      "harness": "pi",
      "task": "gse297023_q2_senescence_not_mechanism",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1458.123606,
      "cost": 1.0241128000000002,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "candidate_is_senescence",
            "role": "gate",
            "field": "$.candidate_mechanism",
            "kind": "value",
            "op": "in",
            "expected": [
              "cellular_senescence",
              "oncogene_induced_senescence",
              "replicative_senescence"
            ]
          },
          {
            "name": "arrest_core_not_engaged",
            "role": "gate",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_not_engaged"
            ]
          },
          {
            "name": "secretory_arm_is_inflammaging",
            "role": "gate",
            "field": "$.secretory_arm_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "acute_phase_inflammaging_response"
            ]
          },
          {
            "name": "secretory_arm_broadly_elevated",
            "role": "gate",
            "field": "$.secretory_arm_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "broadly_elevated"
            ]
          },
          {
            "name": "carries_aging_program",
            "role": "gate",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "carries_bona_fide_aging_program_distinct_from_nominated"
            ]
          },
          {
            "name": "conclusion_senescence_unlikely",
            "role": "gate",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "unlikely_major_contributor"
            ]
          },
          {
            "name": "cf_arrest_core_engaged_recall",
            "role": "hard_fail",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_engaged"
            ]
          },
          {
            "name": "cf_aging_program_conflation_or_explain_away",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "aging_program_indistinguishable_from_nominated_program",
              "aging_program_is_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_no_aging_program_sensitivity_failure",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_detectable_aging_program"
            ]
          },
          {
            "name": "cf_senescence_driver_conclusion",
            "role": "hard_fail",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "likely_primary_driver",
              "plausible_supported_contributor"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-5-pi",
      "modelName": "claude-sonnet-5",
      "harness": "pi",
      "task": "gse297023_q3_age_retained_mapk_reduced",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 1,
      "passed": true,
      "score": null,
      "durationS": 2903.511062,
      "cost": 1.4450575000000003,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "age_retained",
            "role": "gate",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_retained"
            ]
          },
          {
            "name": "identity_is_aging",
            "role": "gate",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_aging_signatures"
            ]
          },
          {
            "name": "nature_state_shift",
            "role": "gate",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_state_expression_shift"
            ]
          },
          {
            "name": "pathway_is_mapk",
            "role": "gate",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "mapk"
            ]
          },
          {
            "name": "pathway_reduced_in_aged",
            "role": "gate",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "reduced_in_aged"
            ]
          },
          {
            "name": "cf_mapk_increased_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_in_aged"
            ]
          },
          {
            "name": "cf_age_erased",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_erased"
            ]
          },
          {
            "name": "cf_pervasive_overcall",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_pervasive_artifactual"
            ]
          },
          {
            "name": "cf_explain_away_aging",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_compositional_or_identity",
            "role": "hard_fail",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_proportion_shift",
              "cell_identity_shift",
              "composition_dominant_state_secondary",
              "both_state_and_composition_equally"
            ]
          },
          {
            "name": "cf_biggest_inflammatory_signal_not_mapk",
            "role": "hard_fail",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "jak_stat",
              "nfkb",
              "tnfa"
            ]
          },
          {
            "name": "cf_senescence_misattribution",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_senescence_signature_specifically"
            ]
          },
          {
            "name": "cf_dismiss_as_batch_artifact",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_batch_or_technical_artifact"
            ]
          },
          {
            "name": "cf_mapk_comparable_constitutive_kras_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "comparable_across_ages"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-5-pi",
      "modelName": "claude-sonnet-5",
      "harness": "pi",
      "task": "gse297023_q3_age_retained_mapk_reduced",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 2125.210694,
      "cost": 0.9798496999999998,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "age_retained",
            "role": "gate",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_retained"
            ]
          },
          {
            "name": "identity_is_aging",
            "role": "gate",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_aging_signatures"
            ]
          },
          {
            "name": "nature_state_shift",
            "role": "gate",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_state_expression_shift"
            ]
          },
          {
            "name": "pathway_is_mapk",
            "role": "gate",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "mapk"
            ]
          },
          {
            "name": "pathway_reduced_in_aged",
            "role": "gate",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "reduced_in_aged"
            ]
          },
          {
            "name": "cf_mapk_increased_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_in_aged"
            ]
          },
          {
            "name": "cf_age_erased",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_erased"
            ]
          },
          {
            "name": "cf_pervasive_overcall",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_pervasive_artifactual"
            ]
          },
          {
            "name": "cf_explain_away_aging",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_compositional_or_identity",
            "role": "hard_fail",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_proportion_shift",
              "cell_identity_shift",
              "composition_dominant_state_secondary",
              "both_state_and_composition_equally"
            ]
          },
          {
            "name": "cf_biggest_inflammatory_signal_not_mapk",
            "role": "hard_fail",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "jak_stat",
              "nfkb",
              "tnfa"
            ]
          },
          {
            "name": "cf_senescence_misattribution",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_senescence_signature_specifically"
            ]
          },
          {
            "name": "cf_dismiss_as_batch_artifact",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_batch_or_technical_artifact"
            ]
          },
          {
            "name": "cf_mapk_comparable_constitutive_kras_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "comparable_across_ages"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-5-pi",
      "modelName": "claude-sonnet-5",
      "harness": "pi",
      "task": "gse297023_q3_age_retained_mapk_reduced",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 3083.235429,
      "cost": 1.4590860000000005,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "age_retained",
            "role": "gate",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_retained"
            ]
          },
          {
            "name": "identity_is_aging",
            "role": "gate",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_aging_signatures"
            ]
          },
          {
            "name": "nature_state_shift",
            "role": "gate",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_state_expression_shift"
            ]
          },
          {
            "name": "pathway_is_mapk",
            "role": "gate",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "mapk"
            ]
          },
          {
            "name": "pathway_reduced_in_aged",
            "role": "gate",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "reduced_in_aged"
            ]
          },
          {
            "name": "cf_mapk_increased_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_in_aged"
            ]
          },
          {
            "name": "cf_age_erased",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_erased"
            ]
          },
          {
            "name": "cf_pervasive_overcall",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_pervasive_artifactual"
            ]
          },
          {
            "name": "cf_explain_away_aging",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_compositional_or_identity",
            "role": "hard_fail",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_proportion_shift",
              "cell_identity_shift",
              "composition_dominant_state_secondary",
              "both_state_and_composition_equally"
            ]
          },
          {
            "name": "cf_biggest_inflammatory_signal_not_mapk",
            "role": "hard_fail",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "jak_stat",
              "nfkb",
              "tnfa"
            ]
          },
          {
            "name": "cf_senescence_misattribution",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_senescence_signature_specifically"
            ]
          },
          {
            "name": "cf_dismiss_as_batch_artifact",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_batch_or_technical_artifact"
            ]
          },
          {
            "name": "cf_mapk_comparable_constitutive_kras_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "comparable_across_ages"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-5-pi",
      "modelName": "claude-sonnet-5",
      "harness": "pi",
      "task": "gse297023_q5_tme_age_dependent_remodeling",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 7222.248037,
      "cost": 3.5193899999999996,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "perturbation_is_pten",
            "role": "gate",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "pten"
            ]
          },
          {
            "name": "tme_non_cell_autonomous",
            "role": "gate",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "substantial_non_cell_autonomous_remodeling"
            ]
          },
          {
            "name": "tme_state_not_composition",
            "role": "gate",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "tme_age_dependent",
            "role": "gate",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_dependent"
            ]
          },
          {
            "name": "tme_response_broader_in_aged",
            "role": "gate",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_aged"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "most_responsive_noncancer_cell_types",
            "kind": "list",
            "matchKey": "cell_type",
            "k": 2,
            "expected": [
              {
                "key": "alveolar_macrophage",
                "fields": {}
              },
              {
                "key": "endothelial",
                "fields": {}
              }
            ]
          },
          {
            "name": "cf_cell_intrinsic_only",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "negligible_cell_intrinsic_only_effect",
              "remodeling_unresolved",
              "modest_remodeling_few_cell_types"
            ]
          },
          {
            "name": "cf_remodeling_is_contamination",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "apparent_remodeling_reflects_cancer_contamination"
            ]
          },
          {
            "name": "cf_perturbation_design_misread",
            "role": "hard_fail",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "multiple_tumor_suppressors_pooled",
              "no_additional_gene_difference_kras_only",
              "rb1_cdkn2a_double",
              "pcna_essential_gene_control",
              "difference_is_oncogene_not_suppressor"
            ]
          },
          {
            "name": "cf_age_independent_tme",
            "role": "hard_fail",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_independent"
            ]
          },
          {
            "name": "cf_compositional_shift",
            "role": "hard_fail",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_frequency_shift",
              "compositional_immune_infiltration",
              "composition_dominant_state_secondary",
              "both_state_and_composition"
            ]
          },
          {
            "name": "cf_epithelial_or_cancer_substitution",
            "role": "hard_fail",
            "field": "$.most_responsive_noncancer_cell_types[*].cell_type",
            "kind": "avoid",
            "expected": [
              "AT2_cell",
              "AT1_cell",
              "cancer_cell",
              "ciliated",
              "secretory",
              "club_cell",
              "basal_cell",
              "goblet_cell"
            ]
          },
          {
            "name": "cf_tme_breadth_broader_in_young",
            "role": "hard_fail",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_young"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-5-pi",
      "modelName": "claude-sonnet-5",
      "harness": "pi",
      "task": "gse297023_q5_tme_age_dependent_remodeling",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 7644.957751,
      "cost": 3.378250900000001,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "perturbation_is_pten",
            "role": "gate",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "pten"
            ]
          },
          {
            "name": "tme_non_cell_autonomous",
            "role": "gate",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "substantial_non_cell_autonomous_remodeling"
            ]
          },
          {
            "name": "tme_state_not_composition",
            "role": "gate",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "tme_age_dependent",
            "role": "gate",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_dependent"
            ]
          },
          {
            "name": "tme_response_broader_in_aged",
            "role": "gate",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_aged"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "most_responsive_noncancer_cell_types",
            "kind": "list",
            "matchKey": "cell_type",
            "k": 2,
            "expected": [
              {
                "key": "alveolar_macrophage",
                "fields": {}
              },
              {
                "key": "endothelial",
                "fields": {}
              }
            ]
          },
          {
            "name": "cf_cell_intrinsic_only",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "negligible_cell_intrinsic_only_effect",
              "remodeling_unresolved",
              "modest_remodeling_few_cell_types"
            ]
          },
          {
            "name": "cf_remodeling_is_contamination",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "apparent_remodeling_reflects_cancer_contamination"
            ]
          },
          {
            "name": "cf_perturbation_design_misread",
            "role": "hard_fail",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "multiple_tumor_suppressors_pooled",
              "no_additional_gene_difference_kras_only",
              "rb1_cdkn2a_double",
              "pcna_essential_gene_control",
              "difference_is_oncogene_not_suppressor"
            ]
          },
          {
            "name": "cf_age_independent_tme",
            "role": "hard_fail",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_independent"
            ]
          },
          {
            "name": "cf_compositional_shift",
            "role": "hard_fail",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_frequency_shift",
              "compositional_immune_infiltration",
              "composition_dominant_state_secondary",
              "both_state_and_composition"
            ]
          },
          {
            "name": "cf_epithelial_or_cancer_substitution",
            "role": "hard_fail",
            "field": "$.most_responsive_noncancer_cell_types[*].cell_type",
            "kind": "avoid",
            "expected": [
              "AT2_cell",
              "AT1_cell",
              "cancer_cell",
              "ciliated",
              "secretory",
              "club_cell",
              "basal_cell",
              "goblet_cell"
            ]
          },
          {
            "name": "cf_tme_breadth_broader_in_young",
            "role": "hard_fail",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_young"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-5-pi",
      "modelName": "claude-sonnet-5",
      "harness": "pi",
      "task": "gse297023_q5_tme_age_dependent_remodeling",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 4540.469829,
      "cost": 2.5281198000000003,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "perturbation_is_pten",
            "role": "gate",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "pten"
            ]
          },
          {
            "name": "tme_non_cell_autonomous",
            "role": "gate",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "substantial_non_cell_autonomous_remodeling"
            ]
          },
          {
            "name": "tme_state_not_composition",
            "role": "gate",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "tme_age_dependent",
            "role": "gate",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_dependent"
            ]
          },
          {
            "name": "tme_response_broader_in_aged",
            "role": "gate",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_aged"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "most_responsive_noncancer_cell_types",
            "kind": "list",
            "matchKey": "cell_type",
            "k": 2,
            "expected": [
              {
                "key": "alveolar_macrophage",
                "fields": {}
              },
              {
                "key": "endothelial",
                "fields": {}
              }
            ]
          },
          {
            "name": "cf_cell_intrinsic_only",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "negligible_cell_intrinsic_only_effect",
              "remodeling_unresolved",
              "modest_remodeling_few_cell_types"
            ]
          },
          {
            "name": "cf_remodeling_is_contamination",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "apparent_remodeling_reflects_cancer_contamination"
            ]
          },
          {
            "name": "cf_perturbation_design_misread",
            "role": "hard_fail",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "multiple_tumor_suppressors_pooled",
              "no_additional_gene_difference_kras_only",
              "rb1_cdkn2a_double",
              "pcna_essential_gene_control",
              "difference_is_oncogene_not_suppressor"
            ]
          },
          {
            "name": "cf_age_independent_tme",
            "role": "hard_fail",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_independent"
            ]
          },
          {
            "name": "cf_compositional_shift",
            "role": "hard_fail",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_frequency_shift",
              "compositional_immune_infiltration",
              "composition_dominant_state_secondary",
              "both_state_and_composition"
            ]
          },
          {
            "name": "cf_epithelial_or_cancer_substitution",
            "role": "hard_fail",
            "field": "$.most_responsive_noncancer_cell_types[*].cell_type",
            "kind": "avoid",
            "expected": [
              "AT2_cell",
              "AT1_cell",
              "cancer_cell",
              "ciliated",
              "secretory",
              "club_cell",
              "basal_cell",
              "goblet_cell"
            ]
          },
          {
            "name": "cf_tme_breadth_broader_in_young",
            "role": "hard_fail",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_young"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-5-pi",
      "modelName": "claude-sonnet-5",
      "harness": "pi",
      "task": "gse297023_q6_age_immune_infiltration",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 1,
      "passed": true,
      "score": null,
      "durationS": 2815.090314,
      "cost": 1.104568,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "composition_stable_with_age",
            "role": "gate",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "frequencies_largely_stable"
            ]
          },
          {
            "name": "age_signal_is_state_not_composition",
            "role": "gate",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "frequencies_uninformative_for_infiltration",
            "role": "gate",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_cannot_establish_infiltration"
            ]
          },
          {
            "name": "infiltration_increases_with_age",
            "role": "gate",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_with_age"
            ]
          },
          {
            "name": "infiltration_strength_calibrated",
            "role": "gate",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "significant_increase_strongest_markers_only",
              "consistent_increase_not_statistically_robust"
            ]
          },
          {
            "name": "genotype_marker_fraction_lower_in_pten",
            "role": "gate",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "lower_in_pten_deficient"
            ]
          },
          {
            "name": "cf_composition_shift_reading",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "major_compositional_shift_with_age",
              "increased_immune_fraction_with_age"
            ]
          },
          {
            "name": "cf_lineage_specific_composition_claim",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "myeloid_expansion_with_age",
              "lymphoid_expansion_with_age",
              "minor_shifts_in_some_cell_types"
            ]
          },
          {
            "name": "cf_immunosenescence_depletion_prior",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "immune_depletion_with_age"
            ]
          },
          {
            "name": "cf_age_modality_compositional",
            "role": "hard_fail",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_type_composition_shift",
              "both_equally",
              "composition_dominant_state_secondary"
            ]
          },
          {
            "name": "cf_overread_frequencies_as_infiltration",
            "role": "hard_fail",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_shows_no_infiltration_change",
              "single_cell_quantifies_infiltration",
              "frequencies_show_increased_infiltration",
              "frequencies_show_decreased_infiltration"
            ]
          },
          {
            "name": "cf_infiltration_decreases_immunosenescence",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "decreased_with_age"
            ]
          },
          {
            "name": "cf_infiltration_unchanged_scrna_overread",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "unchanged_with_age"
            ]
          },
          {
            "name": "cf_infiltration_indeterminate_unused_node",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "direction_indeterminate_from_data"
            ]
          },
          {
            "name": "cf_infiltration_mixed_direction",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "mixed_by_immune_cell_type"
            ]
          },
          {
            "name": "cf_infiltration_genotype_confusion",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_only_in_pten_deficient",
              "decreased_only_in_pten_deficient"
            ]
          },
          {
            "name": "cf_pten_infiltration_paper_text_recall",
            "role": "hard_fail",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "higher_in_pten_deficient",
              "higher_in_pten_deficient_only_in_young",
              "higher_in_pten_deficient_only_in_aged"
            ]
          },
          {
            "name": "cf_pseudoreplication_overclaim",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_increase_across_markers"
            ]
          },
          {
            "name": "cf_infiltration_strength_decrease_direction",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_decrease_across_markers",
              "significant_decrease_strongest_markers_only",
              "consistent_decrease_not_statistically_robust"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-5-pi",
      "modelName": "claude-sonnet-5",
      "harness": "pi",
      "task": "gse297023_q6_age_immune_infiltration",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 1639.771279,
      "cost": 0.7338704,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "composition_stable_with_age",
            "role": "gate",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "frequencies_largely_stable"
            ]
          },
          {
            "name": "age_signal_is_state_not_composition",
            "role": "gate",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "frequencies_uninformative_for_infiltration",
            "role": "gate",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_cannot_establish_infiltration"
            ]
          },
          {
            "name": "infiltration_increases_with_age",
            "role": "gate",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_with_age"
            ]
          },
          {
            "name": "infiltration_strength_calibrated",
            "role": "gate",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "significant_increase_strongest_markers_only",
              "consistent_increase_not_statistically_robust"
            ]
          },
          {
            "name": "genotype_marker_fraction_lower_in_pten",
            "role": "gate",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "lower_in_pten_deficient"
            ]
          },
          {
            "name": "cf_composition_shift_reading",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "major_compositional_shift_with_age",
              "increased_immune_fraction_with_age"
            ]
          },
          {
            "name": "cf_lineage_specific_composition_claim",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "myeloid_expansion_with_age",
              "lymphoid_expansion_with_age",
              "minor_shifts_in_some_cell_types"
            ]
          },
          {
            "name": "cf_immunosenescence_depletion_prior",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "immune_depletion_with_age"
            ]
          },
          {
            "name": "cf_age_modality_compositional",
            "role": "hard_fail",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_type_composition_shift",
              "both_equally",
              "composition_dominant_state_secondary"
            ]
          },
          {
            "name": "cf_overread_frequencies_as_infiltration",
            "role": "hard_fail",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_shows_no_infiltration_change",
              "single_cell_quantifies_infiltration",
              "frequencies_show_increased_infiltration",
              "frequencies_show_decreased_infiltration"
            ]
          },
          {
            "name": "cf_infiltration_decreases_immunosenescence",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "decreased_with_age"
            ]
          },
          {
            "name": "cf_infiltration_unchanged_scrna_overread",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "unchanged_with_age"
            ]
          },
          {
            "name": "cf_infiltration_indeterminate_unused_node",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "direction_indeterminate_from_data"
            ]
          },
          {
            "name": "cf_infiltration_mixed_direction",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "mixed_by_immune_cell_type"
            ]
          },
          {
            "name": "cf_infiltration_genotype_confusion",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_only_in_pten_deficient",
              "decreased_only_in_pten_deficient"
            ]
          },
          {
            "name": "cf_pten_infiltration_paper_text_recall",
            "role": "hard_fail",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "higher_in_pten_deficient",
              "higher_in_pten_deficient_only_in_young",
              "higher_in_pten_deficient_only_in_aged"
            ]
          },
          {
            "name": "cf_pseudoreplication_overclaim",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_increase_across_markers"
            ]
          },
          {
            "name": "cf_infiltration_strength_decrease_direction",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_decrease_across_markers",
              "significant_decrease_strongest_markers_only",
              "consistent_decrease_not_statistically_robust"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-sonnet-5-pi",
      "modelName": "claude-sonnet-5",
      "harness": "pi",
      "task": "gse297023_q6_age_immune_infiltration",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1809.618501,
      "cost": 1.1631610000000003,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "composition_stable_with_age",
            "role": "gate",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "frequencies_largely_stable"
            ]
          },
          {
            "name": "age_signal_is_state_not_composition",
            "role": "gate",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "frequencies_uninformative_for_infiltration",
            "role": "gate",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_cannot_establish_infiltration"
            ]
          },
          {
            "name": "infiltration_increases_with_age",
            "role": "gate",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_with_age"
            ]
          },
          {
            "name": "infiltration_strength_calibrated",
            "role": "gate",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "significant_increase_strongest_markers_only",
              "consistent_increase_not_statistically_robust"
            ]
          },
          {
            "name": "genotype_marker_fraction_lower_in_pten",
            "role": "gate",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "lower_in_pten_deficient"
            ]
          },
          {
            "name": "cf_composition_shift_reading",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "major_compositional_shift_with_age",
              "increased_immune_fraction_with_age"
            ]
          },
          {
            "name": "cf_lineage_specific_composition_claim",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "myeloid_expansion_with_age",
              "lymphoid_expansion_with_age",
              "minor_shifts_in_some_cell_types"
            ]
          },
          {
            "name": "cf_immunosenescence_depletion_prior",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "immune_depletion_with_age"
            ]
          },
          {
            "name": "cf_age_modality_compositional",
            "role": "hard_fail",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_type_composition_shift",
              "both_equally",
              "composition_dominant_state_secondary"
            ]
          },
          {
            "name": "cf_overread_frequencies_as_infiltration",
            "role": "hard_fail",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_shows_no_infiltration_change",
              "single_cell_quantifies_infiltration",
              "frequencies_show_increased_infiltration",
              "frequencies_show_decreased_infiltration"
            ]
          },
          {
            "name": "cf_infiltration_decreases_immunosenescence",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "decreased_with_age"
            ]
          },
          {
            "name": "cf_infiltration_unchanged_scrna_overread",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "unchanged_with_age"
            ]
          },
          {
            "name": "cf_infiltration_indeterminate_unused_node",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "direction_indeterminate_from_data"
            ]
          },
          {
            "name": "cf_infiltration_mixed_direction",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "mixed_by_immune_cell_type"
            ]
          },
          {
            "name": "cf_infiltration_genotype_confusion",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_only_in_pten_deficient",
              "decreased_only_in_pten_deficient"
            ]
          },
          {
            "name": "cf_pten_infiltration_paper_text_recall",
            "role": "hard_fail",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "higher_in_pten_deficient",
              "higher_in_pten_deficient_only_in_young",
              "higher_in_pten_deficient_only_in_aged"
            ]
          },
          {
            "name": "cf_pseudoreplication_overclaim",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_increase_across_markers"
            ]
          },
          {
            "name": "cf_infiltration_strength_decrease_direction",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_decrease_across_markers",
              "significant_decrease_strongest_markers_only",
              "consistent_decrease_not_statistically_robust"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-luna-pi",
      "modelName": "gpt-5.6-luna",
      "harness": "pi",
      "task": "gse155381_q4_pseudotime_delay",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 2992.675749,
      "cost": 0.89010425,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "pseudotime_direction_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "E17_chronological_match_weighted",
            "role": "additive",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pluripotency_exit_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pseudotime_direction_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_synchronous_with_monkey_EPI",
              "chimera_human_EPI_developmentally_advanced",
              "chimera_human_EPI_delayed_at_pre_implantation_only_resolved_post_implantation"
            ]
          },
          {
            "name": "E17_chronological_match_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "resembles_monkey_E17_Gast_EPI",
              "resembles_chimera_monkey_E17_EPI_specifically",
              "chimera_human_E17_EPI_more_advanced_than_monkey_E17_Gast"
            ]
          },
          {
            "name": "pluripotency_exit_paper_inverted_or_prior_driven",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_advances_pluripotency_exit_prematurely",
              "naive_pluripotency_program_specifically_engaged",
              "pluripotency_core_retained_via_KLF17_naive_marker_specifically"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-luna-pi",
      "modelName": "gpt-5.6-luna",
      "harness": "pi",
      "task": "gse155381_q4_pseudotime_delay",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 7878.444249,
      "cost": 1.2004824,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "pseudotime_direction_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "E17_chronological_match_weighted",
            "role": "additive",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pluripotency_exit_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pseudotime_direction_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_synchronous_with_monkey_EPI",
              "chimera_human_EPI_developmentally_advanced",
              "chimera_human_EPI_delayed_at_pre_implantation_only_resolved_post_implantation"
            ]
          },
          {
            "name": "E17_chronological_match_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "resembles_monkey_E17_Gast_EPI",
              "resembles_chimera_monkey_E17_EPI_specifically",
              "chimera_human_E17_EPI_more_advanced_than_monkey_E17_Gast"
            ]
          },
          {
            "name": "pluripotency_exit_paper_inverted_or_prior_driven",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_advances_pluripotency_exit_prematurely",
              "naive_pluripotency_program_specifically_engaged",
              "pluripotency_core_retained_via_KLF17_naive_marker_specifically"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-luna-pi",
      "modelName": "gpt-5.6-luna",
      "harness": "pi",
      "task": "gse155381_q4_pseudotime_delay",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 948.05935,
      "cost": 0.9762819000000003,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "pseudotime_direction_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "E17_chronological_match_weighted",
            "role": "additive",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pluripotency_exit_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pseudotime_direction_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_synchronous_with_monkey_EPI",
              "chimera_human_EPI_developmentally_advanced",
              "chimera_human_EPI_delayed_at_pre_implantation_only_resolved_post_implantation"
            ]
          },
          {
            "name": "E17_chronological_match_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "resembles_monkey_E17_Gast_EPI",
              "resembles_chimera_monkey_E17_EPI_specifically",
              "chimera_human_E17_EPI_more_advanced_than_monkey_E17_Gast"
            ]
          },
          {
            "name": "pluripotency_exit_paper_inverted_or_prior_driven",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_advances_pluripotency_exit_prematurely",
              "naive_pluripotency_program_specifically_engaged",
              "pluripotency_core_retained_via_KLF17_naive_marker_specifically"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-luna-pi",
      "modelName": "gpt-5.6-luna",
      "harness": "pi",
      "task": "gse155381_q5_chimera_monkey_perturbation",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 505.986487,
      "cost": 0.4422791500000001,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "deg_direction_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "comparison_subset_choice_weighted",
            "role": "additive",
            "field": "$.comparison_subset_choice",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "panel_resolved_strong_target_ccn1_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN1",
              "CYR61"
            ]
          },
          {
            "name": "panel_resolved_strong_target_ccn2_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN2",
              "CTGF"
            ]
          },
          {
            "name": "hippo_pathway_weighted",
            "role": "additive",
            "field": "$.hippo_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "tgf_beta_pathway_weighted",
            "role": "additive",
            "field": "$.tgf_beta_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "deg_direction_batch_effect_uncorrected",
            "role": "hard_fail",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_dominated_global",
              "up_dominated_in_chosen_subset"
            ]
          },
          {
            "name": "silent_panel_dropout_paired_with_directional_pathway_claim",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "hippo_pathway_direction": {
                "op": "in",
                "values": [
                  "hippo_pathway_upregulated",
                  "hippo_pathway_unchanged",
                  "wnt_pathway_specifically_perturbed",
                  "stress_response_pathway_dominantly_engaged",
                  "no_pathway_signal_above_confound",
                  "yap_tead_direct_target_panel_downregulated",
                  "yap_tead_panel_dependent_direction",
                  "hippo_pathway_downregulated"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-luna-pi",
      "modelName": "gpt-5.6-luna",
      "harness": "pi",
      "task": "gse155381_q5_chimera_monkey_perturbation",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 1183.322223,
      "cost": 0.47233610000000004,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "deg_direction_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "comparison_subset_choice_weighted",
            "role": "additive",
            "field": "$.comparison_subset_choice",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "panel_resolved_strong_target_ccn1_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN1",
              "CYR61"
            ]
          },
          {
            "name": "panel_resolved_strong_target_ccn2_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN2",
              "CTGF"
            ]
          },
          {
            "name": "hippo_pathway_weighted",
            "role": "additive",
            "field": "$.hippo_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "tgf_beta_pathway_weighted",
            "role": "additive",
            "field": "$.tgf_beta_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "deg_direction_batch_effect_uncorrected",
            "role": "hard_fail",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_dominated_global",
              "up_dominated_in_chosen_subset"
            ]
          },
          {
            "name": "silent_panel_dropout_paired_with_directional_pathway_claim",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "hippo_pathway_direction": {
                "op": "in",
                "values": [
                  "hippo_pathway_upregulated",
                  "hippo_pathway_unchanged",
                  "wnt_pathway_specifically_perturbed",
                  "stress_response_pathway_dominantly_engaged",
                  "no_pathway_signal_above_confound",
                  "yap_tead_direct_target_panel_downregulated",
                  "yap_tead_panel_dependent_direction",
                  "hippo_pathway_downregulated"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-luna-pi",
      "modelName": "gpt-5.6-luna",
      "harness": "pi",
      "task": "gse155381_q5_chimera_monkey_perturbation",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 695.858962,
      "cost": 0.48036680000000004,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "deg_direction_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "comparison_subset_choice_weighted",
            "role": "additive",
            "field": "$.comparison_subset_choice",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "panel_resolved_strong_target_ccn1_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN1",
              "CYR61"
            ]
          },
          {
            "name": "panel_resolved_strong_target_ccn2_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN2",
              "CTGF"
            ]
          },
          {
            "name": "hippo_pathway_weighted",
            "role": "additive",
            "field": "$.hippo_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "tgf_beta_pathway_weighted",
            "role": "additive",
            "field": "$.tgf_beta_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "deg_direction_batch_effect_uncorrected",
            "role": "hard_fail",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_dominated_global",
              "up_dominated_in_chosen_subset"
            ]
          },
          {
            "name": "silent_panel_dropout_paired_with_directional_pathway_claim",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "hippo_pathway_direction": {
                "op": "in",
                "values": [
                  "hippo_pathway_upregulated",
                  "hippo_pathway_unchanged",
                  "wnt_pathway_specifically_perturbed",
                  "stress_response_pathway_dominantly_engaged",
                  "no_pathway_signal_above_confound",
                  "yap_tead_direct_target_panel_downregulated",
                  "yap_tead_panel_dependent_direction",
                  "hippo_pathway_downregulated"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-luna-pi",
      "modelName": "gpt-5.6-luna",
      "harness": "pi",
      "task": "gse155381_q7_xspecies_epi_communication",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 1455.028817,
      "cost": 0.9863804500000002,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "richness_weighted",
            "role": "additive",
            "field": "$.xspecies_signaling_richness",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_weighted",
            "role": "additive",
            "field": "$.dominant_signaling_axis",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_wnt_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "WNT_FZD_LRP_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_fgf_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "FGF_FGFR_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_notch_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "NOTCH_DLL_JAG_family"
                ]
              }
            }
          },
          {
            "name": "signaling_directionality_weighted",
            "role": "additive",
            "field": "$.signaling_directionality",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "top_signal_interaction_class_weighted",
            "role": "additive",
            "field": "$.top_signal_interaction_class",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-luna-pi",
      "modelName": "gpt-5.6-luna",
      "harness": "pi",
      "task": "gse155381_q7_xspecies_epi_communication",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 634.672678,
      "cost": 0.5891525000000001,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "richness_weighted",
            "role": "additive",
            "field": "$.xspecies_signaling_richness",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_weighted",
            "role": "additive",
            "field": "$.dominant_signaling_axis",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_wnt_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "WNT_FZD_LRP_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_fgf_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "FGF_FGFR_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_notch_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "NOTCH_DLL_JAG_family"
                ]
              }
            }
          },
          {
            "name": "signaling_directionality_weighted",
            "role": "additive",
            "field": "$.signaling_directionality",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "top_signal_interaction_class_weighted",
            "role": "additive",
            "field": "$.top_signal_interaction_class",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-luna-pi",
      "modelName": "gpt-5.6-luna",
      "harness": "pi",
      "task": "gse155381_q7_xspecies_epi_communication",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1117.546158,
      "cost": 1.0397634500000001,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "richness_weighted",
            "role": "additive",
            "field": "$.xspecies_signaling_richness",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_weighted",
            "role": "additive",
            "field": "$.dominant_signaling_axis",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_wnt_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "WNT_FZD_LRP_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_fgf_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "FGF_FGFR_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_notch_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "NOTCH_DLL_JAG_family"
                ]
              }
            }
          },
          {
            "name": "signaling_directionality_weighted",
            "role": "additive",
            "field": "$.signaling_directionality",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "top_signal_interaction_class_weighted",
            "role": "additive",
            "field": "$.top_signal_interaction_class",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-luna-pi",
      "modelName": "gpt-5.6-luna",
      "harness": "pi",
      "task": "gse155381_q8_bidirectional_convergence",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 1853.132333,
      "cost": 3.4210354500000006,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "chimera_human_identity_shift_target_weighted",
            "role": "additive",
            "field": "$.chimera_human_identity_shift_target",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_in_vivo_state_assignment_weighted",
            "role": "additive",
            "field": "$.chimera_human_in_vivo_state_assignment",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_shift_magnitude_weighted",
            "role": "additive",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_monkey_identity_shift_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "both_arms_null_contradicted_by_calibration",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "chimera_human_identity_shift_target": {
                "op": "equals",
                "values": [
                  "chimera_human_maintains_origin_identity_no_measurable_shift"
                ]
              },
              "chimera_monkey_identity_shift": {
                "op": "equals",
                "values": [
                  "chimera_monkey_unchanged_relative_to_control"
                ]
              }
            }
          },
          {
            "name": "chimera_monkey_paper_claim_on_confounded_input",
            "role": "hard_fail",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "value",
            "expected": "chimera_monkey_shifts_toward_chimera_human_identity"
          },
          {
            "name": "chimera_human_magnitude_wrong_attribution_signature",
            "role": "hard_fail",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "value",
            "op": "in",
            "expected": [
              "dramatic_complete_replacement_of_origin_identity",
              "magnitude_dominated_by_culture_or_metabolic_axis",
              "magnitude_dominated_by_developmental_stage_axis",
              "magnitude_dominated_by_cell_cycle_axis",
              "magnitude_dominated_by_batch_effect_axis"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-luna-pi",
      "modelName": "gpt-5.6-luna",
      "harness": "pi",
      "task": "gse155381_q8_bidirectional_convergence",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 777.219386,
      "cost": 0.7650059500000002,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "chimera_human_identity_shift_target_weighted",
            "role": "additive",
            "field": "$.chimera_human_identity_shift_target",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_in_vivo_state_assignment_weighted",
            "role": "additive",
            "field": "$.chimera_human_in_vivo_state_assignment",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_shift_magnitude_weighted",
            "role": "additive",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_monkey_identity_shift_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "both_arms_null_contradicted_by_calibration",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "chimera_human_identity_shift_target": {
                "op": "equals",
                "values": [
                  "chimera_human_maintains_origin_identity_no_measurable_shift"
                ]
              },
              "chimera_monkey_identity_shift": {
                "op": "equals",
                "values": [
                  "chimera_monkey_unchanged_relative_to_control"
                ]
              }
            }
          },
          {
            "name": "chimera_monkey_paper_claim_on_confounded_input",
            "role": "hard_fail",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "value",
            "expected": "chimera_monkey_shifts_toward_chimera_human_identity"
          },
          {
            "name": "chimera_human_magnitude_wrong_attribution_signature",
            "role": "hard_fail",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "value",
            "op": "in",
            "expected": [
              "dramatic_complete_replacement_of_origin_identity",
              "magnitude_dominated_by_culture_or_metabolic_axis",
              "magnitude_dominated_by_developmental_stage_axis",
              "magnitude_dominated_by_cell_cycle_axis",
              "magnitude_dominated_by_batch_effect_axis"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-luna-pi",
      "modelName": "gpt-5.6-luna",
      "harness": "pi",
      "task": "gse155381_q8_bidirectional_convergence",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 608.282432,
      "cost": 1.0568839,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "chimera_human_identity_shift_target_weighted",
            "role": "additive",
            "field": "$.chimera_human_identity_shift_target",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_in_vivo_state_assignment_weighted",
            "role": "additive",
            "field": "$.chimera_human_in_vivo_state_assignment",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_shift_magnitude_weighted",
            "role": "additive",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_monkey_identity_shift_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "both_arms_null_contradicted_by_calibration",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "chimera_human_identity_shift_target": {
                "op": "equals",
                "values": [
                  "chimera_human_maintains_origin_identity_no_measurable_shift"
                ]
              },
              "chimera_monkey_identity_shift": {
                "op": "equals",
                "values": [
                  "chimera_monkey_unchanged_relative_to_control"
                ]
              }
            }
          },
          {
            "name": "chimera_monkey_paper_claim_on_confounded_input",
            "role": "hard_fail",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "value",
            "expected": "chimera_monkey_shifts_toward_chimera_human_identity"
          },
          {
            "name": "chimera_human_magnitude_wrong_attribution_signature",
            "role": "hard_fail",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "value",
            "op": "in",
            "expected": [
              "dramatic_complete_replacement_of_origin_identity",
              "magnitude_dominated_by_culture_or_metabolic_axis",
              "magnitude_dominated_by_developmental_stage_axis",
              "magnitude_dominated_by_cell_cycle_axis",
              "magnitude_dominated_by_batch_effect_axis"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-luna-pi",
      "modelName": "gpt-5.6-luna",
      "harness": "pi",
      "task": "gse171524_ectopic_airway_rare_cell",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 883.247751,
      "cost": 0.5911363000000001,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "focus_population_identity_gate",
            "role": "gate",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "chemosensory_marker_expressing_population_alpha"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "focus_population_marker_calls",
            "kind": "list",
            "matchKey": "marker",
            "k": 3,
            "expected": [
              {
                "key": "POU2F3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "CHAT",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "TRPM5",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "LRMP",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNAT3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "AVIL",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNG13",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              }
            ]
          },
          {
            "name": "focus_population_condition_distribution_gate",
            "role": "gate",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_enriched_with_substantial_control_presence",
              "covid_predominant",
              "covid_only",
              "covid_enriched_trend_not_significant"
            ]
          },
          {
            "name": "focus_population_location_gate",
            "role": "gate",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "ectopic_parenchyma_in_covid",
              "parenchyma_only",
              "bronchiolized_or_peribronchial_parenchyma",
              "airway_and_parenchyma"
            ]
          },
          {
            "name": "focus_population_recovery_mode_gate",
            "role": "gate",
            "field": "$.focus_population_recovery_mode",
            "kind": "value",
            "op": "in",
            "expected": [
              "continuum_not_discrete_cluster"
            ]
          },
          {
            "name": "focus_population_identity_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "ionocyte_like_beta",
              "neuroendocrine_like_gamma",
              "basal_stem_progenitor_like_delta",
              "aberrant_basaloid_transitional_like_epsilon",
              "neuronal_or_neural_gene_minor_population_zeta",
              "deuterosomal_ciliated_progenitor_like_eta",
              "hillock_transitional_like_theta",
              "low_quality_doublet_or_debris",
              "no_rare_cell_recovered",
              "not_assessed"
            ]
          },
          {
            "name": "cf_wrong_lineage_marker_called_positive",
            "role": "hard_fail",
            "field": "$.focus_population_marker_calls[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "focus_population_condition_distribution_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "equally_distributed",
              "control_predominant",
              "not_assessed"
            ]
          },
          {
            "name": "focus_population_location_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "airway_only",
              "not_assessed"
            ]
          },
          {
            "name": "cross_axis_canonical_identity_with_airway_only_location_incoherence",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "focus_population_identity": {
                "op": "in",
                "values": [
                  "chemosensory_marker_expressing_population_alpha"
                ]
              },
              "focus_population_location": {
                "op": "in",
                "values": [
                  "airway_only"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-luna-pi",
      "modelName": "gpt-5.6-luna",
      "harness": "pi",
      "task": "gse171524_ectopic_airway_rare_cell",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 1736.198386,
      "cost": 1.5321927000000006,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "focus_population_identity_gate",
            "role": "gate",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "chemosensory_marker_expressing_population_alpha"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "focus_population_marker_calls",
            "kind": "list",
            "matchKey": "marker",
            "k": 3,
            "expected": [
              {
                "key": "POU2F3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "CHAT",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "TRPM5",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "LRMP",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNAT3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "AVIL",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNG13",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              }
            ]
          },
          {
            "name": "focus_population_condition_distribution_gate",
            "role": "gate",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_enriched_with_substantial_control_presence",
              "covid_predominant",
              "covid_only",
              "covid_enriched_trend_not_significant"
            ]
          },
          {
            "name": "focus_population_location_gate",
            "role": "gate",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "ectopic_parenchyma_in_covid",
              "parenchyma_only",
              "bronchiolized_or_peribronchial_parenchyma",
              "airway_and_parenchyma"
            ]
          },
          {
            "name": "focus_population_recovery_mode_gate",
            "role": "gate",
            "field": "$.focus_population_recovery_mode",
            "kind": "value",
            "op": "in",
            "expected": [
              "continuum_not_discrete_cluster"
            ]
          },
          {
            "name": "focus_population_identity_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "ionocyte_like_beta",
              "neuroendocrine_like_gamma",
              "basal_stem_progenitor_like_delta",
              "aberrant_basaloid_transitional_like_epsilon",
              "neuronal_or_neural_gene_minor_population_zeta",
              "deuterosomal_ciliated_progenitor_like_eta",
              "hillock_transitional_like_theta",
              "low_quality_doublet_or_debris",
              "no_rare_cell_recovered",
              "not_assessed"
            ]
          },
          {
            "name": "cf_wrong_lineage_marker_called_positive",
            "role": "hard_fail",
            "field": "$.focus_population_marker_calls[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "focus_population_condition_distribution_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "equally_distributed",
              "control_predominant",
              "not_assessed"
            ]
          },
          {
            "name": "focus_population_location_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "airway_only",
              "not_assessed"
            ]
          },
          {
            "name": "cross_axis_canonical_identity_with_airway_only_location_incoherence",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "focus_population_identity": {
                "op": "in",
                "values": [
                  "chemosensory_marker_expressing_population_alpha"
                ]
              },
              "focus_population_location": {
                "op": "in",
                "values": [
                  "airway_only"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-luna-pi",
      "modelName": "gpt-5.6-luna",
      "harness": "pi",
      "task": "gse171524_ectopic_airway_rare_cell",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1063.51468,
      "cost": 0.5194121,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "focus_population_identity_gate",
            "role": "gate",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "chemosensory_marker_expressing_population_alpha"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "focus_population_marker_calls",
            "kind": "list",
            "matchKey": "marker",
            "k": 3,
            "expected": [
              {
                "key": "POU2F3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "CHAT",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "TRPM5",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "LRMP",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNAT3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "AVIL",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNG13",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              }
            ]
          },
          {
            "name": "focus_population_condition_distribution_gate",
            "role": "gate",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_enriched_with_substantial_control_presence",
              "covid_predominant",
              "covid_only",
              "covid_enriched_trend_not_significant"
            ]
          },
          {
            "name": "focus_population_location_gate",
            "role": "gate",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "ectopic_parenchyma_in_covid",
              "parenchyma_only",
              "bronchiolized_or_peribronchial_parenchyma",
              "airway_and_parenchyma"
            ]
          },
          {
            "name": "focus_population_recovery_mode_gate",
            "role": "gate",
            "field": "$.focus_population_recovery_mode",
            "kind": "value",
            "op": "in",
            "expected": [
              "continuum_not_discrete_cluster"
            ]
          },
          {
            "name": "focus_population_identity_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "ionocyte_like_beta",
              "neuroendocrine_like_gamma",
              "basal_stem_progenitor_like_delta",
              "aberrant_basaloid_transitional_like_epsilon",
              "neuronal_or_neural_gene_minor_population_zeta",
              "deuterosomal_ciliated_progenitor_like_eta",
              "hillock_transitional_like_theta",
              "low_quality_doublet_or_debris",
              "no_rare_cell_recovered",
              "not_assessed"
            ]
          },
          {
            "name": "cf_wrong_lineage_marker_called_positive",
            "role": "hard_fail",
            "field": "$.focus_population_marker_calls[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "focus_population_condition_distribution_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "equally_distributed",
              "control_predominant",
              "not_assessed"
            ]
          },
          {
            "name": "focus_population_location_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "airway_only",
              "not_assessed"
            ]
          },
          {
            "name": "cross_axis_canonical_identity_with_airway_only_location_incoherence",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "focus_population_identity": {
                "op": "in",
                "values": [
                  "chemosensory_marker_expressing_population_alpha"
                ]
              },
              "focus_population_location": {
                "op": "in",
                "values": [
                  "airway_only"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-luna-pi",
      "modelName": "gpt-5.6-luna",
      "harness": "pi",
      "task": "gse171524_pathological_fibroblast_master_regulator_druggable_targets",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 1,
      "passed": true,
      "score": null,
      "durationS": 1220.26078,
      "cost": 0.7464563500000002,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "disease_associated_fibroblast_subtype_weighted",
            "role": "additive",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "mr_tf_junb_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_alpha"
            ]
          },
          {
            "name": "mr_tf_jund_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_beta"
            ]
          },
          {
            "name": "drug_mmp14_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_alpha"
            ]
          },
          {
            "name": "drug_stat3_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_beta"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_weighted",
            "role": "additive",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "disease_associated_fibroblast_subtype_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "value",
            "op": "in",
            "expected": [
              "subtype_lambda",
              "subtype_epsilon",
              "subtype_iota",
              "subtype_beta",
              "subtype_eta",
              "subtype_zeta",
              "subtype_not_assessed"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "simple_de_ranking_by_log_fc",
              "marker_based_de_only",
              "not_computed"
            ]
          },
          {
            "name": "top_master_regulator_transcription_factors_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_not_computed"
            ]
          },
          {
            "name": "top_druggable_proteins_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_not_assessed"
            ]
          },
          {
            "name": "cross_axis_circular_marker_in_druggable_proteins",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_gamma",
              "drug_target_delta",
              "drug_target_eta"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-luna-pi",
      "modelName": "gpt-5.6-luna",
      "harness": "pi",
      "task": "gse171524_pathological_fibroblast_master_regulator_druggable_targets",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 1634.567797,
      "cost": 0.7378578,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "disease_associated_fibroblast_subtype_weighted",
            "role": "additive",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "mr_tf_junb_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_alpha"
            ]
          },
          {
            "name": "mr_tf_jund_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_beta"
            ]
          },
          {
            "name": "drug_mmp14_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_alpha"
            ]
          },
          {
            "name": "drug_stat3_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_beta"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_weighted",
            "role": "additive",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "disease_associated_fibroblast_subtype_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "value",
            "op": "in",
            "expected": [
              "subtype_lambda",
              "subtype_epsilon",
              "subtype_iota",
              "subtype_beta",
              "subtype_eta",
              "subtype_zeta",
              "subtype_not_assessed"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "simple_de_ranking_by_log_fc",
              "marker_based_de_only",
              "not_computed"
            ]
          },
          {
            "name": "top_master_regulator_transcription_factors_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_not_computed"
            ]
          },
          {
            "name": "top_druggable_proteins_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_not_assessed"
            ]
          },
          {
            "name": "cross_axis_circular_marker_in_druggable_proteins",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_gamma",
              "drug_target_delta",
              "drug_target_eta"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-luna-pi",
      "modelName": "gpt-5.6-luna",
      "harness": "pi",
      "task": "gse171524_pathological_fibroblast_master_regulator_druggable_targets",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 10913.54331,
      "cost": 0.6815355000000003,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "disease_associated_fibroblast_subtype_weighted",
            "role": "additive",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "mr_tf_junb_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_alpha"
            ]
          },
          {
            "name": "mr_tf_jund_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_beta"
            ]
          },
          {
            "name": "drug_mmp14_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_alpha"
            ]
          },
          {
            "name": "drug_stat3_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_beta"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_weighted",
            "role": "additive",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "disease_associated_fibroblast_subtype_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "value",
            "op": "in",
            "expected": [
              "subtype_lambda",
              "subtype_epsilon",
              "subtype_iota",
              "subtype_beta",
              "subtype_eta",
              "subtype_zeta",
              "subtype_not_assessed"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "simple_de_ranking_by_log_fc",
              "marker_based_de_only",
              "not_computed"
            ]
          },
          {
            "name": "top_master_regulator_transcription_factors_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_not_computed"
            ]
          },
          {
            "name": "top_druggable_proteins_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_not_assessed"
            ]
          },
          {
            "name": "cross_axis_circular_marker_in_druggable_proteins",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_gamma",
              "drug_target_delta",
              "drug_target_eta"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-luna-pi",
      "modelName": "gpt-5.6-luna",
      "harness": "pi",
      "task": "gse171524_plasma_bcr_neutralizing_clonotype",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 203.612441,
      "cost": 0.29565245,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dominant_pairing_heavy_gate",
            "role": "gate",
            "field": "$.dominant_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV4-34"
            ]
          },
          {
            "name": "dominant_pairing_light_gate",
            "role": "gate",
            "field": "$.dominant_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGLV3-19"
            ]
          },
          {
            "name": "dominant_neutralizer_match_gate",
            "role": "gate",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_known_neutralizer_match"
            ]
          },
          {
            "name": "neutralizer_pairing_heavy_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV1-18"
            ]
          },
          {
            "name": "neutralizer_pairing_light_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGKV3-20"
            ]
          },
          {
            "name": "neutralizer_recurrence_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "observed_in_2_to_5_donors"
            ]
          },
          {
            "name": "neutralizer_rank_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "top_5_pairing",
              "top_6_to_20_pairing"
            ]
          },
          {
            "name": "dominant_clone_isotype_gate",
            "role": "gate",
            "field": "$.dominant_clone_isotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "IgG"
            ]
          },
          {
            "name": "blineage_class_switch_contrast_gate",
            "role": "gate",
            "field": "$.blineage_class_switch_contrast",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_class_switched_control_higher_unswitched",
              "class_switched_in_both_cohorts"
            ]
          },
          {
            "name": "dominant_match_narrative_trap_or_null",
            "role": "hard_fail",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_rbd_neutralizer",
              "matches_known_autoreactive_clone",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_heavy",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_light",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_recurrence_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_rank_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "cross_axis_found_dominant_but_failed_to_scan_for_embedded_neutralizer",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_pairing_neutralizer_match": {
                "op": "in",
                "values": [
                  "no_known_neutralizer_match"
                ]
              },
              "neutralizer_pairing_heavy": {
                "op": "in",
                "values": [
                  "none_detected"
                ]
              }
            }
          },
          {
            "name": "neutralizer_pairing_not_exactly_the_s309_pair",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "op": "not",
            "expected": {
              "op": "and",
              "args": [
                {
                  "op": "field",
                  "name": "neutralizer_pairing_heavy",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGHV1-18"
                    ]
                  }
                },
                {
                  "op": "field",
                  "name": "neutralizer_pairing_light",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGKV3-20"
                    ]
                  }
                }
              ]
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-luna-pi",
      "modelName": "gpt-5.6-luna",
      "harness": "pi",
      "task": "gse171524_plasma_bcr_neutralizing_clonotype",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 338.641991,
      "cost": 0.366556,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dominant_pairing_heavy_gate",
            "role": "gate",
            "field": "$.dominant_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV4-34"
            ]
          },
          {
            "name": "dominant_pairing_light_gate",
            "role": "gate",
            "field": "$.dominant_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGLV3-19"
            ]
          },
          {
            "name": "dominant_neutralizer_match_gate",
            "role": "gate",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_known_neutralizer_match"
            ]
          },
          {
            "name": "neutralizer_pairing_heavy_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV1-18"
            ]
          },
          {
            "name": "neutralizer_pairing_light_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGKV3-20"
            ]
          },
          {
            "name": "neutralizer_recurrence_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "observed_in_2_to_5_donors"
            ]
          },
          {
            "name": "neutralizer_rank_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "top_5_pairing",
              "top_6_to_20_pairing"
            ]
          },
          {
            "name": "dominant_clone_isotype_gate",
            "role": "gate",
            "field": "$.dominant_clone_isotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "IgG"
            ]
          },
          {
            "name": "blineage_class_switch_contrast_gate",
            "role": "gate",
            "field": "$.blineage_class_switch_contrast",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_class_switched_control_higher_unswitched",
              "class_switched_in_both_cohorts"
            ]
          },
          {
            "name": "dominant_match_narrative_trap_or_null",
            "role": "hard_fail",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_rbd_neutralizer",
              "matches_known_autoreactive_clone",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_heavy",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_light",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_recurrence_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_rank_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "cross_axis_found_dominant_but_failed_to_scan_for_embedded_neutralizer",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_pairing_neutralizer_match": {
                "op": "in",
                "values": [
                  "no_known_neutralizer_match"
                ]
              },
              "neutralizer_pairing_heavy": {
                "op": "in",
                "values": [
                  "none_detected"
                ]
              }
            }
          },
          {
            "name": "neutralizer_pairing_not_exactly_the_s309_pair",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "op": "not",
            "expected": {
              "op": "and",
              "args": [
                {
                  "op": "field",
                  "name": "neutralizer_pairing_heavy",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGHV1-18"
                    ]
                  }
                },
                {
                  "op": "field",
                  "name": "neutralizer_pairing_light",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGKV3-20"
                    ]
                  }
                }
              ]
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-luna-pi",
      "modelName": "gpt-5.6-luna",
      "harness": "pi",
      "task": "gse171524_plasma_bcr_neutralizing_clonotype",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 287.2393,
      "cost": 0.30908405,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dominant_pairing_heavy_gate",
            "role": "gate",
            "field": "$.dominant_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV4-34"
            ]
          },
          {
            "name": "dominant_pairing_light_gate",
            "role": "gate",
            "field": "$.dominant_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGLV3-19"
            ]
          },
          {
            "name": "dominant_neutralizer_match_gate",
            "role": "gate",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_known_neutralizer_match"
            ]
          },
          {
            "name": "neutralizer_pairing_heavy_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV1-18"
            ]
          },
          {
            "name": "neutralizer_pairing_light_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGKV3-20"
            ]
          },
          {
            "name": "neutralizer_recurrence_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "observed_in_2_to_5_donors"
            ]
          },
          {
            "name": "neutralizer_rank_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "top_5_pairing",
              "top_6_to_20_pairing"
            ]
          },
          {
            "name": "dominant_clone_isotype_gate",
            "role": "gate",
            "field": "$.dominant_clone_isotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "IgG"
            ]
          },
          {
            "name": "blineage_class_switch_contrast_gate",
            "role": "gate",
            "field": "$.blineage_class_switch_contrast",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_class_switched_control_higher_unswitched",
              "class_switched_in_both_cohorts"
            ]
          },
          {
            "name": "dominant_match_narrative_trap_or_null",
            "role": "hard_fail",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_rbd_neutralizer",
              "matches_known_autoreactive_clone",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_heavy",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_light",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_recurrence_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_rank_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "cross_axis_found_dominant_but_failed_to_scan_for_embedded_neutralizer",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_pairing_neutralizer_match": {
                "op": "in",
                "values": [
                  "no_known_neutralizer_match"
                ]
              },
              "neutralizer_pairing_heavy": {
                "op": "in",
                "values": [
                  "none_detected"
                ]
              }
            }
          },
          {
            "name": "neutralizer_pairing_not_exactly_the_s309_pair",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "op": "not",
            "expected": {
              "op": "and",
              "args": [
                {
                  "op": "field",
                  "name": "neutralizer_pairing_heavy",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGHV1-18"
                    ]
                  }
                },
                {
                  "op": "field",
                  "name": "neutralizer_pairing_light",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGKV3-20"
                    ]
                  }
                }
              ]
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-luna-pi",
      "modelName": "gpt-5.6-luna",
      "harness": "pi",
      "task": "q6_myeloid_subset_restricted_de",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 1,
      "passed": true,
      "score": null,
      "durationS": 372.767733,
      "cost": 0.39879220000000004,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "myeloid_subset_with_strongest_change_gate",
            "role": "gate",
            "field": "$.myeloid_subset_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "subset_gamma"
            ]
          },
          {
            "name": "gene_with_strongest_change_gate",
            "role": "gate",
            "field": "$.gene_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "AXL",
              "PROS1"
            ]
          },
          {
            "name": "direction_of_change_gate",
            "role": "gate",
            "field": "$.direction_of_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "down_in_covid"
            ]
          },
          {
            "name": "biological_function_interpretation_gate",
            "role": "gate",
            "field": "$.biological_function_interpretation",
            "kind": "value",
            "op": "in",
            "expected": [
              "efferocytosis",
              "phagocytosis"
            ]
          },
          {
            "name": "myeloid_compartment_abundance_gate",
            "role": "gate",
            "field": "$.myeloid_compartment_abundance",
            "kind": "value",
            "op": "in",
            "expected": [
              "expands_in_covid"
            ]
          },
          {
            "name": "myeloid_expansion_localization_gate",
            "role": "gate",
            "field": "$.myeloid_expansion_localization",
            "kind": "value",
            "op": "in",
            "expected": [
              "recruited_driven_resident_dysfunction"
            ]
          },
          {
            "name": "dominant_signature_gate",
            "role": "gate",
            "field": "$.dominant_signature",
            "kind": "value",
            "op": "in",
            "expected": [
              "technical_or_agonal_artifact"
            ]
          },
          {
            "name": "dominant_signature_direction_gate",
            "role": "gate",
            "field": "$.dominant_signature_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_in_covid"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-luna-pi",
      "modelName": "gpt-5.6-luna",
      "harness": "pi",
      "task": "q6_myeloid_subset_restricted_de",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 1827.497083,
      "cost": 0.45515955,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "myeloid_subset_with_strongest_change_gate",
            "role": "gate",
            "field": "$.myeloid_subset_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "subset_gamma"
            ]
          },
          {
            "name": "gene_with_strongest_change_gate",
            "role": "gate",
            "field": "$.gene_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "AXL",
              "PROS1"
            ]
          },
          {
            "name": "direction_of_change_gate",
            "role": "gate",
            "field": "$.direction_of_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "down_in_covid"
            ]
          },
          {
            "name": "biological_function_interpretation_gate",
            "role": "gate",
            "field": "$.biological_function_interpretation",
            "kind": "value",
            "op": "in",
            "expected": [
              "efferocytosis",
              "phagocytosis"
            ]
          },
          {
            "name": "myeloid_compartment_abundance_gate",
            "role": "gate",
            "field": "$.myeloid_compartment_abundance",
            "kind": "value",
            "op": "in",
            "expected": [
              "expands_in_covid"
            ]
          },
          {
            "name": "myeloid_expansion_localization_gate",
            "role": "gate",
            "field": "$.myeloid_expansion_localization",
            "kind": "value",
            "op": "in",
            "expected": [
              "recruited_driven_resident_dysfunction"
            ]
          },
          {
            "name": "dominant_signature_gate",
            "role": "gate",
            "field": "$.dominant_signature",
            "kind": "value",
            "op": "in",
            "expected": [
              "technical_or_agonal_artifact"
            ]
          },
          {
            "name": "dominant_signature_direction_gate",
            "role": "gate",
            "field": "$.dominant_signature_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_in_covid"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-luna-pi",
      "modelName": "gpt-5.6-luna",
      "harness": "pi",
      "task": "q6_myeloid_subset_restricted_de",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 675.15908,
      "cost": 0.8615224499999999,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "myeloid_subset_with_strongest_change_gate",
            "role": "gate",
            "field": "$.myeloid_subset_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "subset_gamma"
            ]
          },
          {
            "name": "gene_with_strongest_change_gate",
            "role": "gate",
            "field": "$.gene_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "AXL",
              "PROS1"
            ]
          },
          {
            "name": "direction_of_change_gate",
            "role": "gate",
            "field": "$.direction_of_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "down_in_covid"
            ]
          },
          {
            "name": "biological_function_interpretation_gate",
            "role": "gate",
            "field": "$.biological_function_interpretation",
            "kind": "value",
            "op": "in",
            "expected": [
              "efferocytosis",
              "phagocytosis"
            ]
          },
          {
            "name": "myeloid_compartment_abundance_gate",
            "role": "gate",
            "field": "$.myeloid_compartment_abundance",
            "kind": "value",
            "op": "in",
            "expected": [
              "expands_in_covid"
            ]
          },
          {
            "name": "myeloid_expansion_localization_gate",
            "role": "gate",
            "field": "$.myeloid_expansion_localization",
            "kind": "value",
            "op": "in",
            "expected": [
              "recruited_driven_resident_dysfunction"
            ]
          },
          {
            "name": "dominant_signature_gate",
            "role": "gate",
            "field": "$.dominant_signature",
            "kind": "value",
            "op": "in",
            "expected": [
              "technical_or_agonal_artifact"
            ]
          },
          {
            "name": "dominant_signature_direction_gate",
            "role": "gate",
            "field": "$.dominant_signature_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_in_covid"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-luna-pi",
      "modelName": "gpt-5.6-luna",
      "harness": "pi",
      "task": "snrnaseq_per_donor_composition_shifts_lethal_covid_lung",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 1,
      "passed": true,
      "score": null,
      "durationS": 1937.958747,
      "cost": 0.21126139999999996,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dict_match",
            "role": "gate",
            "field": "cell_type_direction_map",
            "kind": "dict",
            "expected": {
              "Myeloid": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "Epithelia": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Fibroblasts": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "T_NK": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "B_plasma": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Endothelial": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Mast": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "APC": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Neuronal": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              }
            }
          },
          {
            "name": "over_merge_immune_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Immune",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_leukocyte_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Leukocyte",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_lymphoid_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Lymphoid",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_stromal_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Stromal",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-luna-pi",
      "modelName": "gpt-5.6-luna",
      "harness": "pi",
      "task": "snrnaseq_per_donor_composition_shifts_lethal_covid_lung",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 331.699468,
      "cost": 0.17694845,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dict_match",
            "role": "gate",
            "field": "cell_type_direction_map",
            "kind": "dict",
            "expected": {
              "Myeloid": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "Epithelia": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Fibroblasts": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "T_NK": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "B_plasma": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Endothelial": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Mast": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "APC": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Neuronal": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              }
            }
          },
          {
            "name": "over_merge_immune_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Immune",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_leukocyte_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Leukocyte",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_lymphoid_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Lymphoid",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_stromal_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Stromal",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-luna-pi",
      "modelName": "gpt-5.6-luna",
      "harness": "pi",
      "task": "snrnaseq_per_donor_composition_shifts_lethal_covid_lung",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 631.08571,
      "cost": 0.1507813,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dict_match",
            "role": "gate",
            "field": "cell_type_direction_map",
            "kind": "dict",
            "expected": {
              "Myeloid": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "Epithelia": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Fibroblasts": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "T_NK": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "B_plasma": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Endothelial": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Mast": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "APC": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Neuronal": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              }
            }
          },
          {
            "name": "over_merge_immune_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Immune",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_leukocyte_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Leukocyte",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_lymphoid_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Lymphoid",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_stromal_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Stromal",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-luna-pi",
      "modelName": "gpt-5.6-luna",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_enriched_apc_states_v2",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 3075.107329,
      "cost": 1.3369667499999998,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "top_3_cluster_enriched_apc_states",
            "kind": "list",
            "matchKey": "apc_state",
            "k": 3,
            "expected": [
              {
                "key": "plasmacytoid_DC",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "T_cell_stimulatory_role_supported",
                      "role_direction_unresolved"
                    ],
                    "canonical": "T_cell_stimulatory_role_supported"
                  }
                }
              },
              {
                "key": "C1qhigh_lipid_associated_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "C1qhigh_inflammatory_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "plasma_cell",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "cohort_significant_per_patient_inconsistent"
                    ],
                    "canonical": "cohort_significant_per_patient_inconsistent"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CD80_CD86": {
                "op": "in",
                "value": [
                  "mreg_DC",
                  "shared_across_APC_states"
                ]
              },
              "PD_L1_in_APC_clusters": {
                "op": "in",
                "value": [
                  "C1qhigh_macrophage_either_subtype",
                  "shared_across_APC_states",
                  "mreg_DC"
                ]
              },
              "type_I_IFN": {
                "op": "in",
                "value": [
                  "plasmacytoid_DC",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "cf_mreg_dc_claimed_enriched",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "mreg_DC",
              "mature_DC_enriched_in_immunoregulatory_molecules"
            ]
          },
          {
            "name": "cf_framework_offcompartment_or_generic_substitution",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "cDC1",
              "Maier_mregDC",
              "Cheng_C1QC_TAM",
              "Cheng_SPP1_TAM",
              "Cheng_FOLR2_TAM",
              "Villani_pDC",
              "Zhang_inflammatory_monocyte",
              "LAMP3_pos_DC",
              "myeloid_derived_suppressor_cell",
              "MDSC",
              "tumor_associated_neutrophil",
              "mast_cell",
              "APC",
              "antigen_presenting_cell",
              "macrophage",
              "dendritic_cell",
              "tumor_associated_macrophage",
              "TAM"
            ]
          },
          {
            "name": "cf_direction_commit_on_unresolvable_slot",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_not_resolvable_hedge",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].functional_attribution_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_type_I_IFN_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.type_I_IFN",
            "kind": "value",
            "op": "in",
            "expected": [
              "mreg_DC",
              "C1qhigh_macrophage_either_subtype"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-luna-pi",
      "modelName": "gpt-5.6-luna",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_enriched_apc_states_v2",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 2131.462033,
      "cost": 0.9313822500000001,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "top_3_cluster_enriched_apc_states",
            "kind": "list",
            "matchKey": "apc_state",
            "k": 3,
            "expected": [
              {
                "key": "plasmacytoid_DC",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "T_cell_stimulatory_role_supported",
                      "role_direction_unresolved"
                    ],
                    "canonical": "T_cell_stimulatory_role_supported"
                  }
                }
              },
              {
                "key": "C1qhigh_lipid_associated_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "C1qhigh_inflammatory_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "plasma_cell",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "cohort_significant_per_patient_inconsistent"
                    ],
                    "canonical": "cohort_significant_per_patient_inconsistent"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CD80_CD86": {
                "op": "in",
                "value": [
                  "mreg_DC",
                  "shared_across_APC_states"
                ]
              },
              "PD_L1_in_APC_clusters": {
                "op": "in",
                "value": [
                  "C1qhigh_macrophage_either_subtype",
                  "shared_across_APC_states",
                  "mreg_DC"
                ]
              },
              "type_I_IFN": {
                "op": "in",
                "value": [
                  "plasmacytoid_DC",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "cf_mreg_dc_claimed_enriched",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "mreg_DC",
              "mature_DC_enriched_in_immunoregulatory_molecules"
            ]
          },
          {
            "name": "cf_framework_offcompartment_or_generic_substitution",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "cDC1",
              "Maier_mregDC",
              "Cheng_C1QC_TAM",
              "Cheng_SPP1_TAM",
              "Cheng_FOLR2_TAM",
              "Villani_pDC",
              "Zhang_inflammatory_monocyte",
              "LAMP3_pos_DC",
              "myeloid_derived_suppressor_cell",
              "MDSC",
              "tumor_associated_neutrophil",
              "mast_cell",
              "APC",
              "antigen_presenting_cell",
              "macrophage",
              "dendritic_cell",
              "tumor_associated_macrophage",
              "TAM"
            ]
          },
          {
            "name": "cf_direction_commit_on_unresolvable_slot",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_not_resolvable_hedge",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].functional_attribution_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_type_I_IFN_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.type_I_IFN",
            "kind": "value",
            "op": "in",
            "expected": [
              "mreg_DC",
              "C1qhigh_macrophage_either_subtype"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-luna-pi",
      "modelName": "gpt-5.6-luna",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_enriched_apc_states_v2",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1915.185601,
      "cost": 1.3775835,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "top_3_cluster_enriched_apc_states",
            "kind": "list",
            "matchKey": "apc_state",
            "k": 3,
            "expected": [
              {
                "key": "plasmacytoid_DC",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "T_cell_stimulatory_role_supported",
                      "role_direction_unresolved"
                    ],
                    "canonical": "T_cell_stimulatory_role_supported"
                  }
                }
              },
              {
                "key": "C1qhigh_lipid_associated_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "C1qhigh_inflammatory_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "plasma_cell",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "cohort_significant_per_patient_inconsistent"
                    ],
                    "canonical": "cohort_significant_per_patient_inconsistent"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CD80_CD86": {
                "op": "in",
                "value": [
                  "mreg_DC",
                  "shared_across_APC_states"
                ]
              },
              "PD_L1_in_APC_clusters": {
                "op": "in",
                "value": [
                  "C1qhigh_macrophage_either_subtype",
                  "shared_across_APC_states",
                  "mreg_DC"
                ]
              },
              "type_I_IFN": {
                "op": "in",
                "value": [
                  "plasmacytoid_DC",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "cf_mreg_dc_claimed_enriched",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "mreg_DC",
              "mature_DC_enriched_in_immunoregulatory_molecules"
            ]
          },
          {
            "name": "cf_framework_offcompartment_or_generic_substitution",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "cDC1",
              "Maier_mregDC",
              "Cheng_C1QC_TAM",
              "Cheng_SPP1_TAM",
              "Cheng_FOLR2_TAM",
              "Villani_pDC",
              "Zhang_inflammatory_monocyte",
              "LAMP3_pos_DC",
              "myeloid_derived_suppressor_cell",
              "MDSC",
              "tumor_associated_neutrophil",
              "mast_cell",
              "APC",
              "antigen_presenting_cell",
              "macrophage",
              "dendritic_cell",
              "tumor_associated_macrophage",
              "TAM"
            ]
          },
          {
            "name": "cf_direction_commit_on_unresolvable_slot",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_not_resolvable_hedge",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].functional_attribution_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_type_I_IFN_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.type_I_IFN",
            "kind": "value",
            "op": "in",
            "expected": [
              "mreg_DC",
              "C1qhigh_macrophage_either_subtype"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-luna-pi",
      "modelName": "gpt-5.6-luna",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_tumor_state_causation_v5",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 1084.881976,
      "cost": 1.16775605,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "enriched_tumor_states",
            "kind": "list",
            "matchKey": "tumor_state",
            "k": 3,
            "expected": [
              {
                "key": "immune_response",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "direction_unresolved_cross_sectional"
                    ],
                    "canonical": "direction_unresolved_cross_sectional"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "IFN_response_axis",
                      "broad_antigen_presentation_MHC"
                    ],
                    "canonical": "IFN_response_axis"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "enrichment_diminished_by_decontamination",
                      "threshold_sensitive"
                    ],
                    "canonical": "enrichment_diminished_by_decontamination"
                  }
                }
              },
              {
                "key": "stress_hypoxia",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "data_supports_cell_intrinsic_cause"
                    ],
                    "canonical": "data_supports_cell_intrinsic_cause"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "HIF_hypoxia_program_specific"
                    ],
                    "canonical": "HIF_hypoxia_program_specific"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "robust_across_methods",
                      "robust_after_decontamination"
                    ],
                    "canonical": "robust_across_methods"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CXCL9_CXCL10": {
                "op": "in",
                "value": [
                  "immune_response_tumor_state"
                ]
              },
              "CCL5": {
                "op": "in",
                "value": [
                  "shared_across_tumor_states",
                  "partner_lymphocyte_derived",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "gate_conjugate_gate_confounder",
            "role": "gate",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "partner_lymphocyte_capture"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "antigen_presentation_decomposition",
            "kind": "dict",
            "expected": {
              "MHC_class_I": {
                "op": "in",
                "value": [
                  "co_regulated_with_dominant_program",
                  "tumor_intrinsic_enriched_independent"
                ]
              },
              "MHC_class_II": {
                "op": "in",
                "value": [
                  "partner_cell_attributable",
                  "flat_not_enriched",
                  "method_sensitive_no_robust_call",
                  "co_regulated_with_dominant_program"
                ]
              }
            }
          },
          {
            "name": "cf_non_robust_state_claimed_enriched",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "transitory_melanocytic",
              "intermediate_melanocytic",
              "melanocytic",
              "MITF_high",
              "neural_crest_like",
              "mitotic",
              "stress_p53",
              "undifferentiated",
              "dedifferentiated",
              "mesenchymal_tumor",
              "AXL_high_tumor",
              "invasive_tumor",
              "senescence_like",
              "EMT_transition",
              "cancer_stem_cell_like",
              "oxidative_stress_tumor"
            ]
          },
          {
            "name": "cf_framework_generic_or_offtissue_substitution",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "Tsoi_undifferentiated",
              "Tsoi_neural_crest_like",
              "Tsoi_transitory",
              "Tsoi_melanocytic",
              "Rambow_NCSC",
              "Wouters_mesenchymal",
              "Pozniak_TCF4_dependent",
              "tumor_cell",
              "melanoma_cell",
              "malignant_cell",
              "tumor_subpopulation",
              "keratinocyte",
              "hepatocyte",
              "lung_adenocarcinoma_cell",
              "colon_adenocarcinoma_cell",
              "Tirosh_cell_cycle_program",
              "Jerby_Arnon_program",
              "glioblastoma_cell",
              "breast_carcinoma_cell"
            ]
          },
          {
            "name": "cf_immune_response_direction_committed",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_immune_robustness_recall",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_ambiguous_hedge",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].causation_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_stress_generic_stress_response_evidence",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_conjugate_confounder_contamination_denied",
            "role": "hard_fail",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected"
            ]
          },
          {
            "name": "cf_ccl5_recall_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CCL5",
            "kind": "avoid",
            "expected": [
              "immune_response_tumor_state"
            ]
          },
          {
            "name": "cf_cxcl9_cxcl10_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CXCL9_CXCL10",
            "kind": "avoid",
            "expected": [
              "stress_hypoxia_tumor_state"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-luna-pi",
      "modelName": "gpt-5.6-luna",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_tumor_state_causation_v5",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 1133.427197,
      "cost": 0.8196834999999998,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "enriched_tumor_states",
            "kind": "list",
            "matchKey": "tumor_state",
            "k": 3,
            "expected": [
              {
                "key": "immune_response",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "direction_unresolved_cross_sectional"
                    ],
                    "canonical": "direction_unresolved_cross_sectional"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "IFN_response_axis",
                      "broad_antigen_presentation_MHC"
                    ],
                    "canonical": "IFN_response_axis"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "enrichment_diminished_by_decontamination",
                      "threshold_sensitive"
                    ],
                    "canonical": "enrichment_diminished_by_decontamination"
                  }
                }
              },
              {
                "key": "stress_hypoxia",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "data_supports_cell_intrinsic_cause"
                    ],
                    "canonical": "data_supports_cell_intrinsic_cause"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "HIF_hypoxia_program_specific"
                    ],
                    "canonical": "HIF_hypoxia_program_specific"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "robust_across_methods",
                      "robust_after_decontamination"
                    ],
                    "canonical": "robust_across_methods"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CXCL9_CXCL10": {
                "op": "in",
                "value": [
                  "immune_response_tumor_state"
                ]
              },
              "CCL5": {
                "op": "in",
                "value": [
                  "shared_across_tumor_states",
                  "partner_lymphocyte_derived",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "gate_conjugate_gate_confounder",
            "role": "gate",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "partner_lymphocyte_capture"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "antigen_presentation_decomposition",
            "kind": "dict",
            "expected": {
              "MHC_class_I": {
                "op": "in",
                "value": [
                  "co_regulated_with_dominant_program",
                  "tumor_intrinsic_enriched_independent"
                ]
              },
              "MHC_class_II": {
                "op": "in",
                "value": [
                  "partner_cell_attributable",
                  "flat_not_enriched",
                  "method_sensitive_no_robust_call",
                  "co_regulated_with_dominant_program"
                ]
              }
            }
          },
          {
            "name": "cf_non_robust_state_claimed_enriched",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "transitory_melanocytic",
              "intermediate_melanocytic",
              "melanocytic",
              "MITF_high",
              "neural_crest_like",
              "mitotic",
              "stress_p53",
              "undifferentiated",
              "dedifferentiated",
              "mesenchymal_tumor",
              "AXL_high_tumor",
              "invasive_tumor",
              "senescence_like",
              "EMT_transition",
              "cancer_stem_cell_like",
              "oxidative_stress_tumor"
            ]
          },
          {
            "name": "cf_framework_generic_or_offtissue_substitution",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "Tsoi_undifferentiated",
              "Tsoi_neural_crest_like",
              "Tsoi_transitory",
              "Tsoi_melanocytic",
              "Rambow_NCSC",
              "Wouters_mesenchymal",
              "Pozniak_TCF4_dependent",
              "tumor_cell",
              "melanoma_cell",
              "malignant_cell",
              "tumor_subpopulation",
              "keratinocyte",
              "hepatocyte",
              "lung_adenocarcinoma_cell",
              "colon_adenocarcinoma_cell",
              "Tirosh_cell_cycle_program",
              "Jerby_Arnon_program",
              "glioblastoma_cell",
              "breast_carcinoma_cell"
            ]
          },
          {
            "name": "cf_immune_response_direction_committed",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_immune_robustness_recall",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_ambiguous_hedge",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].causation_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_stress_generic_stress_response_evidence",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_conjugate_confounder_contamination_denied",
            "role": "hard_fail",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected"
            ]
          },
          {
            "name": "cf_ccl5_recall_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CCL5",
            "kind": "avoid",
            "expected": [
              "immune_response_tumor_state"
            ]
          },
          {
            "name": "cf_cxcl9_cxcl10_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CXCL9_CXCL10",
            "kind": "avoid",
            "expected": [
              "stress_hypoxia_tumor_state"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-luna-pi",
      "modelName": "gpt-5.6-luna",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_tumor_state_causation_v5",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 13230.304014,
      "cost": 1.4211263500000002,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "enriched_tumor_states",
            "kind": "list",
            "matchKey": "tumor_state",
            "k": 3,
            "expected": [
              {
                "key": "immune_response",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "direction_unresolved_cross_sectional"
                    ],
                    "canonical": "direction_unresolved_cross_sectional"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "IFN_response_axis",
                      "broad_antigen_presentation_MHC"
                    ],
                    "canonical": "IFN_response_axis"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "enrichment_diminished_by_decontamination",
                      "threshold_sensitive"
                    ],
                    "canonical": "enrichment_diminished_by_decontamination"
                  }
                }
              },
              {
                "key": "stress_hypoxia",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "data_supports_cell_intrinsic_cause"
                    ],
                    "canonical": "data_supports_cell_intrinsic_cause"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "HIF_hypoxia_program_specific"
                    ],
                    "canonical": "HIF_hypoxia_program_specific"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "robust_across_methods",
                      "robust_after_decontamination"
                    ],
                    "canonical": "robust_across_methods"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CXCL9_CXCL10": {
                "op": "in",
                "value": [
                  "immune_response_tumor_state"
                ]
              },
              "CCL5": {
                "op": "in",
                "value": [
                  "shared_across_tumor_states",
                  "partner_lymphocyte_derived",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "gate_conjugate_gate_confounder",
            "role": "gate",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "partner_lymphocyte_capture"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "antigen_presentation_decomposition",
            "kind": "dict",
            "expected": {
              "MHC_class_I": {
                "op": "in",
                "value": [
                  "co_regulated_with_dominant_program",
                  "tumor_intrinsic_enriched_independent"
                ]
              },
              "MHC_class_II": {
                "op": "in",
                "value": [
                  "partner_cell_attributable",
                  "flat_not_enriched",
                  "method_sensitive_no_robust_call",
                  "co_regulated_with_dominant_program"
                ]
              }
            }
          },
          {
            "name": "cf_non_robust_state_claimed_enriched",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "transitory_melanocytic",
              "intermediate_melanocytic",
              "melanocytic",
              "MITF_high",
              "neural_crest_like",
              "mitotic",
              "stress_p53",
              "undifferentiated",
              "dedifferentiated",
              "mesenchymal_tumor",
              "AXL_high_tumor",
              "invasive_tumor",
              "senescence_like",
              "EMT_transition",
              "cancer_stem_cell_like",
              "oxidative_stress_tumor"
            ]
          },
          {
            "name": "cf_framework_generic_or_offtissue_substitution",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "Tsoi_undifferentiated",
              "Tsoi_neural_crest_like",
              "Tsoi_transitory",
              "Tsoi_melanocytic",
              "Rambow_NCSC",
              "Wouters_mesenchymal",
              "Pozniak_TCF4_dependent",
              "tumor_cell",
              "melanoma_cell",
              "malignant_cell",
              "tumor_subpopulation",
              "keratinocyte",
              "hepatocyte",
              "lung_adenocarcinoma_cell",
              "colon_adenocarcinoma_cell",
              "Tirosh_cell_cycle_program",
              "Jerby_Arnon_program",
              "glioblastoma_cell",
              "breast_carcinoma_cell"
            ]
          },
          {
            "name": "cf_immune_response_direction_committed",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_immune_robustness_recall",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_ambiguous_hedge",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].causation_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_stress_generic_stress_response_evidence",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_conjugate_confounder_contamination_denied",
            "role": "hard_fail",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected"
            ]
          },
          {
            "name": "cf_ccl5_recall_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CCL5",
            "kind": "avoid",
            "expected": [
              "immune_response_tumor_state"
            ]
          },
          {
            "name": "cf_cxcl9_cxcl10_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CXCL9_CXCL10",
            "kind": "avoid",
            "expected": [
              "stress_hypoxia_tumor_state"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-luna-pi",
      "modelName": "gpt-5.6-luna",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_vs_cd39_tumor_reactivity_dominance",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 851.072932,
      "cost": 0.6047683,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "population_findings",
            "kind": "list",
            "matchKey": "capture_gate",
            "k": 4,
            "expected": [
              {
                "key": "tumor_cell_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "apc_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "combined_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "unselected_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "naive",
                      "central_memory",
                      "effector_memory"
                    ],
                    "canonical": "naive"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "low"
                    ],
                    "canonical": "low"
                  },
                  "purity_caveat": {
                    "op": "equals",
                    "values": [
                      "pure_T_cell"
                    ],
                    "canonical": "pure_T_cell"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "no_reactivity_evidence"
                    ],
                    "canonical": "no_reactivity_evidence"
                  }
                }
              },
              {
                "key": "marker_sorted_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "equals",
                    "values": [
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "terminal_exhausted_LAG3_high"
                  },
                  "clonal_expansion_level": {
                    "op": "in",
                    "values": [
                      "high",
                      "intermediate"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "marker_on_non_T_lineages",
                      "single_cell_gating_loses_target"
                    ],
                    "canonical": "marker_on_non_T_lineages"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "marker_or_exhaustion_defined_only"
                    ],
                    "canonical": "marker_or_exhaustion_defined_only"
                  }
                }
              }
            ]
          },
          {
            "name": "best_capture_approach_gate",
            "role": "gate",
            "field": "best_capture_approach",
            "kind": "value",
            "expected": "heterotypic_cell_conjugate"
          },
          {
            "name": "best_selection_marker_gate",
            "role": "gate",
            "field": "best_selection_marker",
            "kind": "value",
            "expected": "none"
          },
          {
            "name": "reactivity_criterion_gate",
            "role": "gate",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "clonal_expansion_and_physical_engagement"
          },
          {
            "name": "cross_patient_robustness_gate",
            "role": "gate",
            "field": "cross_patient_robustness",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_across_patients",
              "marginal_in_some_patients"
            ]
          },
          {
            "name": "marker_selection_consequence_gate",
            "role": "gate",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "discards_clonally_expanded_reactive_cells"
          },
          {
            "name": "superiority_claim_basis_gate",
            "role": "gate",
            "field": "superiority_claim_basis",
            "kind": "value",
            "expected": "requires_functional_reactivity_assay"
          },
          {
            "name": "conjugate_enriched_state_gate",
            "role": "gate",
            "field": "conjugate_enriched_state",
            "kind": "value",
            "expected": "progenitor_exhausted_TCF7_pos_stem_like"
          },
          {
            "name": "cf_circular_exhaustion_criterion",
            "role": "hard_fail",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "exhaustion_signature_intensity"
          },
          {
            "name": "cf_marker_sort_prior",
            "role": "hard_fail",
            "field": "best_selection_marker",
            "kind": "value",
            "op": "in",
            "expected": [
              "CD39",
              "PD1",
              "PD1_high",
              "TIM3",
              "PD1_CD39",
              "CD39_CD103"
            ]
          },
          {
            "name": "cf_single_cell_capture_prior",
            "role": "hard_fail",
            "field": "best_capture_approach",
            "kind": "value",
            "op": "in",
            "expected": [
              "dissociated_single_cell",
              "dissociated_bulk_population"
            ]
          },
          {
            "name": "cf_off_data_capture_gate",
            "role": "hard_fail",
            "field": "$.population_findings[*].capture_gate",
            "kind": "avoid",
            "expected": [
              "dissociated_bulk",
              "matched_peripheral_blood",
              "matched_lymph_node",
              "post_hoc_expanded_clonotype_group"
            ]
          },
          {
            "name": "cf_conjugate_mischaracterized",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_marker_singlet_called_engaged",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "contains",
            "matchKey": "capture_gate",
            "entity": "marker_sorted_singlet",
            "valueField": "reactivity_evidence",
            "expected": [
              "engagement_and_clonal_expansion"
            ]
          },
          {
            "name": "cf_no_engaged_reactive_population",
            "role": "hard_fail",
            "field": "$.population_findings[*].reactivity_evidence",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_marker_captures_all_reactive",
            "role": "hard_fail",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "captures_full_reactive_repertoire"
          },
          {
            "name": "cf_superiority_overclaimed_from_phenotype",
            "role": "hard_fail",
            "field": "superiority_claim_basis",
            "kind": "value",
            "op": "in",
            "expected": [
              "established_by_scrna_phenotype",
              "established_by_reactivity_proxy_scores"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-luna-pi",
      "modelName": "gpt-5.6-luna",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_vs_cd39_tumor_reactivity_dominance",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 512.713391,
      "cost": 0.48419130000000005,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "population_findings",
            "kind": "list",
            "matchKey": "capture_gate",
            "k": 4,
            "expected": [
              {
                "key": "tumor_cell_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "apc_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "combined_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "unselected_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "naive",
                      "central_memory",
                      "effector_memory"
                    ],
                    "canonical": "naive"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "low"
                    ],
                    "canonical": "low"
                  },
                  "purity_caveat": {
                    "op": "equals",
                    "values": [
                      "pure_T_cell"
                    ],
                    "canonical": "pure_T_cell"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "no_reactivity_evidence"
                    ],
                    "canonical": "no_reactivity_evidence"
                  }
                }
              },
              {
                "key": "marker_sorted_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "equals",
                    "values": [
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "terminal_exhausted_LAG3_high"
                  },
                  "clonal_expansion_level": {
                    "op": "in",
                    "values": [
                      "high",
                      "intermediate"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "marker_on_non_T_lineages",
                      "single_cell_gating_loses_target"
                    ],
                    "canonical": "marker_on_non_T_lineages"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "marker_or_exhaustion_defined_only"
                    ],
                    "canonical": "marker_or_exhaustion_defined_only"
                  }
                }
              }
            ]
          },
          {
            "name": "best_capture_approach_gate",
            "role": "gate",
            "field": "best_capture_approach",
            "kind": "value",
            "expected": "heterotypic_cell_conjugate"
          },
          {
            "name": "best_selection_marker_gate",
            "role": "gate",
            "field": "best_selection_marker",
            "kind": "value",
            "expected": "none"
          },
          {
            "name": "reactivity_criterion_gate",
            "role": "gate",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "clonal_expansion_and_physical_engagement"
          },
          {
            "name": "cross_patient_robustness_gate",
            "role": "gate",
            "field": "cross_patient_robustness",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_across_patients",
              "marginal_in_some_patients"
            ]
          },
          {
            "name": "marker_selection_consequence_gate",
            "role": "gate",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "discards_clonally_expanded_reactive_cells"
          },
          {
            "name": "superiority_claim_basis_gate",
            "role": "gate",
            "field": "superiority_claim_basis",
            "kind": "value",
            "expected": "requires_functional_reactivity_assay"
          },
          {
            "name": "conjugate_enriched_state_gate",
            "role": "gate",
            "field": "conjugate_enriched_state",
            "kind": "value",
            "expected": "progenitor_exhausted_TCF7_pos_stem_like"
          },
          {
            "name": "cf_circular_exhaustion_criterion",
            "role": "hard_fail",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "exhaustion_signature_intensity"
          },
          {
            "name": "cf_marker_sort_prior",
            "role": "hard_fail",
            "field": "best_selection_marker",
            "kind": "value",
            "op": "in",
            "expected": [
              "CD39",
              "PD1",
              "PD1_high",
              "TIM3",
              "PD1_CD39",
              "CD39_CD103"
            ]
          },
          {
            "name": "cf_single_cell_capture_prior",
            "role": "hard_fail",
            "field": "best_capture_approach",
            "kind": "value",
            "op": "in",
            "expected": [
              "dissociated_single_cell",
              "dissociated_bulk_population"
            ]
          },
          {
            "name": "cf_off_data_capture_gate",
            "role": "hard_fail",
            "field": "$.population_findings[*].capture_gate",
            "kind": "avoid",
            "expected": [
              "dissociated_bulk",
              "matched_peripheral_blood",
              "matched_lymph_node",
              "post_hoc_expanded_clonotype_group"
            ]
          },
          {
            "name": "cf_conjugate_mischaracterized",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_marker_singlet_called_engaged",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "contains",
            "matchKey": "capture_gate",
            "entity": "marker_sorted_singlet",
            "valueField": "reactivity_evidence",
            "expected": [
              "engagement_and_clonal_expansion"
            ]
          },
          {
            "name": "cf_no_engaged_reactive_population",
            "role": "hard_fail",
            "field": "$.population_findings[*].reactivity_evidence",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_marker_captures_all_reactive",
            "role": "hard_fail",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "captures_full_reactive_repertoire"
          },
          {
            "name": "cf_superiority_overclaimed_from_phenotype",
            "role": "hard_fail",
            "field": "superiority_claim_basis",
            "kind": "value",
            "op": "in",
            "expected": [
              "established_by_scrna_phenotype",
              "established_by_reactivity_proxy_scores"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-luna-pi",
      "modelName": "gpt-5.6-luna",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_vs_cd39_tumor_reactivity_dominance",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 507.156207,
      "cost": 0.34439960000000003,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "population_findings",
            "kind": "list",
            "matchKey": "capture_gate",
            "k": 4,
            "expected": [
              {
                "key": "tumor_cell_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "apc_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "combined_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "unselected_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "naive",
                      "central_memory",
                      "effector_memory"
                    ],
                    "canonical": "naive"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "low"
                    ],
                    "canonical": "low"
                  },
                  "purity_caveat": {
                    "op": "equals",
                    "values": [
                      "pure_T_cell"
                    ],
                    "canonical": "pure_T_cell"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "no_reactivity_evidence"
                    ],
                    "canonical": "no_reactivity_evidence"
                  }
                }
              },
              {
                "key": "marker_sorted_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "equals",
                    "values": [
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "terminal_exhausted_LAG3_high"
                  },
                  "clonal_expansion_level": {
                    "op": "in",
                    "values": [
                      "high",
                      "intermediate"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "marker_on_non_T_lineages",
                      "single_cell_gating_loses_target"
                    ],
                    "canonical": "marker_on_non_T_lineages"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "marker_or_exhaustion_defined_only"
                    ],
                    "canonical": "marker_or_exhaustion_defined_only"
                  }
                }
              }
            ]
          },
          {
            "name": "best_capture_approach_gate",
            "role": "gate",
            "field": "best_capture_approach",
            "kind": "value",
            "expected": "heterotypic_cell_conjugate"
          },
          {
            "name": "best_selection_marker_gate",
            "role": "gate",
            "field": "best_selection_marker",
            "kind": "value",
            "expected": "none"
          },
          {
            "name": "reactivity_criterion_gate",
            "role": "gate",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "clonal_expansion_and_physical_engagement"
          },
          {
            "name": "cross_patient_robustness_gate",
            "role": "gate",
            "field": "cross_patient_robustness",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_across_patients",
              "marginal_in_some_patients"
            ]
          },
          {
            "name": "marker_selection_consequence_gate",
            "role": "gate",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "discards_clonally_expanded_reactive_cells"
          },
          {
            "name": "superiority_claim_basis_gate",
            "role": "gate",
            "field": "superiority_claim_basis",
            "kind": "value",
            "expected": "requires_functional_reactivity_assay"
          },
          {
            "name": "conjugate_enriched_state_gate",
            "role": "gate",
            "field": "conjugate_enriched_state",
            "kind": "value",
            "expected": "progenitor_exhausted_TCF7_pos_stem_like"
          },
          {
            "name": "cf_circular_exhaustion_criterion",
            "role": "hard_fail",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "exhaustion_signature_intensity"
          },
          {
            "name": "cf_marker_sort_prior",
            "role": "hard_fail",
            "field": "best_selection_marker",
            "kind": "value",
            "op": "in",
            "expected": [
              "CD39",
              "PD1",
              "PD1_high",
              "TIM3",
              "PD1_CD39",
              "CD39_CD103"
            ]
          },
          {
            "name": "cf_single_cell_capture_prior",
            "role": "hard_fail",
            "field": "best_capture_approach",
            "kind": "value",
            "op": "in",
            "expected": [
              "dissociated_single_cell",
              "dissociated_bulk_population"
            ]
          },
          {
            "name": "cf_off_data_capture_gate",
            "role": "hard_fail",
            "field": "$.population_findings[*].capture_gate",
            "kind": "avoid",
            "expected": [
              "dissociated_bulk",
              "matched_peripheral_blood",
              "matched_lymph_node",
              "post_hoc_expanded_clonotype_group"
            ]
          },
          {
            "name": "cf_conjugate_mischaracterized",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_marker_singlet_called_engaged",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "contains",
            "matchKey": "capture_gate",
            "entity": "marker_sorted_singlet",
            "valueField": "reactivity_evidence",
            "expected": [
              "engagement_and_clonal_expansion"
            ]
          },
          {
            "name": "cf_no_engaged_reactive_population",
            "role": "hard_fail",
            "field": "$.population_findings[*].reactivity_evidence",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_marker_captures_all_reactive",
            "role": "hard_fail",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "captures_full_reactive_repertoire"
          },
          {
            "name": "cf_superiority_overclaimed_from_phenotype",
            "role": "hard_fail",
            "field": "superiority_claim_basis",
            "kind": "value",
            "op": "in",
            "expected": [
              "established_by_scrna_phenotype",
              "established_by_reactivity_proxy_scores"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-luna-pi",
      "modelName": "gpt-5.6-luna",
      "harness": "pi",
      "task": "10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 1,
      "passed": true,
      "score": null,
      "durationS": 1742.417702,
      "cost": 0.7735782500000001,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_singlet_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "TOXhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "LAG3high_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "GZMKhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "intermediate_exhausted_gzmk"
                    ],
                    "canonical": "intermediate_exhausted_gzmk"
                  }
                }
              },
              {
                "key": "TCF7_stem_like_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "stem_like_exhausted"
                    ],
                    "canonical": "stem_like_exhausted"
                  }
                }
              },
              {
                "key": "MKI67_pos_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              },
              {
                "key": "MKI67high_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_tumor_cluster_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "Tn",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "Tn_Tmem",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "early_Tem_GZMK",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              },
              {
                "key": "Tem_GZMH",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              }
            ]
          },
          {
            "name": "clonotype_expansion_direction_data_true",
            "role": "gate",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_substantially_more_expanded_than_cluster",
              "singlet_modestly_more_expanded_than_cluster"
            ]
          },
          {
            "name": "biological_inference_data_true",
            "role": "gate",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_antigen_experienced_cluster_bystander"
            ]
          },
          {
            "name": "singlet_top3_contains_naive_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Tn",
              "Tn_Tmem",
              "early_Tem_GZMK",
              "Tem_GZMH"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "TOXhigh_Tex",
              "LAG3high_Tex",
              "GZMKhigh_Tex",
              "TCF7_stem_like_Tex",
              "MKI67_pos_Tex_Tprol",
              "MKI67high_Tex_Tprol"
            ]
          },
          {
            "name": "singlet_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "TOXhigh_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "TCF7_stem_like_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "MKI67_state_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "Tn_or_Tn_Tmem_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_stemlike_precursor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_reported_canonical_expansion_direction",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_substantially_more_expanded_than_singlet",
              "cluster_modestly_more_expanded_than_singlet"
            ]
          },
          {
            "name": "cf_expansion_lazy_defer",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "expansion_pattern_not_detectable"
            ]
          },
          {
            "name": "cf_reported_canonical_expected_biology",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_antigen_experienced_singlet_bystander"
            ]
          },
          {
            "name": "cf_biological_inference_null_artifact_or_lineage",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "differences_reflect_dissociation_or_sorting_artifact",
              "cluster_and_singlet_capture_same_biology",
              "cluster_and_singlet_capture_distinct_cd8_lineages"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "biological_relations",
            "kind": "dict",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_greater_than",
                  "greater_than"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "less_than",
                  "much_less_than"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than",
                  "approximately_equal"
                ]
              },
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal"
                ]
              },
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal",
                  "greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_exhaustion_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_naive_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_proliferation_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_expansion_richness_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_multichain_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_intermediate_gzmk_overgeneralized",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_melanocytic_conjugate_overcall",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "values": [
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_relations_mass_hedge",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-luna-pi",
      "modelName": "gpt-5.6-luna",
      "harness": "pi",
      "task": "10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 2,
      "passed": true,
      "score": null,
      "durationS": 1275.132117,
      "cost": 0.9084919000000002,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_singlet_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "TOXhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "LAG3high_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "GZMKhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "intermediate_exhausted_gzmk"
                    ],
                    "canonical": "intermediate_exhausted_gzmk"
                  }
                }
              },
              {
                "key": "TCF7_stem_like_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "stem_like_exhausted"
                    ],
                    "canonical": "stem_like_exhausted"
                  }
                }
              },
              {
                "key": "MKI67_pos_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              },
              {
                "key": "MKI67high_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_tumor_cluster_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "Tn",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "Tn_Tmem",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "early_Tem_GZMK",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              },
              {
                "key": "Tem_GZMH",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              }
            ]
          },
          {
            "name": "clonotype_expansion_direction_data_true",
            "role": "gate",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_substantially_more_expanded_than_cluster",
              "singlet_modestly_more_expanded_than_cluster"
            ]
          },
          {
            "name": "biological_inference_data_true",
            "role": "gate",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_antigen_experienced_cluster_bystander"
            ]
          },
          {
            "name": "singlet_top3_contains_naive_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Tn",
              "Tn_Tmem",
              "early_Tem_GZMK",
              "Tem_GZMH"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "TOXhigh_Tex",
              "LAG3high_Tex",
              "GZMKhigh_Tex",
              "TCF7_stem_like_Tex",
              "MKI67_pos_Tex_Tprol",
              "MKI67high_Tex_Tprol"
            ]
          },
          {
            "name": "singlet_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "TOXhigh_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "TCF7_stem_like_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "MKI67_state_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "Tn_or_Tn_Tmem_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_stemlike_precursor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_reported_canonical_expansion_direction",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_substantially_more_expanded_than_singlet",
              "cluster_modestly_more_expanded_than_singlet"
            ]
          },
          {
            "name": "cf_expansion_lazy_defer",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "expansion_pattern_not_detectable"
            ]
          },
          {
            "name": "cf_reported_canonical_expected_biology",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_antigen_experienced_singlet_bystander"
            ]
          },
          {
            "name": "cf_biological_inference_null_artifact_or_lineage",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "differences_reflect_dissociation_or_sorting_artifact",
              "cluster_and_singlet_capture_same_biology",
              "cluster_and_singlet_capture_distinct_cd8_lineages"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "biological_relations",
            "kind": "dict",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_greater_than",
                  "greater_than"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "less_than",
                  "much_less_than"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than",
                  "approximately_equal"
                ]
              },
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal"
                ]
              },
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal",
                  "greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_exhaustion_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_naive_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_proliferation_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_expansion_richness_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_multichain_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_intermediate_gzmk_overgeneralized",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_melanocytic_conjugate_overcall",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "values": [
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_relations_mass_hedge",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-luna-pi",
      "modelName": "gpt-5.6-luna",
      "harness": "pi",
      "task": "10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 717.972636,
      "cost": 0.3869037,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_singlet_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "TOXhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "LAG3high_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "GZMKhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "intermediate_exhausted_gzmk"
                    ],
                    "canonical": "intermediate_exhausted_gzmk"
                  }
                }
              },
              {
                "key": "TCF7_stem_like_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "stem_like_exhausted"
                    ],
                    "canonical": "stem_like_exhausted"
                  }
                }
              },
              {
                "key": "MKI67_pos_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              },
              {
                "key": "MKI67high_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_tumor_cluster_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "Tn",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "Tn_Tmem",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "early_Tem_GZMK",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              },
              {
                "key": "Tem_GZMH",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              }
            ]
          },
          {
            "name": "clonotype_expansion_direction_data_true",
            "role": "gate",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_substantially_more_expanded_than_cluster",
              "singlet_modestly_more_expanded_than_cluster"
            ]
          },
          {
            "name": "biological_inference_data_true",
            "role": "gate",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_antigen_experienced_cluster_bystander"
            ]
          },
          {
            "name": "singlet_top3_contains_naive_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Tn",
              "Tn_Tmem",
              "early_Tem_GZMK",
              "Tem_GZMH"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "TOXhigh_Tex",
              "LAG3high_Tex",
              "GZMKhigh_Tex",
              "TCF7_stem_like_Tex",
              "MKI67_pos_Tex_Tprol",
              "MKI67high_Tex_Tprol"
            ]
          },
          {
            "name": "singlet_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "TOXhigh_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "TCF7_stem_like_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "MKI67_state_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "Tn_or_Tn_Tmem_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_stemlike_precursor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_reported_canonical_expansion_direction",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_substantially_more_expanded_than_singlet",
              "cluster_modestly_more_expanded_than_singlet"
            ]
          },
          {
            "name": "cf_expansion_lazy_defer",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "expansion_pattern_not_detectable"
            ]
          },
          {
            "name": "cf_reported_canonical_expected_biology",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_antigen_experienced_singlet_bystander"
            ]
          },
          {
            "name": "cf_biological_inference_null_artifact_or_lineage",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "differences_reflect_dissociation_or_sorting_artifact",
              "cluster_and_singlet_capture_same_biology",
              "cluster_and_singlet_capture_distinct_cd8_lineages"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "biological_relations",
            "kind": "dict",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_greater_than",
                  "greater_than"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "less_than",
                  "much_less_than"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than",
                  "approximately_equal"
                ]
              },
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal"
                ]
              },
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal",
                  "greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_exhaustion_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_naive_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_proliferation_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_expansion_richness_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_multichain_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_intermediate_gzmk_overgeneralized",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_melanocytic_conjugate_overcall",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "values": [
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_relations_mass_hedge",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-luna-pi",
      "modelName": "gpt-5.6-luna",
      "harness": "pi",
      "task": "10x_5p_melanoma_tcr_matched_apc_vs_tumor_exhaustion",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 1,
      "passed": true,
      "score": null,
      "durationS": 832.706528,
      "cost": 0.6054970500000001,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "state_discriminating_genes",
            "kind": "list",
            "matchKey": "gene",
            "k": 3,
            "expected": [
              {
                "key": "CTLA4",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TOX",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "ENTPD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "HAVCR2",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TIGIT",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "PDCD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "difference_attributable_to",
            "role": "gate",
            "field": "difference_attributable_to",
            "kind": "value",
            "expected": "covaries_with_apc_conjugate_state"
          },
          {
            "name": "causal_status",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "direction_unresolved_selection_confound"
          },
          {
            "name": "functional_interpretation",
            "role": "gate",
            "field": "functional_interpretation",
            "kind": "value",
            "expected": "costimulation_signature_function_unresolved"
          },
          {
            "name": "cf_contamination_gene_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "MLANA",
              "PMEL",
              "DCT",
              "TYR",
              "TYRP1",
              "SOX10",
              "MITF",
              "MAGEA4"
            ]
          },
          {
            "name": "cf_apc_partner_contamination_gene",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "HLA-DRA",
              "CD68",
              "LYZ",
              "CD14",
              "C1QA",
              "ITGAX",
              "CD1C"
            ]
          },
          {
            "name": "cf_ligand_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "CD274",
              "PDCD1LG2",
              "CD80",
              "CD86"
            ]
          },
          {
            "name": "cf_attribution_causal_or_baseline",
            "role": "hard_fail",
            "field": "$.difference_attributable_to",
            "kind": "avoid",
            "expected": [
              "apc_engagement",
              "tumor_cell_engagement",
              "clonotype_intrinsic_baseline"
            ]
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "engagement_induces_exhausted_state",
              "pre_exhausted_cells_select_into_apc_conjugates"
            ]
          },
          {
            "name": "cf_causal_fabrication",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "resolved_by_pseudotemporal_ordering",
              "resolved_by_perturbation_experiment"
            ]
          },
          {
            "name": "cf_functional_overclaim",
            "role": "hard_fail",
            "field": "$.functional_interpretation",
            "kind": "avoid",
            "expected": [
              "costimulation_licensed_reinvigoration_competent",
              "costimulation_licensed_checkpoint_responsive",
              "tumor_driven_terminal_exhaustion"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-luna-pi",
      "modelName": "gpt-5.6-luna",
      "harness": "pi",
      "task": "10x_5p_melanoma_tcr_matched_apc_vs_tumor_exhaustion",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 2275.633354,
      "cost": 0.47489054999999997,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "state_discriminating_genes",
            "kind": "list",
            "matchKey": "gene",
            "k": 3,
            "expected": [
              {
                "key": "CTLA4",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TOX",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "ENTPD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "HAVCR2",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TIGIT",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "PDCD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "difference_attributable_to",
            "role": "gate",
            "field": "difference_attributable_to",
            "kind": "value",
            "expected": "covaries_with_apc_conjugate_state"
          },
          {
            "name": "causal_status",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "direction_unresolved_selection_confound"
          },
          {
            "name": "functional_interpretation",
            "role": "gate",
            "field": "functional_interpretation",
            "kind": "value",
            "expected": "costimulation_signature_function_unresolved"
          },
          {
            "name": "cf_contamination_gene_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "MLANA",
              "PMEL",
              "DCT",
              "TYR",
              "TYRP1",
              "SOX10",
              "MITF",
              "MAGEA4"
            ]
          },
          {
            "name": "cf_apc_partner_contamination_gene",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "HLA-DRA",
              "CD68",
              "LYZ",
              "CD14",
              "C1QA",
              "ITGAX",
              "CD1C"
            ]
          },
          {
            "name": "cf_ligand_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "CD274",
              "PDCD1LG2",
              "CD80",
              "CD86"
            ]
          },
          {
            "name": "cf_attribution_causal_or_baseline",
            "role": "hard_fail",
            "field": "$.difference_attributable_to",
            "kind": "avoid",
            "expected": [
              "apc_engagement",
              "tumor_cell_engagement",
              "clonotype_intrinsic_baseline"
            ]
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "engagement_induces_exhausted_state",
              "pre_exhausted_cells_select_into_apc_conjugates"
            ]
          },
          {
            "name": "cf_causal_fabrication",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "resolved_by_pseudotemporal_ordering",
              "resolved_by_perturbation_experiment"
            ]
          },
          {
            "name": "cf_functional_overclaim",
            "role": "hard_fail",
            "field": "$.functional_interpretation",
            "kind": "avoid",
            "expected": [
              "costimulation_licensed_reinvigoration_competent",
              "costimulation_licensed_checkpoint_responsive",
              "tumor_driven_terminal_exhaustion"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-luna-pi",
      "modelName": "gpt-5.6-luna",
      "harness": "pi",
      "task": "10x_5p_melanoma_tcr_matched_apc_vs_tumor_exhaustion",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1022.140459,
      "cost": 0.42187525000000003,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "state_discriminating_genes",
            "kind": "list",
            "matchKey": "gene",
            "k": 3,
            "expected": [
              {
                "key": "CTLA4",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TOX",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "ENTPD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "HAVCR2",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TIGIT",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "PDCD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "difference_attributable_to",
            "role": "gate",
            "field": "difference_attributable_to",
            "kind": "value",
            "expected": "covaries_with_apc_conjugate_state"
          },
          {
            "name": "causal_status",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "direction_unresolved_selection_confound"
          },
          {
            "name": "functional_interpretation",
            "role": "gate",
            "field": "functional_interpretation",
            "kind": "value",
            "expected": "costimulation_signature_function_unresolved"
          },
          {
            "name": "cf_contamination_gene_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "MLANA",
              "PMEL",
              "DCT",
              "TYR",
              "TYRP1",
              "SOX10",
              "MITF",
              "MAGEA4"
            ]
          },
          {
            "name": "cf_apc_partner_contamination_gene",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "HLA-DRA",
              "CD68",
              "LYZ",
              "CD14",
              "C1QA",
              "ITGAX",
              "CD1C"
            ]
          },
          {
            "name": "cf_ligand_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "CD274",
              "PDCD1LG2",
              "CD80",
              "CD86"
            ]
          },
          {
            "name": "cf_attribution_causal_or_baseline",
            "role": "hard_fail",
            "field": "$.difference_attributable_to",
            "kind": "avoid",
            "expected": [
              "apc_engagement",
              "tumor_cell_engagement",
              "clonotype_intrinsic_baseline"
            ]
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "engagement_induces_exhausted_state",
              "pre_exhausted_cells_select_into_apc_conjugates"
            ]
          },
          {
            "name": "cf_causal_fabrication",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "resolved_by_pseudotemporal_ordering",
              "resolved_by_perturbation_experiment"
            ]
          },
          {
            "name": "cf_functional_overclaim",
            "role": "hard_fail",
            "field": "$.functional_interpretation",
            "kind": "avoid",
            "expected": [
              "costimulation_licensed_reinvigoration_competent",
              "costimulation_licensed_checkpoint_responsive",
              "tumor_driven_terminal_exhaustion"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-luna-pi",
      "modelName": "gpt-5.6-luna",
      "harness": "pi",
      "task": "green_2_tme_context_dependence",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 2002.772182,
      "cost": 1.0910492000000003,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "program_map",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "sig_includes_itgae",
            "role": "gate",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Itgae"
            ]
          },
          {
            "name": "declared_a_known_confounder",
            "role": "gate",
            "field": "$.excluded_confounders[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "cf_signature_gut_contaminant",
            "role": "hard_fail",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "generic_signature_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd103_program_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "<"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "per_model_regulatory_state",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "poised_unfired"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "late_activation"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "closing_committed"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_tuples",
            "kind": "list",
            "matchKey": "regulator",
            "k": 1,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "role": {
                    "op": "equals",
                    "values": [
                      "repressor"
                    ]
                  },
                  "gating_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  },
                  "evidence_basis": {
                    "op": "equals",
                    "values": [
                      "target_program_behavior"
                    ]
                  },
                  "expr_vs_activity": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "locus_differential_gate",
            "role": "gate",
            "field": "locus_differential",
            "kind": "value",
            "expected": "uniform_across_models"
          },
          {
            "name": "formation_limited_by_gate",
            "role": "gate",
            "field": "formation_limited_by",
            "kind": "value",
            "expected": "absent_induction"
          },
          {
            "name": "locus_competent_in_naive_gate",
            "role": "gate",
            "field": "locus_competent_in_naive",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "ready_silent_fraction_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 3,
            "expected": [
              {
                "key": "B16_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_KPC_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "activator_chromatin_footprint_gate",
            "role": "gate",
            "field": "activator_chromatin_footprint",
            "kind": "value",
            "expected": "inconsistent_across_formers"
          },
          {
            "name": "exhaustion_comparable_gate",
            "role": "gate",
            "field": "exhaustion_comparable_across_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_nondeterminant_gate",
            "role": "gate",
            "field": "exhaustion_explains_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_separable_program_gate",
            "role": "gate",
            "field": "exhaustion_separable_program",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "exhaustion_counterexample_gate",
            "role": "gate",
            "field": "exhaustion_counterexample_model",
            "kind": "value",
            "expected": "KPC_GP"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "exhaustion_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "cf_inconsistent_present_without_activator",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "program",
            "entity": "present",
            "valueField": "activator_state",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_no_block_but_absent",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "none",
            "valueField": "program",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_late_block_without_poised_locus",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "late_activation",
            "valueField": "locus_state",
            "expected": [
              "closing_committed",
              "active"
            ]
          },
          {
            "name": "cf_inconsistent_early_block_with_brake_released",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "early_release",
            "valueField": "brake_state",
            "expected": [
              "released"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-luna-pi",
      "modelName": "gpt-5.6-luna",
      "harness": "pi",
      "task": "green_2_tme_context_dependence",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 6440.525578,
      "cost": 1.4924927000000001,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "program_map",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "sig_includes_itgae",
            "role": "gate",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Itgae"
            ]
          },
          {
            "name": "declared_a_known_confounder",
            "role": "gate",
            "field": "$.excluded_confounders[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "cf_signature_gut_contaminant",
            "role": "hard_fail",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "generic_signature_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd103_program_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "<"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "per_model_regulatory_state",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "poised_unfired"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "late_activation"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "closing_committed"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_tuples",
            "kind": "list",
            "matchKey": "regulator",
            "k": 1,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "role": {
                    "op": "equals",
                    "values": [
                      "repressor"
                    ]
                  },
                  "gating_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  },
                  "evidence_basis": {
                    "op": "equals",
                    "values": [
                      "target_program_behavior"
                    ]
                  },
                  "expr_vs_activity": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "locus_differential_gate",
            "role": "gate",
            "field": "locus_differential",
            "kind": "value",
            "expected": "uniform_across_models"
          },
          {
            "name": "formation_limited_by_gate",
            "role": "gate",
            "field": "formation_limited_by",
            "kind": "value",
            "expected": "absent_induction"
          },
          {
            "name": "locus_competent_in_naive_gate",
            "role": "gate",
            "field": "locus_competent_in_naive",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "ready_silent_fraction_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 3,
            "expected": [
              {
                "key": "B16_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_KPC_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "activator_chromatin_footprint_gate",
            "role": "gate",
            "field": "activator_chromatin_footprint",
            "kind": "value",
            "expected": "inconsistent_across_formers"
          },
          {
            "name": "exhaustion_comparable_gate",
            "role": "gate",
            "field": "exhaustion_comparable_across_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_nondeterminant_gate",
            "role": "gate",
            "field": "exhaustion_explains_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_separable_program_gate",
            "role": "gate",
            "field": "exhaustion_separable_program",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "exhaustion_counterexample_gate",
            "role": "gate",
            "field": "exhaustion_counterexample_model",
            "kind": "value",
            "expected": "KPC_GP"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "exhaustion_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "cf_inconsistent_present_without_activator",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "program",
            "entity": "present",
            "valueField": "activator_state",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_no_block_but_absent",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "none",
            "valueField": "program",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_late_block_without_poised_locus",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "late_activation",
            "valueField": "locus_state",
            "expected": [
              "closing_committed",
              "active"
            ]
          },
          {
            "name": "cf_inconsistent_early_block_with_brake_released",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "early_release",
            "valueField": "brake_state",
            "expected": [
              "released"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-luna-pi",
      "modelName": "gpt-5.6-luna",
      "harness": "pi",
      "task": "green_2_tme_context_dependence",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1012.452034,
      "cost": 0.9293665000000004,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "program_map",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "sig_includes_itgae",
            "role": "gate",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Itgae"
            ]
          },
          {
            "name": "declared_a_known_confounder",
            "role": "gate",
            "field": "$.excluded_confounders[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "cf_signature_gut_contaminant",
            "role": "hard_fail",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "generic_signature_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd103_program_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "<"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "per_model_regulatory_state",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "poised_unfired"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "late_activation"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "closing_committed"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_tuples",
            "kind": "list",
            "matchKey": "regulator",
            "k": 1,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "role": {
                    "op": "equals",
                    "values": [
                      "repressor"
                    ]
                  },
                  "gating_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  },
                  "evidence_basis": {
                    "op": "equals",
                    "values": [
                      "target_program_behavior"
                    ]
                  },
                  "expr_vs_activity": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "locus_differential_gate",
            "role": "gate",
            "field": "locus_differential",
            "kind": "value",
            "expected": "uniform_across_models"
          },
          {
            "name": "formation_limited_by_gate",
            "role": "gate",
            "field": "formation_limited_by",
            "kind": "value",
            "expected": "absent_induction"
          },
          {
            "name": "locus_competent_in_naive_gate",
            "role": "gate",
            "field": "locus_competent_in_naive",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "ready_silent_fraction_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 3,
            "expected": [
              {
                "key": "B16_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_KPC_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "activator_chromatin_footprint_gate",
            "role": "gate",
            "field": "activator_chromatin_footprint",
            "kind": "value",
            "expected": "inconsistent_across_formers"
          },
          {
            "name": "exhaustion_comparable_gate",
            "role": "gate",
            "field": "exhaustion_comparable_across_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_nondeterminant_gate",
            "role": "gate",
            "field": "exhaustion_explains_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_separable_program_gate",
            "role": "gate",
            "field": "exhaustion_separable_program",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "exhaustion_counterexample_gate",
            "role": "gate",
            "field": "exhaustion_counterexample_model",
            "kind": "value",
            "expected": "KPC_GP"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "exhaustion_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "cf_inconsistent_present_without_activator",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "program",
            "entity": "present",
            "valueField": "activator_state",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_no_block_but_absent",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "none",
            "valueField": "program",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_late_block_without_poised_locus",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "late_activation",
            "valueField": "locus_state",
            "expected": [
              "closing_committed",
              "active"
            ]
          },
          {
            "name": "cf_inconsistent_early_block_with_brake_released",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "early_release",
            "valueField": "brake_state",
            "expected": [
              "released"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-luna-pi",
      "modelName": "gpt-5.6-luna",
      "harness": "pi",
      "task": "green_klf2_hidden_repressor_of_trm_like_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 2181.530549,
      "cost": 0.7136723500000001,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "regulator_gate",
            "role": "gate",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "KLF2",
              "Klf2"
            ]
          },
          {
            "name": "controlling_node_character_gate",
            "role": "gate",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "high_differential_expression"
          },
          {
            "name": "expression_vs_activity_gate",
            "role": "gate",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "expression_exceeds_activity"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "target_program_partition",
            "kind": "list",
            "matchKey": "gene",
            "k": 7,
            "expected": [
              {
                "key": "S1pr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "S1pr4",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Klrg1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Cx3cr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Itgae",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Hic1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "P2rx7",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              }
            ]
          },
          {
            "name": "causal_status_gate",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "association_only_sufficiency_requires_perturbation"
          },
          {
            "name": "residency_control_logic_gate",
            "role": "gate",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_release_necessary_not_sufficient_multi_regulon"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "trm_like_program_by_model",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_absent"
                    ],
                    "canonical": "niche_absent"
                  }
                }
              }
            ]
          },
          {
            "name": "cf_regulator_activator_or_wrong_node",
            "role": "hard_fail",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "RUNX3",
              "Runx3",
              "PRDM1",
              "Prdm1",
              "BLIMP1",
              "Blimp1",
              "BHLHE40",
              "Bhlhe40",
              "ZNF683",
              "Zfp683",
              "HOBIT",
              "Hobit",
              "AHR",
              "Ahr",
              "BATF",
              "Batf",
              "TOX",
              "Tox",
              "NR4A1",
              "Nr4a1",
              "NR4A2",
              "Nr4a2",
              "NR4A3",
              "Nr4a3",
              "TCF7",
              "Tcf7",
              "TCF1",
              "Tcf1",
              "BACH2",
              "Bach2",
              "TBX21",
              "Tbx21",
              "TBET",
              "Tbet",
              "FLI1",
              "Fli1",
              "NR3C1",
              "Nr3c1",
              "EOMES",
              "Eomes",
              "ZEB2",
              "Zeb2",
              "ID2",
              "Id2",
              "ID3",
              "Id3",
              "IRF4",
              "Irf4"
            ]
          },
          {
            "name": "cf_node_character_hidden_driver_recall",
            "role": "hard_fail",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "hidden_driver_activity_not_expression"
          },
          {
            "name": "cf_effect_relation_hidden_pattern",
            "role": "hard_fail",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "activity_exceeds_expression"
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "causal_status",
            "kind": "value",
            "op": "in",
            "expected": [
              "sufficiency_established",
              "necessity_established",
              "causal_established_in_dataset",
              "sufficiency_via_perturbation_in_dataset"
            ]
          },
          {
            "name": "cf_residency_logic_activator_prior",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "residency_activator_induction_primary"
          },
          {
            "name": "cf_residency_logic_single_regulon",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_controls_entire_residency_program"
          },
          {
            "name": "cf_decoy_called_target",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_distractor_model_reported",
            "role": "hard_fail",
            "field": "$.trm_like_program_by_model[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_chromatin_only_decoy",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-luna-pi",
      "modelName": "gpt-5.6-luna",
      "harness": "pi",
      "task": "green_klf2_hidden_repressor_of_trm_like_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 4021.339724,
      "cost": 2.33995835,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "regulator_gate",
            "role": "gate",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "KLF2",
              "Klf2"
            ]
          },
          {
            "name": "controlling_node_character_gate",
            "role": "gate",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "high_differential_expression"
          },
          {
            "name": "expression_vs_activity_gate",
            "role": "gate",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "expression_exceeds_activity"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "target_program_partition",
            "kind": "list",
            "matchKey": "gene",
            "k": 7,
            "expected": [
              {
                "key": "S1pr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "S1pr4",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Klrg1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Cx3cr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Itgae",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Hic1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "P2rx7",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              }
            ]
          },
          {
            "name": "causal_status_gate",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "association_only_sufficiency_requires_perturbation"
          },
          {
            "name": "residency_control_logic_gate",
            "role": "gate",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_release_necessary_not_sufficient_multi_regulon"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "trm_like_program_by_model",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_absent"
                    ],
                    "canonical": "niche_absent"
                  }
                }
              }
            ]
          },
          {
            "name": "cf_regulator_activator_or_wrong_node",
            "role": "hard_fail",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "RUNX3",
              "Runx3",
              "PRDM1",
              "Prdm1",
              "BLIMP1",
              "Blimp1",
              "BHLHE40",
              "Bhlhe40",
              "ZNF683",
              "Zfp683",
              "HOBIT",
              "Hobit",
              "AHR",
              "Ahr",
              "BATF",
              "Batf",
              "TOX",
              "Tox",
              "NR4A1",
              "Nr4a1",
              "NR4A2",
              "Nr4a2",
              "NR4A3",
              "Nr4a3",
              "TCF7",
              "Tcf7",
              "TCF1",
              "Tcf1",
              "BACH2",
              "Bach2",
              "TBX21",
              "Tbx21",
              "TBET",
              "Tbet",
              "FLI1",
              "Fli1",
              "NR3C1",
              "Nr3c1",
              "EOMES",
              "Eomes",
              "ZEB2",
              "Zeb2",
              "ID2",
              "Id2",
              "ID3",
              "Id3",
              "IRF4",
              "Irf4"
            ]
          },
          {
            "name": "cf_node_character_hidden_driver_recall",
            "role": "hard_fail",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "hidden_driver_activity_not_expression"
          },
          {
            "name": "cf_effect_relation_hidden_pattern",
            "role": "hard_fail",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "activity_exceeds_expression"
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "causal_status",
            "kind": "value",
            "op": "in",
            "expected": [
              "sufficiency_established",
              "necessity_established",
              "causal_established_in_dataset",
              "sufficiency_via_perturbation_in_dataset"
            ]
          },
          {
            "name": "cf_residency_logic_activator_prior",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "residency_activator_induction_primary"
          },
          {
            "name": "cf_residency_logic_single_regulon",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_controls_entire_residency_program"
          },
          {
            "name": "cf_decoy_called_target",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_distractor_model_reported",
            "role": "hard_fail",
            "field": "$.trm_like_program_by_model[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_chromatin_only_decoy",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-luna-pi",
      "modelName": "gpt-5.6-luna",
      "harness": "pi",
      "task": "green_klf2_hidden_repressor_of_trm_like_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1206.584722,
      "cost": 1.4445207499999995,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "regulator_gate",
            "role": "gate",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "KLF2",
              "Klf2"
            ]
          },
          {
            "name": "controlling_node_character_gate",
            "role": "gate",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "high_differential_expression"
          },
          {
            "name": "expression_vs_activity_gate",
            "role": "gate",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "expression_exceeds_activity"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "target_program_partition",
            "kind": "list",
            "matchKey": "gene",
            "k": 7,
            "expected": [
              {
                "key": "S1pr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "S1pr4",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Klrg1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Cx3cr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Itgae",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Hic1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "P2rx7",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              }
            ]
          },
          {
            "name": "causal_status_gate",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "association_only_sufficiency_requires_perturbation"
          },
          {
            "name": "residency_control_logic_gate",
            "role": "gate",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_release_necessary_not_sufficient_multi_regulon"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "trm_like_program_by_model",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_absent"
                    ],
                    "canonical": "niche_absent"
                  }
                }
              }
            ]
          },
          {
            "name": "cf_regulator_activator_or_wrong_node",
            "role": "hard_fail",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "RUNX3",
              "Runx3",
              "PRDM1",
              "Prdm1",
              "BLIMP1",
              "Blimp1",
              "BHLHE40",
              "Bhlhe40",
              "ZNF683",
              "Zfp683",
              "HOBIT",
              "Hobit",
              "AHR",
              "Ahr",
              "BATF",
              "Batf",
              "TOX",
              "Tox",
              "NR4A1",
              "Nr4a1",
              "NR4A2",
              "Nr4a2",
              "NR4A3",
              "Nr4a3",
              "TCF7",
              "Tcf7",
              "TCF1",
              "Tcf1",
              "BACH2",
              "Bach2",
              "TBX21",
              "Tbx21",
              "TBET",
              "Tbet",
              "FLI1",
              "Fli1",
              "NR3C1",
              "Nr3c1",
              "EOMES",
              "Eomes",
              "ZEB2",
              "Zeb2",
              "ID2",
              "Id2",
              "ID3",
              "Id3",
              "IRF4",
              "Irf4"
            ]
          },
          {
            "name": "cf_node_character_hidden_driver_recall",
            "role": "hard_fail",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "hidden_driver_activity_not_expression"
          },
          {
            "name": "cf_effect_relation_hidden_pattern",
            "role": "hard_fail",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "activity_exceeds_expression"
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "causal_status",
            "kind": "value",
            "op": "in",
            "expected": [
              "sufficiency_established",
              "necessity_established",
              "causal_established_in_dataset",
              "sufficiency_via_perturbation_in_dataset"
            ]
          },
          {
            "name": "cf_residency_logic_activator_prior",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "residency_activator_induction_primary"
          },
          {
            "name": "cf_residency_logic_single_regulon",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_controls_entire_residency_program"
          },
          {
            "name": "cf_decoy_called_target",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_distractor_model_reported",
            "role": "hard_fail",
            "field": "$.trm_like_program_by_model[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_chromatin_only_decoy",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-luna-pi",
      "modelName": "gpt-5.6-luna",
      "harness": "pi",
      "task": "green_shared_vs_distinct_regulators_trm_vs_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 1093.342674,
      "cost": 1.0259136000000004,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_program_map",
            "kind": "list",
            "matchKey": "regulator",
            "k": 3,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "both"
                    ],
                    "canonical": "both"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "represses"
                    ],
                    "canonical": "represses"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Batf",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "trm_like_til_specific"
                    ],
                    "canonical": "trm_like_til_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "chromatin_over_rna"
                    ],
                    "canonical": "chromatin_over_rna"
                  }
                }
              },
              {
                "key": "Ahr",
                "fields": {
                  "program_specificity": {
                    "op": "in",
                    "values": [
                      "trm_specific",
                      "both"
                    ],
                    "canonical": "trm_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "rna_over_chromatin"
                    ],
                    "canonical": "rna_over_chromatin"
                  }
                }
              }
            ]
          },
          {
            "name": "regulator_set_no_extras_gate",
            "role": "gate",
            "field": "$.regulator_program_map[*]",
            "kind": "other",
            "expected": null
          },
          {
            "name": "program_identity_conclusion_gate",
            "role": "gate",
            "field": "program_identity_conclusion",
            "kind": "value",
            "expected": "distinct_state_shared_brake_divergent_activators"
          },
          {
            "name": "residency_install_logic_gate",
            "role": "gate",
            "field": "residency_install_logic",
            "kind": "value",
            "expected": "brake_release_permits_context_matched_activator"
          },
          {
            "name": "activator_context_dependence_gate",
            "role": "gate",
            "field": "activator_context_dependence",
            "kind": "value",
            "expected": "context_matched_crossover"
          },
          {
            "name": "evidence_ceiling_gate",
            "role": "gate",
            "field": "evidence_ceiling",
            "kind": "value",
            "expected": "specificity_dissociations_require_perturbation"
          },
          {
            "name": "cf_install_logic_flat_or_single_step",
            "role": "hard_fail",
            "field": "residency_install_logic",
            "kind": "value",
            "op": "in",
            "expected": [
              "flat_shared_residency_toolkit",
              "activator_induction_sufficient",
              "brake_release_alone_sufficient"
            ]
          },
          {
            "name": "cf_program_identity_lumped_or_wrong",
            "role": "hard_fail",
            "field": "program_identity_conclusion",
            "kind": "value",
            "op": "in",
            "expected": [
              "trm_like_til_is_tumor_trm",
              "trm_like_til_is_exhausted_subset",
              "trm_like_til_is_effector_state",
              "trm_like_til_is_transitional_intermediate",
              "trm_like_til_is_shared_program_distinct_niche"
            ]
          },
          {
            "name": "cf_ahr_assigned_to_til",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Ahr",
            "valueField": "program_specificity",
            "expected": [
              "trm_like_til_specific"
            ]
          },
          {
            "name": "cf_batf_assigned_to_trm_or_shared",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Batf",
            "valueField": "program_specificity",
            "expected": [
              "trm_specific",
              "both"
            ]
          },
          {
            "name": "cf_klf2_called_activator",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Klf2",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          },
          {
            "name": "cf_evidence_ceiling_overclaim",
            "role": "hard_fail",
            "field": "evidence_ceiling",
            "kind": "value",
            "op": "in",
            "expected": [
              "specificity_established_from_regulon_activity",
              "specificity_established_in_dataset",
              "causation_established_by_cross_sectional_concordance",
              "necessity_and_sufficiency_established"
            ]
          },
          {
            "name": "cf_activator_context_shared_or_single",
            "role": "hard_fail",
            "field": "activator_context_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "shared_activators_both_contexts",
              "single_global_activator",
              "redundant_activators"
            ]
          },
          {
            "name": "cf_single_modality_rule",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_hic1_motif_brake_mislabel",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_runx3_anticanonical_as_controller",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Runx3",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-luna-pi",
      "modelName": "gpt-5.6-luna",
      "harness": "pi",
      "task": "green_shared_vs_distinct_regulators_trm_vs_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 1433.341493,
      "cost": 1.9286683499999997,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_program_map",
            "kind": "list",
            "matchKey": "regulator",
            "k": 3,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "both"
                    ],
                    "canonical": "both"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "represses"
                    ],
                    "canonical": "represses"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Batf",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "trm_like_til_specific"
                    ],
                    "canonical": "trm_like_til_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "chromatin_over_rna"
                    ],
                    "canonical": "chromatin_over_rna"
                  }
                }
              },
              {
                "key": "Ahr",
                "fields": {
                  "program_specificity": {
                    "op": "in",
                    "values": [
                      "trm_specific",
                      "both"
                    ],
                    "canonical": "trm_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "rna_over_chromatin"
                    ],
                    "canonical": "rna_over_chromatin"
                  }
                }
              }
            ]
          },
          {
            "name": "regulator_set_no_extras_gate",
            "role": "gate",
            "field": "$.regulator_program_map[*]",
            "kind": "other",
            "expected": null
          },
          {
            "name": "program_identity_conclusion_gate",
            "role": "gate",
            "field": "program_identity_conclusion",
            "kind": "value",
            "expected": "distinct_state_shared_brake_divergent_activators"
          },
          {
            "name": "residency_install_logic_gate",
            "role": "gate",
            "field": "residency_install_logic",
            "kind": "value",
            "expected": "brake_release_permits_context_matched_activator"
          },
          {
            "name": "activator_context_dependence_gate",
            "role": "gate",
            "field": "activator_context_dependence",
            "kind": "value",
            "expected": "context_matched_crossover"
          },
          {
            "name": "evidence_ceiling_gate",
            "role": "gate",
            "field": "evidence_ceiling",
            "kind": "value",
            "expected": "specificity_dissociations_require_perturbation"
          },
          {
            "name": "cf_install_logic_flat_or_single_step",
            "role": "hard_fail",
            "field": "residency_install_logic",
            "kind": "value",
            "op": "in",
            "expected": [
              "flat_shared_residency_toolkit",
              "activator_induction_sufficient",
              "brake_release_alone_sufficient"
            ]
          },
          {
            "name": "cf_program_identity_lumped_or_wrong",
            "role": "hard_fail",
            "field": "program_identity_conclusion",
            "kind": "value",
            "op": "in",
            "expected": [
              "trm_like_til_is_tumor_trm",
              "trm_like_til_is_exhausted_subset",
              "trm_like_til_is_effector_state",
              "trm_like_til_is_transitional_intermediate",
              "trm_like_til_is_shared_program_distinct_niche"
            ]
          },
          {
            "name": "cf_ahr_assigned_to_til",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Ahr",
            "valueField": "program_specificity",
            "expected": [
              "trm_like_til_specific"
            ]
          },
          {
            "name": "cf_batf_assigned_to_trm_or_shared",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Batf",
            "valueField": "program_specificity",
            "expected": [
              "trm_specific",
              "both"
            ]
          },
          {
            "name": "cf_klf2_called_activator",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Klf2",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          },
          {
            "name": "cf_evidence_ceiling_overclaim",
            "role": "hard_fail",
            "field": "evidence_ceiling",
            "kind": "value",
            "op": "in",
            "expected": [
              "specificity_established_from_regulon_activity",
              "specificity_established_in_dataset",
              "causation_established_by_cross_sectional_concordance",
              "necessity_and_sufficiency_established"
            ]
          },
          {
            "name": "cf_activator_context_shared_or_single",
            "role": "hard_fail",
            "field": "activator_context_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "shared_activators_both_contexts",
              "single_global_activator",
              "redundant_activators"
            ]
          },
          {
            "name": "cf_single_modality_rule",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_hic1_motif_brake_mislabel",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_runx3_anticanonical_as_controller",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Runx3",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-luna-pi",
      "modelName": "gpt-5.6-luna",
      "harness": "pi",
      "task": "green_shared_vs_distinct_regulators_trm_vs_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 2853.550419,
      "cost": 2.268986900000001,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_program_map",
            "kind": "list",
            "matchKey": "regulator",
            "k": 3,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "both"
                    ],
                    "canonical": "both"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "represses"
                    ],
                    "canonical": "represses"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Batf",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "trm_like_til_specific"
                    ],
                    "canonical": "trm_like_til_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "chromatin_over_rna"
                    ],
                    "canonical": "chromatin_over_rna"
                  }
                }
              },
              {
                "key": "Ahr",
                "fields": {
                  "program_specificity": {
                    "op": "in",
                    "values": [
                      "trm_specific",
                      "both"
                    ],
                    "canonical": "trm_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "rna_over_chromatin"
                    ],
                    "canonical": "rna_over_chromatin"
                  }
                }
              }
            ]
          },
          {
            "name": "regulator_set_no_extras_gate",
            "role": "gate",
            "field": "$.regulator_program_map[*]",
            "kind": "other",
            "expected": null
          },
          {
            "name": "program_identity_conclusion_gate",
            "role": "gate",
            "field": "program_identity_conclusion",
            "kind": "value",
            "expected": "distinct_state_shared_brake_divergent_activators"
          },
          {
            "name": "residency_install_logic_gate",
            "role": "gate",
            "field": "residency_install_logic",
            "kind": "value",
            "expected": "brake_release_permits_context_matched_activator"
          },
          {
            "name": "activator_context_dependence_gate",
            "role": "gate",
            "field": "activator_context_dependence",
            "kind": "value",
            "expected": "context_matched_crossover"
          },
          {
            "name": "evidence_ceiling_gate",
            "role": "gate",
            "field": "evidence_ceiling",
            "kind": "value",
            "expected": "specificity_dissociations_require_perturbation"
          },
          {
            "name": "cf_install_logic_flat_or_single_step",
            "role": "hard_fail",
            "field": "residency_install_logic",
            "kind": "value",
            "op": "in",
            "expected": [
              "flat_shared_residency_toolkit",
              "activator_induction_sufficient",
              "brake_release_alone_sufficient"
            ]
          },
          {
            "name": "cf_program_identity_lumped_or_wrong",
            "role": "hard_fail",
            "field": "program_identity_conclusion",
            "kind": "value",
            "op": "in",
            "expected": [
              "trm_like_til_is_tumor_trm",
              "trm_like_til_is_exhausted_subset",
              "trm_like_til_is_effector_state",
              "trm_like_til_is_transitional_intermediate",
              "trm_like_til_is_shared_program_distinct_niche"
            ]
          },
          {
            "name": "cf_ahr_assigned_to_til",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Ahr",
            "valueField": "program_specificity",
            "expected": [
              "trm_like_til_specific"
            ]
          },
          {
            "name": "cf_batf_assigned_to_trm_or_shared",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Batf",
            "valueField": "program_specificity",
            "expected": [
              "trm_specific",
              "both"
            ]
          },
          {
            "name": "cf_klf2_called_activator",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Klf2",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          },
          {
            "name": "cf_evidence_ceiling_overclaim",
            "role": "hard_fail",
            "field": "evidence_ceiling",
            "kind": "value",
            "op": "in",
            "expected": [
              "specificity_established_from_regulon_activity",
              "specificity_established_in_dataset",
              "causation_established_by_cross_sectional_concordance",
              "necessity_and_sufficiency_established"
            ]
          },
          {
            "name": "cf_activator_context_shared_or_single",
            "role": "hard_fail",
            "field": "activator_context_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "shared_activators_both_contexts",
              "single_global_activator",
              "redundant_activators"
            ]
          },
          {
            "name": "cf_single_modality_rule",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_hic1_motif_brake_mislabel",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_runx3_anticanonical_as_controller",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Runx3",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-luna-pi",
      "modelName": "gpt-5.6-luna",
      "harness": "pi",
      "task": "gse297023_q2_senescence_not_mechanism",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 1,
      "passed": true,
      "score": null,
      "durationS": 573.809571,
      "cost": 0.5931563999999999,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "candidate_is_senescence",
            "role": "gate",
            "field": "$.candidate_mechanism",
            "kind": "value",
            "op": "in",
            "expected": [
              "cellular_senescence",
              "oncogene_induced_senescence",
              "replicative_senescence"
            ]
          },
          {
            "name": "arrest_core_not_engaged",
            "role": "gate",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_not_engaged"
            ]
          },
          {
            "name": "secretory_arm_is_inflammaging",
            "role": "gate",
            "field": "$.secretory_arm_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "acute_phase_inflammaging_response"
            ]
          },
          {
            "name": "secretory_arm_broadly_elevated",
            "role": "gate",
            "field": "$.secretory_arm_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "broadly_elevated"
            ]
          },
          {
            "name": "carries_aging_program",
            "role": "gate",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "carries_bona_fide_aging_program_distinct_from_nominated"
            ]
          },
          {
            "name": "conclusion_senescence_unlikely",
            "role": "gate",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "unlikely_major_contributor"
            ]
          },
          {
            "name": "cf_arrest_core_engaged_recall",
            "role": "hard_fail",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_engaged"
            ]
          },
          {
            "name": "cf_aging_program_conflation_or_explain_away",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "aging_program_indistinguishable_from_nominated_program",
              "aging_program_is_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_no_aging_program_sensitivity_failure",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_detectable_aging_program"
            ]
          },
          {
            "name": "cf_senescence_driver_conclusion",
            "role": "hard_fail",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "likely_primary_driver",
              "plausible_supported_contributor"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-luna-pi",
      "modelName": "gpt-5.6-luna",
      "harness": "pi",
      "task": "gse297023_q2_senescence_not_mechanism",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 337.618232,
      "cost": 0.4445935000000001,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "candidate_is_senescence",
            "role": "gate",
            "field": "$.candidate_mechanism",
            "kind": "value",
            "op": "in",
            "expected": [
              "cellular_senescence",
              "oncogene_induced_senescence",
              "replicative_senescence"
            ]
          },
          {
            "name": "arrest_core_not_engaged",
            "role": "gate",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_not_engaged"
            ]
          },
          {
            "name": "secretory_arm_is_inflammaging",
            "role": "gate",
            "field": "$.secretory_arm_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "acute_phase_inflammaging_response"
            ]
          },
          {
            "name": "secretory_arm_broadly_elevated",
            "role": "gate",
            "field": "$.secretory_arm_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "broadly_elevated"
            ]
          },
          {
            "name": "carries_aging_program",
            "role": "gate",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "carries_bona_fide_aging_program_distinct_from_nominated"
            ]
          },
          {
            "name": "conclusion_senescence_unlikely",
            "role": "gate",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "unlikely_major_contributor"
            ]
          },
          {
            "name": "cf_arrest_core_engaged_recall",
            "role": "hard_fail",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_engaged"
            ]
          },
          {
            "name": "cf_aging_program_conflation_or_explain_away",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "aging_program_indistinguishable_from_nominated_program",
              "aging_program_is_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_no_aging_program_sensitivity_failure",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_detectable_aging_program"
            ]
          },
          {
            "name": "cf_senescence_driver_conclusion",
            "role": "hard_fail",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "likely_primary_driver",
              "plausible_supported_contributor"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-luna-pi",
      "modelName": "gpt-5.6-luna",
      "harness": "pi",
      "task": "gse297023_q2_senescence_not_mechanism",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 2927.901193,
      "cost": 0.5557051000000002,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "candidate_is_senescence",
            "role": "gate",
            "field": "$.candidate_mechanism",
            "kind": "value",
            "op": "in",
            "expected": [
              "cellular_senescence",
              "oncogene_induced_senescence",
              "replicative_senescence"
            ]
          },
          {
            "name": "arrest_core_not_engaged",
            "role": "gate",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_not_engaged"
            ]
          },
          {
            "name": "secretory_arm_is_inflammaging",
            "role": "gate",
            "field": "$.secretory_arm_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "acute_phase_inflammaging_response"
            ]
          },
          {
            "name": "secretory_arm_broadly_elevated",
            "role": "gate",
            "field": "$.secretory_arm_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "broadly_elevated"
            ]
          },
          {
            "name": "carries_aging_program",
            "role": "gate",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "carries_bona_fide_aging_program_distinct_from_nominated"
            ]
          },
          {
            "name": "conclusion_senescence_unlikely",
            "role": "gate",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "unlikely_major_contributor"
            ]
          },
          {
            "name": "cf_arrest_core_engaged_recall",
            "role": "hard_fail",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_engaged"
            ]
          },
          {
            "name": "cf_aging_program_conflation_or_explain_away",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "aging_program_indistinguishable_from_nominated_program",
              "aging_program_is_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_no_aging_program_sensitivity_failure",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_detectable_aging_program"
            ]
          },
          {
            "name": "cf_senescence_driver_conclusion",
            "role": "hard_fail",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "likely_primary_driver",
              "plausible_supported_contributor"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-luna-pi",
      "modelName": "gpt-5.6-luna",
      "harness": "pi",
      "task": "gse297023_q3_age_retained_mapk_reduced",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 1,
      "passed": true,
      "score": null,
      "durationS": 4005.915619,
      "cost": 1.0305551499999999,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "age_retained",
            "role": "gate",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_retained"
            ]
          },
          {
            "name": "identity_is_aging",
            "role": "gate",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_aging_signatures"
            ]
          },
          {
            "name": "nature_state_shift",
            "role": "gate",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_state_expression_shift"
            ]
          },
          {
            "name": "pathway_is_mapk",
            "role": "gate",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "mapk"
            ]
          },
          {
            "name": "pathway_reduced_in_aged",
            "role": "gate",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "reduced_in_aged"
            ]
          },
          {
            "name": "cf_mapk_increased_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_in_aged"
            ]
          },
          {
            "name": "cf_age_erased",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_erased"
            ]
          },
          {
            "name": "cf_pervasive_overcall",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_pervasive_artifactual"
            ]
          },
          {
            "name": "cf_explain_away_aging",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_compositional_or_identity",
            "role": "hard_fail",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_proportion_shift",
              "cell_identity_shift",
              "composition_dominant_state_secondary",
              "both_state_and_composition_equally"
            ]
          },
          {
            "name": "cf_biggest_inflammatory_signal_not_mapk",
            "role": "hard_fail",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "jak_stat",
              "nfkb",
              "tnfa"
            ]
          },
          {
            "name": "cf_senescence_misattribution",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_senescence_signature_specifically"
            ]
          },
          {
            "name": "cf_dismiss_as_batch_artifact",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_batch_or_technical_artifact"
            ]
          },
          {
            "name": "cf_mapk_comparable_constitutive_kras_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "comparable_across_ages"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-luna-pi",
      "modelName": "gpt-5.6-luna",
      "harness": "pi",
      "task": "gse297023_q3_age_retained_mapk_reduced",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 2,
      "passed": true,
      "score": null,
      "durationS": 3752.942289,
      "cost": 0.8747667000000001,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "age_retained",
            "role": "gate",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_retained"
            ]
          },
          {
            "name": "identity_is_aging",
            "role": "gate",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_aging_signatures"
            ]
          },
          {
            "name": "nature_state_shift",
            "role": "gate",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_state_expression_shift"
            ]
          },
          {
            "name": "pathway_is_mapk",
            "role": "gate",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "mapk"
            ]
          },
          {
            "name": "pathway_reduced_in_aged",
            "role": "gate",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "reduced_in_aged"
            ]
          },
          {
            "name": "cf_mapk_increased_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_in_aged"
            ]
          },
          {
            "name": "cf_age_erased",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_erased"
            ]
          },
          {
            "name": "cf_pervasive_overcall",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_pervasive_artifactual"
            ]
          },
          {
            "name": "cf_explain_away_aging",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_compositional_or_identity",
            "role": "hard_fail",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_proportion_shift",
              "cell_identity_shift",
              "composition_dominant_state_secondary",
              "both_state_and_composition_equally"
            ]
          },
          {
            "name": "cf_biggest_inflammatory_signal_not_mapk",
            "role": "hard_fail",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "jak_stat",
              "nfkb",
              "tnfa"
            ]
          },
          {
            "name": "cf_senescence_misattribution",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_senescence_signature_specifically"
            ]
          },
          {
            "name": "cf_dismiss_as_batch_artifact",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_batch_or_technical_artifact"
            ]
          },
          {
            "name": "cf_mapk_comparable_constitutive_kras_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "comparable_across_ages"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-luna-pi",
      "modelName": "gpt-5.6-luna",
      "harness": "pi",
      "task": "gse297023_q3_age_retained_mapk_reduced",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1481.460898,
      "cost": 1.30942095,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "age_retained",
            "role": "gate",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_retained"
            ]
          },
          {
            "name": "identity_is_aging",
            "role": "gate",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_aging_signatures"
            ]
          },
          {
            "name": "nature_state_shift",
            "role": "gate",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_state_expression_shift"
            ]
          },
          {
            "name": "pathway_is_mapk",
            "role": "gate",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "mapk"
            ]
          },
          {
            "name": "pathway_reduced_in_aged",
            "role": "gate",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "reduced_in_aged"
            ]
          },
          {
            "name": "cf_mapk_increased_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_in_aged"
            ]
          },
          {
            "name": "cf_age_erased",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_erased"
            ]
          },
          {
            "name": "cf_pervasive_overcall",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_pervasive_artifactual"
            ]
          },
          {
            "name": "cf_explain_away_aging",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_compositional_or_identity",
            "role": "hard_fail",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_proportion_shift",
              "cell_identity_shift",
              "composition_dominant_state_secondary",
              "both_state_and_composition_equally"
            ]
          },
          {
            "name": "cf_biggest_inflammatory_signal_not_mapk",
            "role": "hard_fail",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "jak_stat",
              "nfkb",
              "tnfa"
            ]
          },
          {
            "name": "cf_senescence_misattribution",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_senescence_signature_specifically"
            ]
          },
          {
            "name": "cf_dismiss_as_batch_artifact",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_batch_or_technical_artifact"
            ]
          },
          {
            "name": "cf_mapk_comparable_constitutive_kras_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "comparable_across_ages"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-luna-pi",
      "modelName": "gpt-5.6-luna",
      "harness": "pi",
      "task": "gse297023_q5_tme_age_dependent_remodeling",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 1274.436484,
      "cost": 0.42678109999999997,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "perturbation_is_pten",
            "role": "gate",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "pten"
            ]
          },
          {
            "name": "tme_non_cell_autonomous",
            "role": "gate",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "substantial_non_cell_autonomous_remodeling"
            ]
          },
          {
            "name": "tme_state_not_composition",
            "role": "gate",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "tme_age_dependent",
            "role": "gate",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_dependent"
            ]
          },
          {
            "name": "tme_response_broader_in_aged",
            "role": "gate",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_aged"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "most_responsive_noncancer_cell_types",
            "kind": "list",
            "matchKey": "cell_type",
            "k": 2,
            "expected": [
              {
                "key": "alveolar_macrophage",
                "fields": {}
              },
              {
                "key": "endothelial",
                "fields": {}
              }
            ]
          },
          {
            "name": "cf_cell_intrinsic_only",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "negligible_cell_intrinsic_only_effect",
              "remodeling_unresolved",
              "modest_remodeling_few_cell_types"
            ]
          },
          {
            "name": "cf_remodeling_is_contamination",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "apparent_remodeling_reflects_cancer_contamination"
            ]
          },
          {
            "name": "cf_perturbation_design_misread",
            "role": "hard_fail",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "multiple_tumor_suppressors_pooled",
              "no_additional_gene_difference_kras_only",
              "rb1_cdkn2a_double",
              "pcna_essential_gene_control",
              "difference_is_oncogene_not_suppressor"
            ]
          },
          {
            "name": "cf_age_independent_tme",
            "role": "hard_fail",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_independent"
            ]
          },
          {
            "name": "cf_compositional_shift",
            "role": "hard_fail",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_frequency_shift",
              "compositional_immune_infiltration",
              "composition_dominant_state_secondary",
              "both_state_and_composition"
            ]
          },
          {
            "name": "cf_epithelial_or_cancer_substitution",
            "role": "hard_fail",
            "field": "$.most_responsive_noncancer_cell_types[*].cell_type",
            "kind": "avoid",
            "expected": [
              "AT2_cell",
              "AT1_cell",
              "cancer_cell",
              "ciliated",
              "secretory",
              "club_cell",
              "basal_cell",
              "goblet_cell"
            ]
          },
          {
            "name": "cf_tme_breadth_broader_in_young",
            "role": "hard_fail",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_young"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-luna-pi",
      "modelName": "gpt-5.6-luna",
      "harness": "pi",
      "task": "gse297023_q5_tme_age_dependent_remodeling",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 708.298289,
      "cost": 0.5608796,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "perturbation_is_pten",
            "role": "gate",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "pten"
            ]
          },
          {
            "name": "tme_non_cell_autonomous",
            "role": "gate",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "substantial_non_cell_autonomous_remodeling"
            ]
          },
          {
            "name": "tme_state_not_composition",
            "role": "gate",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "tme_age_dependent",
            "role": "gate",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_dependent"
            ]
          },
          {
            "name": "tme_response_broader_in_aged",
            "role": "gate",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_aged"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "most_responsive_noncancer_cell_types",
            "kind": "list",
            "matchKey": "cell_type",
            "k": 2,
            "expected": [
              {
                "key": "alveolar_macrophage",
                "fields": {}
              },
              {
                "key": "endothelial",
                "fields": {}
              }
            ]
          },
          {
            "name": "cf_cell_intrinsic_only",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "negligible_cell_intrinsic_only_effect",
              "remodeling_unresolved",
              "modest_remodeling_few_cell_types"
            ]
          },
          {
            "name": "cf_remodeling_is_contamination",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "apparent_remodeling_reflects_cancer_contamination"
            ]
          },
          {
            "name": "cf_perturbation_design_misread",
            "role": "hard_fail",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "multiple_tumor_suppressors_pooled",
              "no_additional_gene_difference_kras_only",
              "rb1_cdkn2a_double",
              "pcna_essential_gene_control",
              "difference_is_oncogene_not_suppressor"
            ]
          },
          {
            "name": "cf_age_independent_tme",
            "role": "hard_fail",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_independent"
            ]
          },
          {
            "name": "cf_compositional_shift",
            "role": "hard_fail",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_frequency_shift",
              "compositional_immune_infiltration",
              "composition_dominant_state_secondary",
              "both_state_and_composition"
            ]
          },
          {
            "name": "cf_epithelial_or_cancer_substitution",
            "role": "hard_fail",
            "field": "$.most_responsive_noncancer_cell_types[*].cell_type",
            "kind": "avoid",
            "expected": [
              "AT2_cell",
              "AT1_cell",
              "cancer_cell",
              "ciliated",
              "secretory",
              "club_cell",
              "basal_cell",
              "goblet_cell"
            ]
          },
          {
            "name": "cf_tme_breadth_broader_in_young",
            "role": "hard_fail",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_young"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-luna-pi",
      "modelName": "gpt-5.6-luna",
      "harness": "pi",
      "task": "gse297023_q5_tme_age_dependent_remodeling",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1556.971741,
      "cost": 0.6643289500000003,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "perturbation_is_pten",
            "role": "gate",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "pten"
            ]
          },
          {
            "name": "tme_non_cell_autonomous",
            "role": "gate",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "substantial_non_cell_autonomous_remodeling"
            ]
          },
          {
            "name": "tme_state_not_composition",
            "role": "gate",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "tme_age_dependent",
            "role": "gate",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_dependent"
            ]
          },
          {
            "name": "tme_response_broader_in_aged",
            "role": "gate",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_aged"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "most_responsive_noncancer_cell_types",
            "kind": "list",
            "matchKey": "cell_type",
            "k": 2,
            "expected": [
              {
                "key": "alveolar_macrophage",
                "fields": {}
              },
              {
                "key": "endothelial",
                "fields": {}
              }
            ]
          },
          {
            "name": "cf_cell_intrinsic_only",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "negligible_cell_intrinsic_only_effect",
              "remodeling_unresolved",
              "modest_remodeling_few_cell_types"
            ]
          },
          {
            "name": "cf_remodeling_is_contamination",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "apparent_remodeling_reflects_cancer_contamination"
            ]
          },
          {
            "name": "cf_perturbation_design_misread",
            "role": "hard_fail",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "multiple_tumor_suppressors_pooled",
              "no_additional_gene_difference_kras_only",
              "rb1_cdkn2a_double",
              "pcna_essential_gene_control",
              "difference_is_oncogene_not_suppressor"
            ]
          },
          {
            "name": "cf_age_independent_tme",
            "role": "hard_fail",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_independent"
            ]
          },
          {
            "name": "cf_compositional_shift",
            "role": "hard_fail",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_frequency_shift",
              "compositional_immune_infiltration",
              "composition_dominant_state_secondary",
              "both_state_and_composition"
            ]
          },
          {
            "name": "cf_epithelial_or_cancer_substitution",
            "role": "hard_fail",
            "field": "$.most_responsive_noncancer_cell_types[*].cell_type",
            "kind": "avoid",
            "expected": [
              "AT2_cell",
              "AT1_cell",
              "cancer_cell",
              "ciliated",
              "secretory",
              "club_cell",
              "basal_cell",
              "goblet_cell"
            ]
          },
          {
            "name": "cf_tme_breadth_broader_in_young",
            "role": "hard_fail",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_young"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-luna-pi",
      "modelName": "gpt-5.6-luna",
      "harness": "pi",
      "task": "gse297023_q6_age_immune_infiltration",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 1,
      "passed": true,
      "score": null,
      "durationS": 407.476822,
      "cost": 0.61869245,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "composition_stable_with_age",
            "role": "gate",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "frequencies_largely_stable"
            ]
          },
          {
            "name": "age_signal_is_state_not_composition",
            "role": "gate",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "frequencies_uninformative_for_infiltration",
            "role": "gate",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_cannot_establish_infiltration"
            ]
          },
          {
            "name": "infiltration_increases_with_age",
            "role": "gate",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_with_age"
            ]
          },
          {
            "name": "infiltration_strength_calibrated",
            "role": "gate",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "significant_increase_strongest_markers_only",
              "consistent_increase_not_statistically_robust"
            ]
          },
          {
            "name": "genotype_marker_fraction_lower_in_pten",
            "role": "gate",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "lower_in_pten_deficient"
            ]
          },
          {
            "name": "cf_composition_shift_reading",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "major_compositional_shift_with_age",
              "increased_immune_fraction_with_age"
            ]
          },
          {
            "name": "cf_lineage_specific_composition_claim",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "myeloid_expansion_with_age",
              "lymphoid_expansion_with_age",
              "minor_shifts_in_some_cell_types"
            ]
          },
          {
            "name": "cf_immunosenescence_depletion_prior",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "immune_depletion_with_age"
            ]
          },
          {
            "name": "cf_age_modality_compositional",
            "role": "hard_fail",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_type_composition_shift",
              "both_equally",
              "composition_dominant_state_secondary"
            ]
          },
          {
            "name": "cf_overread_frequencies_as_infiltration",
            "role": "hard_fail",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_shows_no_infiltration_change",
              "single_cell_quantifies_infiltration",
              "frequencies_show_increased_infiltration",
              "frequencies_show_decreased_infiltration"
            ]
          },
          {
            "name": "cf_infiltration_decreases_immunosenescence",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "decreased_with_age"
            ]
          },
          {
            "name": "cf_infiltration_unchanged_scrna_overread",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "unchanged_with_age"
            ]
          },
          {
            "name": "cf_infiltration_indeterminate_unused_node",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "direction_indeterminate_from_data"
            ]
          },
          {
            "name": "cf_infiltration_mixed_direction",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "mixed_by_immune_cell_type"
            ]
          },
          {
            "name": "cf_infiltration_genotype_confusion",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_only_in_pten_deficient",
              "decreased_only_in_pten_deficient"
            ]
          },
          {
            "name": "cf_pten_infiltration_paper_text_recall",
            "role": "hard_fail",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "higher_in_pten_deficient",
              "higher_in_pten_deficient_only_in_young",
              "higher_in_pten_deficient_only_in_aged"
            ]
          },
          {
            "name": "cf_pseudoreplication_overclaim",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_increase_across_markers"
            ]
          },
          {
            "name": "cf_infiltration_strength_decrease_direction",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_decrease_across_markers",
              "significant_decrease_strongest_markers_only",
              "consistent_decrease_not_statistically_robust"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-luna-pi",
      "modelName": "gpt-5.6-luna",
      "harness": "pi",
      "task": "gse297023_q6_age_immune_infiltration",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 2,
      "passed": true,
      "score": null,
      "durationS": 1116.962839,
      "cost": 0.5078667,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "composition_stable_with_age",
            "role": "gate",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "frequencies_largely_stable"
            ]
          },
          {
            "name": "age_signal_is_state_not_composition",
            "role": "gate",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "frequencies_uninformative_for_infiltration",
            "role": "gate",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_cannot_establish_infiltration"
            ]
          },
          {
            "name": "infiltration_increases_with_age",
            "role": "gate",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_with_age"
            ]
          },
          {
            "name": "infiltration_strength_calibrated",
            "role": "gate",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "significant_increase_strongest_markers_only",
              "consistent_increase_not_statistically_robust"
            ]
          },
          {
            "name": "genotype_marker_fraction_lower_in_pten",
            "role": "gate",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "lower_in_pten_deficient"
            ]
          },
          {
            "name": "cf_composition_shift_reading",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "major_compositional_shift_with_age",
              "increased_immune_fraction_with_age"
            ]
          },
          {
            "name": "cf_lineage_specific_composition_claim",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "myeloid_expansion_with_age",
              "lymphoid_expansion_with_age",
              "minor_shifts_in_some_cell_types"
            ]
          },
          {
            "name": "cf_immunosenescence_depletion_prior",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "immune_depletion_with_age"
            ]
          },
          {
            "name": "cf_age_modality_compositional",
            "role": "hard_fail",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_type_composition_shift",
              "both_equally",
              "composition_dominant_state_secondary"
            ]
          },
          {
            "name": "cf_overread_frequencies_as_infiltration",
            "role": "hard_fail",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_shows_no_infiltration_change",
              "single_cell_quantifies_infiltration",
              "frequencies_show_increased_infiltration",
              "frequencies_show_decreased_infiltration"
            ]
          },
          {
            "name": "cf_infiltration_decreases_immunosenescence",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "decreased_with_age"
            ]
          },
          {
            "name": "cf_infiltration_unchanged_scrna_overread",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "unchanged_with_age"
            ]
          },
          {
            "name": "cf_infiltration_indeterminate_unused_node",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "direction_indeterminate_from_data"
            ]
          },
          {
            "name": "cf_infiltration_mixed_direction",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "mixed_by_immune_cell_type"
            ]
          },
          {
            "name": "cf_infiltration_genotype_confusion",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_only_in_pten_deficient",
              "decreased_only_in_pten_deficient"
            ]
          },
          {
            "name": "cf_pten_infiltration_paper_text_recall",
            "role": "hard_fail",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "higher_in_pten_deficient",
              "higher_in_pten_deficient_only_in_young",
              "higher_in_pten_deficient_only_in_aged"
            ]
          },
          {
            "name": "cf_pseudoreplication_overclaim",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_increase_across_markers"
            ]
          },
          {
            "name": "cf_infiltration_strength_decrease_direction",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_decrease_across_markers",
              "significant_decrease_strongest_markers_only",
              "consistent_decrease_not_statistically_robust"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-luna-pi",
      "modelName": "gpt-5.6-luna",
      "harness": "pi",
      "task": "gse297023_q6_age_immune_infiltration",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 3,
      "passed": true,
      "score": null,
      "durationS": 842.612709,
      "cost": 0.48497570000000007,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "composition_stable_with_age",
            "role": "gate",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "frequencies_largely_stable"
            ]
          },
          {
            "name": "age_signal_is_state_not_composition",
            "role": "gate",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "frequencies_uninformative_for_infiltration",
            "role": "gate",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_cannot_establish_infiltration"
            ]
          },
          {
            "name": "infiltration_increases_with_age",
            "role": "gate",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_with_age"
            ]
          },
          {
            "name": "infiltration_strength_calibrated",
            "role": "gate",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "significant_increase_strongest_markers_only",
              "consistent_increase_not_statistically_robust"
            ]
          },
          {
            "name": "genotype_marker_fraction_lower_in_pten",
            "role": "gate",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "lower_in_pten_deficient"
            ]
          },
          {
            "name": "cf_composition_shift_reading",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "major_compositional_shift_with_age",
              "increased_immune_fraction_with_age"
            ]
          },
          {
            "name": "cf_lineage_specific_composition_claim",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "myeloid_expansion_with_age",
              "lymphoid_expansion_with_age",
              "minor_shifts_in_some_cell_types"
            ]
          },
          {
            "name": "cf_immunosenescence_depletion_prior",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "immune_depletion_with_age"
            ]
          },
          {
            "name": "cf_age_modality_compositional",
            "role": "hard_fail",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_type_composition_shift",
              "both_equally",
              "composition_dominant_state_secondary"
            ]
          },
          {
            "name": "cf_overread_frequencies_as_infiltration",
            "role": "hard_fail",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_shows_no_infiltration_change",
              "single_cell_quantifies_infiltration",
              "frequencies_show_increased_infiltration",
              "frequencies_show_decreased_infiltration"
            ]
          },
          {
            "name": "cf_infiltration_decreases_immunosenescence",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "decreased_with_age"
            ]
          },
          {
            "name": "cf_infiltration_unchanged_scrna_overread",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "unchanged_with_age"
            ]
          },
          {
            "name": "cf_infiltration_indeterminate_unused_node",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "direction_indeterminate_from_data"
            ]
          },
          {
            "name": "cf_infiltration_mixed_direction",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "mixed_by_immune_cell_type"
            ]
          },
          {
            "name": "cf_infiltration_genotype_confusion",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_only_in_pten_deficient",
              "decreased_only_in_pten_deficient"
            ]
          },
          {
            "name": "cf_pten_infiltration_paper_text_recall",
            "role": "hard_fail",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "higher_in_pten_deficient",
              "higher_in_pten_deficient_only_in_young",
              "higher_in_pten_deficient_only_in_aged"
            ]
          },
          {
            "name": "cf_pseudoreplication_overclaim",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_increase_across_markers"
            ]
          },
          {
            "name": "cf_infiltration_strength_decrease_direction",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_decrease_across_markers",
              "significant_decrease_strongest_markers_only",
              "consistent_decrease_not_statistically_robust"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-sol-pi",
      "modelName": "gpt-5.6-sol",
      "harness": "pi",
      "task": "gse155381_q4_pseudotime_delay",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 1,
      "passed": true,
      "score": null,
      "durationS": 1025.943526,
      "cost": 3.2208357499999996,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "pseudotime_direction_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "E17_chronological_match_weighted",
            "role": "additive",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pluripotency_exit_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pseudotime_direction_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_synchronous_with_monkey_EPI",
              "chimera_human_EPI_developmentally_advanced",
              "chimera_human_EPI_delayed_at_pre_implantation_only_resolved_post_implantation"
            ]
          },
          {
            "name": "E17_chronological_match_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "resembles_monkey_E17_Gast_EPI",
              "resembles_chimera_monkey_E17_EPI_specifically",
              "chimera_human_E17_EPI_more_advanced_than_monkey_E17_Gast"
            ]
          },
          {
            "name": "pluripotency_exit_paper_inverted_or_prior_driven",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_advances_pluripotency_exit_prematurely",
              "naive_pluripotency_program_specifically_engaged",
              "pluripotency_core_retained_via_KLF17_naive_marker_specifically"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-sol-pi",
      "modelName": "gpt-5.6-sol",
      "harness": "pi",
      "task": "gse155381_q4_pseudotime_delay",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 1487.817141,
      "cost": 8.00148375,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "pseudotime_direction_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "E17_chronological_match_weighted",
            "role": "additive",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pluripotency_exit_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pseudotime_direction_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_synchronous_with_monkey_EPI",
              "chimera_human_EPI_developmentally_advanced",
              "chimera_human_EPI_delayed_at_pre_implantation_only_resolved_post_implantation"
            ]
          },
          {
            "name": "E17_chronological_match_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "resembles_monkey_E17_Gast_EPI",
              "resembles_chimera_monkey_E17_EPI_specifically",
              "chimera_human_E17_EPI_more_advanced_than_monkey_E17_Gast"
            ]
          },
          {
            "name": "pluripotency_exit_paper_inverted_or_prior_driven",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_advances_pluripotency_exit_prematurely",
              "naive_pluripotency_program_specifically_engaged",
              "pluripotency_core_retained_via_KLF17_naive_marker_specifically"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-sol-pi",
      "modelName": "gpt-5.6-sol",
      "harness": "pi",
      "task": "gse155381_q4_pseudotime_delay",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 2892.363425,
      "cost": 6.39312425,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "pseudotime_direction_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "E17_chronological_match_weighted",
            "role": "additive",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pluripotency_exit_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pseudotime_direction_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_synchronous_with_monkey_EPI",
              "chimera_human_EPI_developmentally_advanced",
              "chimera_human_EPI_delayed_at_pre_implantation_only_resolved_post_implantation"
            ]
          },
          {
            "name": "E17_chronological_match_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "resembles_monkey_E17_Gast_EPI",
              "resembles_chimera_monkey_E17_EPI_specifically",
              "chimera_human_E17_EPI_more_advanced_than_monkey_E17_Gast"
            ]
          },
          {
            "name": "pluripotency_exit_paper_inverted_or_prior_driven",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_advances_pluripotency_exit_prematurely",
              "naive_pluripotency_program_specifically_engaged",
              "pluripotency_core_retained_via_KLF17_naive_marker_specifically"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-sol-pi",
      "modelName": "gpt-5.6-sol",
      "harness": "pi",
      "task": "gse155381_q5_chimera_monkey_perturbation",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 507.257451,
      "cost": 1.555839,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "deg_direction_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "comparison_subset_choice_weighted",
            "role": "additive",
            "field": "$.comparison_subset_choice",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "panel_resolved_strong_target_ccn1_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN1",
              "CYR61"
            ]
          },
          {
            "name": "panel_resolved_strong_target_ccn2_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN2",
              "CTGF"
            ]
          },
          {
            "name": "hippo_pathway_weighted",
            "role": "additive",
            "field": "$.hippo_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "tgf_beta_pathway_weighted",
            "role": "additive",
            "field": "$.tgf_beta_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "deg_direction_batch_effect_uncorrected",
            "role": "hard_fail",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_dominated_global",
              "up_dominated_in_chosen_subset"
            ]
          },
          {
            "name": "silent_panel_dropout_paired_with_directional_pathway_claim",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "hippo_pathway_direction": {
                "op": "in",
                "values": [
                  "hippo_pathway_upregulated",
                  "hippo_pathway_unchanged",
                  "wnt_pathway_specifically_perturbed",
                  "stress_response_pathway_dominantly_engaged",
                  "no_pathway_signal_above_confound",
                  "yap_tead_direct_target_panel_downregulated",
                  "yap_tead_panel_dependent_direction",
                  "hippo_pathway_downregulated"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-sol-pi",
      "modelName": "gpt-5.6-sol",
      "harness": "pi",
      "task": "gse155381_q5_chimera_monkey_perturbation",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 631.470341,
      "cost": 1.8473627499999998,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "deg_direction_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "comparison_subset_choice_weighted",
            "role": "additive",
            "field": "$.comparison_subset_choice",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "panel_resolved_strong_target_ccn1_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN1",
              "CYR61"
            ]
          },
          {
            "name": "panel_resolved_strong_target_ccn2_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN2",
              "CTGF"
            ]
          },
          {
            "name": "hippo_pathway_weighted",
            "role": "additive",
            "field": "$.hippo_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "tgf_beta_pathway_weighted",
            "role": "additive",
            "field": "$.tgf_beta_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "deg_direction_batch_effect_uncorrected",
            "role": "hard_fail",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_dominated_global",
              "up_dominated_in_chosen_subset"
            ]
          },
          {
            "name": "silent_panel_dropout_paired_with_directional_pathway_claim",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "hippo_pathway_direction": {
                "op": "in",
                "values": [
                  "hippo_pathway_upregulated",
                  "hippo_pathway_unchanged",
                  "wnt_pathway_specifically_perturbed",
                  "stress_response_pathway_dominantly_engaged",
                  "no_pathway_signal_above_confound",
                  "yap_tead_direct_target_panel_downregulated",
                  "yap_tead_panel_dependent_direction",
                  "hippo_pathway_downregulated"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-sol-pi",
      "modelName": "gpt-5.6-sol",
      "harness": "pi",
      "task": "gse155381_q5_chimera_monkey_perturbation",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 670.70003,
      "cost": 2.0424272499999994,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "deg_direction_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "comparison_subset_choice_weighted",
            "role": "additive",
            "field": "$.comparison_subset_choice",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "panel_resolved_strong_target_ccn1_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN1",
              "CYR61"
            ]
          },
          {
            "name": "panel_resolved_strong_target_ccn2_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN2",
              "CTGF"
            ]
          },
          {
            "name": "hippo_pathway_weighted",
            "role": "additive",
            "field": "$.hippo_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "tgf_beta_pathway_weighted",
            "role": "additive",
            "field": "$.tgf_beta_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "deg_direction_batch_effect_uncorrected",
            "role": "hard_fail",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_dominated_global",
              "up_dominated_in_chosen_subset"
            ]
          },
          {
            "name": "silent_panel_dropout_paired_with_directional_pathway_claim",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "hippo_pathway_direction": {
                "op": "in",
                "values": [
                  "hippo_pathway_upregulated",
                  "hippo_pathway_unchanged",
                  "wnt_pathway_specifically_perturbed",
                  "stress_response_pathway_dominantly_engaged",
                  "no_pathway_signal_above_confound",
                  "yap_tead_direct_target_panel_downregulated",
                  "yap_tead_panel_dependent_direction",
                  "hippo_pathway_downregulated"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-sol-pi",
      "modelName": "gpt-5.6-sol",
      "harness": "pi",
      "task": "gse155381_q7_xspecies_epi_communication",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 589.468868,
      "cost": 2.14541625,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "richness_weighted",
            "role": "additive",
            "field": "$.xspecies_signaling_richness",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_weighted",
            "role": "additive",
            "field": "$.dominant_signaling_axis",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_wnt_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "WNT_FZD_LRP_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_fgf_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "FGF_FGFR_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_notch_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "NOTCH_DLL_JAG_family"
                ]
              }
            }
          },
          {
            "name": "signaling_directionality_weighted",
            "role": "additive",
            "field": "$.signaling_directionality",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "top_signal_interaction_class_weighted",
            "role": "additive",
            "field": "$.top_signal_interaction_class",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-sol-pi",
      "modelName": "gpt-5.6-sol",
      "harness": "pi",
      "task": "gse155381_q7_xspecies_epi_communication",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 942.207377,
      "cost": 2.01525875,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "richness_weighted",
            "role": "additive",
            "field": "$.xspecies_signaling_richness",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_weighted",
            "role": "additive",
            "field": "$.dominant_signaling_axis",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_wnt_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "WNT_FZD_LRP_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_fgf_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "FGF_FGFR_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_notch_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "NOTCH_DLL_JAG_family"
                ]
              }
            }
          },
          {
            "name": "signaling_directionality_weighted",
            "role": "additive",
            "field": "$.signaling_directionality",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "top_signal_interaction_class_weighted",
            "role": "additive",
            "field": "$.top_signal_interaction_class",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-sol-pi",
      "modelName": "gpt-5.6-sol",
      "harness": "pi",
      "task": "gse155381_q7_xspecies_epi_communication",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 459.854694,
      "cost": 1.221252,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "richness_weighted",
            "role": "additive",
            "field": "$.xspecies_signaling_richness",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_weighted",
            "role": "additive",
            "field": "$.dominant_signaling_axis",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_wnt_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "WNT_FZD_LRP_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_fgf_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "FGF_FGFR_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_notch_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "NOTCH_DLL_JAG_family"
                ]
              }
            }
          },
          {
            "name": "signaling_directionality_weighted",
            "role": "additive",
            "field": "$.signaling_directionality",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "top_signal_interaction_class_weighted",
            "role": "additive",
            "field": "$.top_signal_interaction_class",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-sol-pi",
      "modelName": "gpt-5.6-sol",
      "harness": "pi",
      "task": "gse155381_q8_bidirectional_convergence",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 1404.385692,
      "cost": 6.176279999999998,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "chimera_human_identity_shift_target_weighted",
            "role": "additive",
            "field": "$.chimera_human_identity_shift_target",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_in_vivo_state_assignment_weighted",
            "role": "additive",
            "field": "$.chimera_human_in_vivo_state_assignment",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_shift_magnitude_weighted",
            "role": "additive",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_monkey_identity_shift_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "both_arms_null_contradicted_by_calibration",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "chimera_human_identity_shift_target": {
                "op": "equals",
                "values": [
                  "chimera_human_maintains_origin_identity_no_measurable_shift"
                ]
              },
              "chimera_monkey_identity_shift": {
                "op": "equals",
                "values": [
                  "chimera_monkey_unchanged_relative_to_control"
                ]
              }
            }
          },
          {
            "name": "chimera_monkey_paper_claim_on_confounded_input",
            "role": "hard_fail",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "value",
            "expected": "chimera_monkey_shifts_toward_chimera_human_identity"
          },
          {
            "name": "chimera_human_magnitude_wrong_attribution_signature",
            "role": "hard_fail",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "value",
            "op": "in",
            "expected": [
              "dramatic_complete_replacement_of_origin_identity",
              "magnitude_dominated_by_culture_or_metabolic_axis",
              "magnitude_dominated_by_developmental_stage_axis",
              "magnitude_dominated_by_cell_cycle_axis",
              "magnitude_dominated_by_batch_effect_axis"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-sol-pi",
      "modelName": "gpt-5.6-sol",
      "harness": "pi",
      "task": "gse155381_q8_bidirectional_convergence",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 1076.177169,
      "cost": 4.7061547500000005,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "chimera_human_identity_shift_target_weighted",
            "role": "additive",
            "field": "$.chimera_human_identity_shift_target",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_in_vivo_state_assignment_weighted",
            "role": "additive",
            "field": "$.chimera_human_in_vivo_state_assignment",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_shift_magnitude_weighted",
            "role": "additive",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_monkey_identity_shift_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "both_arms_null_contradicted_by_calibration",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "chimera_human_identity_shift_target": {
                "op": "equals",
                "values": [
                  "chimera_human_maintains_origin_identity_no_measurable_shift"
                ]
              },
              "chimera_monkey_identity_shift": {
                "op": "equals",
                "values": [
                  "chimera_monkey_unchanged_relative_to_control"
                ]
              }
            }
          },
          {
            "name": "chimera_monkey_paper_claim_on_confounded_input",
            "role": "hard_fail",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "value",
            "expected": "chimera_monkey_shifts_toward_chimera_human_identity"
          },
          {
            "name": "chimera_human_magnitude_wrong_attribution_signature",
            "role": "hard_fail",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "value",
            "op": "in",
            "expected": [
              "dramatic_complete_replacement_of_origin_identity",
              "magnitude_dominated_by_culture_or_metabolic_axis",
              "magnitude_dominated_by_developmental_stage_axis",
              "magnitude_dominated_by_cell_cycle_axis",
              "magnitude_dominated_by_batch_effect_axis"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-sol-pi",
      "modelName": "gpt-5.6-sol",
      "harness": "pi",
      "task": "gse155381_q8_bidirectional_convergence",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1136.78618,
      "cost": 5.805573,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "chimera_human_identity_shift_target_weighted",
            "role": "additive",
            "field": "$.chimera_human_identity_shift_target",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_in_vivo_state_assignment_weighted",
            "role": "additive",
            "field": "$.chimera_human_in_vivo_state_assignment",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_shift_magnitude_weighted",
            "role": "additive",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_monkey_identity_shift_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "both_arms_null_contradicted_by_calibration",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "chimera_human_identity_shift_target": {
                "op": "equals",
                "values": [
                  "chimera_human_maintains_origin_identity_no_measurable_shift"
                ]
              },
              "chimera_monkey_identity_shift": {
                "op": "equals",
                "values": [
                  "chimera_monkey_unchanged_relative_to_control"
                ]
              }
            }
          },
          {
            "name": "chimera_monkey_paper_claim_on_confounded_input",
            "role": "hard_fail",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "value",
            "expected": "chimera_monkey_shifts_toward_chimera_human_identity"
          },
          {
            "name": "chimera_human_magnitude_wrong_attribution_signature",
            "role": "hard_fail",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "value",
            "op": "in",
            "expected": [
              "dramatic_complete_replacement_of_origin_identity",
              "magnitude_dominated_by_culture_or_metabolic_axis",
              "magnitude_dominated_by_developmental_stage_axis",
              "magnitude_dominated_by_cell_cycle_axis",
              "magnitude_dominated_by_batch_effect_axis"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-sol-pi",
      "modelName": "gpt-5.6-sol",
      "harness": "pi",
      "task": "gse171524_ectopic_airway_rare_cell",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 1,
      "passed": true,
      "score": null,
      "durationS": null,
      "cost": null,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "focus_population_identity_gate",
            "role": "gate",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "chemosensory_marker_expressing_population_alpha"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "focus_population_marker_calls",
            "kind": "list",
            "matchKey": "marker",
            "k": 3,
            "expected": [
              {
                "key": "POU2F3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "CHAT",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "TRPM5",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "LRMP",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNAT3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "AVIL",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNG13",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              }
            ]
          },
          {
            "name": "focus_population_condition_distribution_gate",
            "role": "gate",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_enriched_with_substantial_control_presence",
              "covid_predominant",
              "covid_only",
              "covid_enriched_trend_not_significant"
            ]
          },
          {
            "name": "focus_population_location_gate",
            "role": "gate",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "ectopic_parenchyma_in_covid",
              "parenchyma_only",
              "bronchiolized_or_peribronchial_parenchyma",
              "airway_and_parenchyma"
            ]
          },
          {
            "name": "focus_population_recovery_mode_gate",
            "role": "gate",
            "field": "$.focus_population_recovery_mode",
            "kind": "value",
            "op": "in",
            "expected": [
              "continuum_not_discrete_cluster"
            ]
          },
          {
            "name": "focus_population_identity_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "ionocyte_like_beta",
              "neuroendocrine_like_gamma",
              "basal_stem_progenitor_like_delta",
              "aberrant_basaloid_transitional_like_epsilon",
              "neuronal_or_neural_gene_minor_population_zeta",
              "deuterosomal_ciliated_progenitor_like_eta",
              "hillock_transitional_like_theta",
              "low_quality_doublet_or_debris",
              "no_rare_cell_recovered",
              "not_assessed"
            ]
          },
          {
            "name": "cf_wrong_lineage_marker_called_positive",
            "role": "hard_fail",
            "field": "$.focus_population_marker_calls[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "focus_population_condition_distribution_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "equally_distributed",
              "control_predominant",
              "not_assessed"
            ]
          },
          {
            "name": "focus_population_location_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "airway_only",
              "not_assessed"
            ]
          },
          {
            "name": "cross_axis_canonical_identity_with_airway_only_location_incoherence",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "focus_population_identity": {
                "op": "in",
                "values": [
                  "chemosensory_marker_expressing_population_alpha"
                ]
              },
              "focus_population_location": {
                "op": "in",
                "values": [
                  "airway_only"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-sol-pi",
      "modelName": "gpt-5.6-sol",
      "harness": "pi",
      "task": "gse171524_ectopic_airway_rare_cell",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 1203.908161,
      "cost": 2.5042940000000002,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "focus_population_identity_gate",
            "role": "gate",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "chemosensory_marker_expressing_population_alpha"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "focus_population_marker_calls",
            "kind": "list",
            "matchKey": "marker",
            "k": 3,
            "expected": [
              {
                "key": "POU2F3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "CHAT",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "TRPM5",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "LRMP",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNAT3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "AVIL",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNG13",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              }
            ]
          },
          {
            "name": "focus_population_condition_distribution_gate",
            "role": "gate",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_enriched_with_substantial_control_presence",
              "covid_predominant",
              "covid_only",
              "covid_enriched_trend_not_significant"
            ]
          },
          {
            "name": "focus_population_location_gate",
            "role": "gate",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "ectopic_parenchyma_in_covid",
              "parenchyma_only",
              "bronchiolized_or_peribronchial_parenchyma",
              "airway_and_parenchyma"
            ]
          },
          {
            "name": "focus_population_recovery_mode_gate",
            "role": "gate",
            "field": "$.focus_population_recovery_mode",
            "kind": "value",
            "op": "in",
            "expected": [
              "continuum_not_discrete_cluster"
            ]
          },
          {
            "name": "focus_population_identity_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "ionocyte_like_beta",
              "neuroendocrine_like_gamma",
              "basal_stem_progenitor_like_delta",
              "aberrant_basaloid_transitional_like_epsilon",
              "neuronal_or_neural_gene_minor_population_zeta",
              "deuterosomal_ciliated_progenitor_like_eta",
              "hillock_transitional_like_theta",
              "low_quality_doublet_or_debris",
              "no_rare_cell_recovered",
              "not_assessed"
            ]
          },
          {
            "name": "cf_wrong_lineage_marker_called_positive",
            "role": "hard_fail",
            "field": "$.focus_population_marker_calls[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "focus_population_condition_distribution_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "equally_distributed",
              "control_predominant",
              "not_assessed"
            ]
          },
          {
            "name": "focus_population_location_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "airway_only",
              "not_assessed"
            ]
          },
          {
            "name": "cross_axis_canonical_identity_with_airway_only_location_incoherence",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "focus_population_identity": {
                "op": "in",
                "values": [
                  "chemosensory_marker_expressing_population_alpha"
                ]
              },
              "focus_population_location": {
                "op": "in",
                "values": [
                  "airway_only"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-sol-pi",
      "modelName": "gpt-5.6-sol",
      "harness": "pi",
      "task": "gse171524_ectopic_airway_rare_cell",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 2515.731609,
      "cost": 7.2974862499999995,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "focus_population_identity_gate",
            "role": "gate",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "chemosensory_marker_expressing_population_alpha"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "focus_population_marker_calls",
            "kind": "list",
            "matchKey": "marker",
            "k": 3,
            "expected": [
              {
                "key": "POU2F3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "CHAT",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "TRPM5",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "LRMP",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNAT3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "AVIL",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNG13",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              }
            ]
          },
          {
            "name": "focus_population_condition_distribution_gate",
            "role": "gate",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_enriched_with_substantial_control_presence",
              "covid_predominant",
              "covid_only",
              "covid_enriched_trend_not_significant"
            ]
          },
          {
            "name": "focus_population_location_gate",
            "role": "gate",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "ectopic_parenchyma_in_covid",
              "parenchyma_only",
              "bronchiolized_or_peribronchial_parenchyma",
              "airway_and_parenchyma"
            ]
          },
          {
            "name": "focus_population_recovery_mode_gate",
            "role": "gate",
            "field": "$.focus_population_recovery_mode",
            "kind": "value",
            "op": "in",
            "expected": [
              "continuum_not_discrete_cluster"
            ]
          },
          {
            "name": "focus_population_identity_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "ionocyte_like_beta",
              "neuroendocrine_like_gamma",
              "basal_stem_progenitor_like_delta",
              "aberrant_basaloid_transitional_like_epsilon",
              "neuronal_or_neural_gene_minor_population_zeta",
              "deuterosomal_ciliated_progenitor_like_eta",
              "hillock_transitional_like_theta",
              "low_quality_doublet_or_debris",
              "no_rare_cell_recovered",
              "not_assessed"
            ]
          },
          {
            "name": "cf_wrong_lineage_marker_called_positive",
            "role": "hard_fail",
            "field": "$.focus_population_marker_calls[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "focus_population_condition_distribution_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "equally_distributed",
              "control_predominant",
              "not_assessed"
            ]
          },
          {
            "name": "focus_population_location_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "airway_only",
              "not_assessed"
            ]
          },
          {
            "name": "cross_axis_canonical_identity_with_airway_only_location_incoherence",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "focus_population_identity": {
                "op": "in",
                "values": [
                  "chemosensory_marker_expressing_population_alpha"
                ]
              },
              "focus_population_location": {
                "op": "in",
                "values": [
                  "airway_only"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-sol-pi",
      "modelName": "gpt-5.6-sol",
      "harness": "pi",
      "task": "gse171524_pathological_fibroblast_master_regulator_druggable_targets",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 1,
      "passed": true,
      "score": null,
      "durationS": 3638.937917,
      "cost": 1.92250075,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "disease_associated_fibroblast_subtype_weighted",
            "role": "additive",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "mr_tf_junb_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_alpha"
            ]
          },
          {
            "name": "mr_tf_jund_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_beta"
            ]
          },
          {
            "name": "drug_mmp14_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_alpha"
            ]
          },
          {
            "name": "drug_stat3_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_beta"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_weighted",
            "role": "additive",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "disease_associated_fibroblast_subtype_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "value",
            "op": "in",
            "expected": [
              "subtype_lambda",
              "subtype_epsilon",
              "subtype_iota",
              "subtype_beta",
              "subtype_eta",
              "subtype_zeta",
              "subtype_not_assessed"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "simple_de_ranking_by_log_fc",
              "marker_based_de_only",
              "not_computed"
            ]
          },
          {
            "name": "top_master_regulator_transcription_factors_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_not_computed"
            ]
          },
          {
            "name": "top_druggable_proteins_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_not_assessed"
            ]
          },
          {
            "name": "cross_axis_circular_marker_in_druggable_proteins",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_gamma",
              "drug_target_delta",
              "drug_target_eta"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-sol-pi",
      "modelName": "gpt-5.6-sol",
      "harness": "pi",
      "task": "gse171524_pathological_fibroblast_master_regulator_druggable_targets",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 2,
      "passed": true,
      "score": null,
      "durationS": 2859.980223,
      "cost": 1.273927,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "disease_associated_fibroblast_subtype_weighted",
            "role": "additive",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "mr_tf_junb_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_alpha"
            ]
          },
          {
            "name": "mr_tf_jund_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_beta"
            ]
          },
          {
            "name": "drug_mmp14_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_alpha"
            ]
          },
          {
            "name": "drug_stat3_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_beta"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_weighted",
            "role": "additive",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "disease_associated_fibroblast_subtype_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "value",
            "op": "in",
            "expected": [
              "subtype_lambda",
              "subtype_epsilon",
              "subtype_iota",
              "subtype_beta",
              "subtype_eta",
              "subtype_zeta",
              "subtype_not_assessed"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "simple_de_ranking_by_log_fc",
              "marker_based_de_only",
              "not_computed"
            ]
          },
          {
            "name": "top_master_regulator_transcription_factors_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_not_computed"
            ]
          },
          {
            "name": "top_druggable_proteins_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_not_assessed"
            ]
          },
          {
            "name": "cross_axis_circular_marker_in_druggable_proteins",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_gamma",
              "drug_target_delta",
              "drug_target_eta"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-sol-pi",
      "modelName": "gpt-5.6-sol",
      "harness": "pi",
      "task": "gse171524_pathological_fibroblast_master_regulator_druggable_targets",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 3,
      "passed": true,
      "score": null,
      "durationS": 1371.806292,
      "cost": 1.54742325,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "disease_associated_fibroblast_subtype_weighted",
            "role": "additive",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "mr_tf_junb_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_alpha"
            ]
          },
          {
            "name": "mr_tf_jund_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_beta"
            ]
          },
          {
            "name": "drug_mmp14_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_alpha"
            ]
          },
          {
            "name": "drug_stat3_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_beta"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_weighted",
            "role": "additive",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "disease_associated_fibroblast_subtype_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "value",
            "op": "in",
            "expected": [
              "subtype_lambda",
              "subtype_epsilon",
              "subtype_iota",
              "subtype_beta",
              "subtype_eta",
              "subtype_zeta",
              "subtype_not_assessed"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "simple_de_ranking_by_log_fc",
              "marker_based_de_only",
              "not_computed"
            ]
          },
          {
            "name": "top_master_regulator_transcription_factors_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_not_computed"
            ]
          },
          {
            "name": "top_druggable_proteins_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_not_assessed"
            ]
          },
          {
            "name": "cross_axis_circular_marker_in_druggable_proteins",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_gamma",
              "drug_target_delta",
              "drug_target_eta"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-sol-pi",
      "modelName": "gpt-5.6-sol",
      "harness": "pi",
      "task": "gse171524_plasma_bcr_neutralizing_clonotype",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": null,
      "cost": null,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dominant_pairing_heavy_gate",
            "role": "gate",
            "field": "$.dominant_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV4-34"
            ]
          },
          {
            "name": "dominant_pairing_light_gate",
            "role": "gate",
            "field": "$.dominant_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGLV3-19"
            ]
          },
          {
            "name": "dominant_neutralizer_match_gate",
            "role": "gate",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_known_neutralizer_match"
            ]
          },
          {
            "name": "neutralizer_pairing_heavy_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV1-18"
            ]
          },
          {
            "name": "neutralizer_pairing_light_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGKV3-20"
            ]
          },
          {
            "name": "neutralizer_recurrence_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "observed_in_2_to_5_donors"
            ]
          },
          {
            "name": "neutralizer_rank_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "top_5_pairing",
              "top_6_to_20_pairing"
            ]
          },
          {
            "name": "dominant_clone_isotype_gate",
            "role": "gate",
            "field": "$.dominant_clone_isotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "IgG"
            ]
          },
          {
            "name": "blineage_class_switch_contrast_gate",
            "role": "gate",
            "field": "$.blineage_class_switch_contrast",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_class_switched_control_higher_unswitched",
              "class_switched_in_both_cohorts"
            ]
          },
          {
            "name": "dominant_match_narrative_trap_or_null",
            "role": "hard_fail",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_rbd_neutralizer",
              "matches_known_autoreactive_clone",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_heavy",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_light",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_recurrence_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_rank_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "cross_axis_found_dominant_but_failed_to_scan_for_embedded_neutralizer",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_pairing_neutralizer_match": {
                "op": "in",
                "values": [
                  "no_known_neutralizer_match"
                ]
              },
              "neutralizer_pairing_heavy": {
                "op": "in",
                "values": [
                  "none_detected"
                ]
              }
            }
          },
          {
            "name": "neutralizer_pairing_not_exactly_the_s309_pair",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "op": "not",
            "expected": {
              "op": "and",
              "args": [
                {
                  "op": "field",
                  "name": "neutralizer_pairing_heavy",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGHV1-18"
                    ]
                  }
                },
                {
                  "op": "field",
                  "name": "neutralizer_pairing_light",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGKV3-20"
                    ]
                  }
                }
              ]
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-sol-pi",
      "modelName": "gpt-5.6-sol",
      "harness": "pi",
      "task": "gse171524_plasma_bcr_neutralizing_clonotype",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": null,
      "cost": null,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dominant_pairing_heavy_gate",
            "role": "gate",
            "field": "$.dominant_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV4-34"
            ]
          },
          {
            "name": "dominant_pairing_light_gate",
            "role": "gate",
            "field": "$.dominant_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGLV3-19"
            ]
          },
          {
            "name": "dominant_neutralizer_match_gate",
            "role": "gate",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_known_neutralizer_match"
            ]
          },
          {
            "name": "neutralizer_pairing_heavy_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV1-18"
            ]
          },
          {
            "name": "neutralizer_pairing_light_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGKV3-20"
            ]
          },
          {
            "name": "neutralizer_recurrence_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "observed_in_2_to_5_donors"
            ]
          },
          {
            "name": "neutralizer_rank_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "top_5_pairing",
              "top_6_to_20_pairing"
            ]
          },
          {
            "name": "dominant_clone_isotype_gate",
            "role": "gate",
            "field": "$.dominant_clone_isotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "IgG"
            ]
          },
          {
            "name": "blineage_class_switch_contrast_gate",
            "role": "gate",
            "field": "$.blineage_class_switch_contrast",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_class_switched_control_higher_unswitched",
              "class_switched_in_both_cohorts"
            ]
          },
          {
            "name": "dominant_match_narrative_trap_or_null",
            "role": "hard_fail",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_rbd_neutralizer",
              "matches_known_autoreactive_clone",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_heavy",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_light",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_recurrence_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_rank_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "cross_axis_found_dominant_but_failed_to_scan_for_embedded_neutralizer",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_pairing_neutralizer_match": {
                "op": "in",
                "values": [
                  "no_known_neutralizer_match"
                ]
              },
              "neutralizer_pairing_heavy": {
                "op": "in",
                "values": [
                  "none_detected"
                ]
              }
            }
          },
          {
            "name": "neutralizer_pairing_not_exactly_the_s309_pair",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "op": "not",
            "expected": {
              "op": "and",
              "args": [
                {
                  "op": "field",
                  "name": "neutralizer_pairing_heavy",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGHV1-18"
                    ]
                  }
                },
                {
                  "op": "field",
                  "name": "neutralizer_pairing_light",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGKV3-20"
                    ]
                  }
                }
              ]
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-sol-pi",
      "modelName": "gpt-5.6-sol",
      "harness": "pi",
      "task": "gse171524_plasma_bcr_neutralizing_clonotype",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": null,
      "cost": null,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dominant_pairing_heavy_gate",
            "role": "gate",
            "field": "$.dominant_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV4-34"
            ]
          },
          {
            "name": "dominant_pairing_light_gate",
            "role": "gate",
            "field": "$.dominant_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGLV3-19"
            ]
          },
          {
            "name": "dominant_neutralizer_match_gate",
            "role": "gate",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_known_neutralizer_match"
            ]
          },
          {
            "name": "neutralizer_pairing_heavy_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV1-18"
            ]
          },
          {
            "name": "neutralizer_pairing_light_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGKV3-20"
            ]
          },
          {
            "name": "neutralizer_recurrence_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "observed_in_2_to_5_donors"
            ]
          },
          {
            "name": "neutralizer_rank_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "top_5_pairing",
              "top_6_to_20_pairing"
            ]
          },
          {
            "name": "dominant_clone_isotype_gate",
            "role": "gate",
            "field": "$.dominant_clone_isotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "IgG"
            ]
          },
          {
            "name": "blineage_class_switch_contrast_gate",
            "role": "gate",
            "field": "$.blineage_class_switch_contrast",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_class_switched_control_higher_unswitched",
              "class_switched_in_both_cohorts"
            ]
          },
          {
            "name": "dominant_match_narrative_trap_or_null",
            "role": "hard_fail",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_rbd_neutralizer",
              "matches_known_autoreactive_clone",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_heavy",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_light",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_recurrence_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_rank_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "cross_axis_found_dominant_but_failed_to_scan_for_embedded_neutralizer",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_pairing_neutralizer_match": {
                "op": "in",
                "values": [
                  "no_known_neutralizer_match"
                ]
              },
              "neutralizer_pairing_heavy": {
                "op": "in",
                "values": [
                  "none_detected"
                ]
              }
            }
          },
          {
            "name": "neutralizer_pairing_not_exactly_the_s309_pair",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "op": "not",
            "expected": {
              "op": "and",
              "args": [
                {
                  "op": "field",
                  "name": "neutralizer_pairing_heavy",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGHV1-18"
                    ]
                  }
                },
                {
                  "op": "field",
                  "name": "neutralizer_pairing_light",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGKV3-20"
                    ]
                  }
                }
              ]
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-sol-pi",
      "modelName": "gpt-5.6-sol",
      "harness": "pi",
      "task": "q6_myeloid_subset_restricted_de",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 1,
      "passed": true,
      "score": null,
      "durationS": null,
      "cost": null,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "myeloid_subset_with_strongest_change_gate",
            "role": "gate",
            "field": "$.myeloid_subset_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "subset_gamma"
            ]
          },
          {
            "name": "gene_with_strongest_change_gate",
            "role": "gate",
            "field": "$.gene_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "AXL",
              "PROS1"
            ]
          },
          {
            "name": "direction_of_change_gate",
            "role": "gate",
            "field": "$.direction_of_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "down_in_covid"
            ]
          },
          {
            "name": "biological_function_interpretation_gate",
            "role": "gate",
            "field": "$.biological_function_interpretation",
            "kind": "value",
            "op": "in",
            "expected": [
              "efferocytosis",
              "phagocytosis"
            ]
          },
          {
            "name": "myeloid_compartment_abundance_gate",
            "role": "gate",
            "field": "$.myeloid_compartment_abundance",
            "kind": "value",
            "op": "in",
            "expected": [
              "expands_in_covid"
            ]
          },
          {
            "name": "myeloid_expansion_localization_gate",
            "role": "gate",
            "field": "$.myeloid_expansion_localization",
            "kind": "value",
            "op": "in",
            "expected": [
              "recruited_driven_resident_dysfunction"
            ]
          },
          {
            "name": "dominant_signature_gate",
            "role": "gate",
            "field": "$.dominant_signature",
            "kind": "value",
            "op": "in",
            "expected": [
              "technical_or_agonal_artifact"
            ]
          },
          {
            "name": "dominant_signature_direction_gate",
            "role": "gate",
            "field": "$.dominant_signature_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_in_covid"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-sol-pi",
      "modelName": "gpt-5.6-sol",
      "harness": "pi",
      "task": "q6_myeloid_subset_restricted_de",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 2,
      "passed": true,
      "score": null,
      "durationS": 1521.656564,
      "cost": 1.1412117499999996,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "myeloid_subset_with_strongest_change_gate",
            "role": "gate",
            "field": "$.myeloid_subset_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "subset_gamma"
            ]
          },
          {
            "name": "gene_with_strongest_change_gate",
            "role": "gate",
            "field": "$.gene_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "AXL",
              "PROS1"
            ]
          },
          {
            "name": "direction_of_change_gate",
            "role": "gate",
            "field": "$.direction_of_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "down_in_covid"
            ]
          },
          {
            "name": "biological_function_interpretation_gate",
            "role": "gate",
            "field": "$.biological_function_interpretation",
            "kind": "value",
            "op": "in",
            "expected": [
              "efferocytosis",
              "phagocytosis"
            ]
          },
          {
            "name": "myeloid_compartment_abundance_gate",
            "role": "gate",
            "field": "$.myeloid_compartment_abundance",
            "kind": "value",
            "op": "in",
            "expected": [
              "expands_in_covid"
            ]
          },
          {
            "name": "myeloid_expansion_localization_gate",
            "role": "gate",
            "field": "$.myeloid_expansion_localization",
            "kind": "value",
            "op": "in",
            "expected": [
              "recruited_driven_resident_dysfunction"
            ]
          },
          {
            "name": "dominant_signature_gate",
            "role": "gate",
            "field": "$.dominant_signature",
            "kind": "value",
            "op": "in",
            "expected": [
              "technical_or_agonal_artifact"
            ]
          },
          {
            "name": "dominant_signature_direction_gate",
            "role": "gate",
            "field": "$.dominant_signature_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_in_covid"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-sol-pi",
      "modelName": "gpt-5.6-sol",
      "harness": "pi",
      "task": "q6_myeloid_subset_restricted_de",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1022.83934,
      "cost": 1.6390309999999997,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "myeloid_subset_with_strongest_change_gate",
            "role": "gate",
            "field": "$.myeloid_subset_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "subset_gamma"
            ]
          },
          {
            "name": "gene_with_strongest_change_gate",
            "role": "gate",
            "field": "$.gene_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "AXL",
              "PROS1"
            ]
          },
          {
            "name": "direction_of_change_gate",
            "role": "gate",
            "field": "$.direction_of_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "down_in_covid"
            ]
          },
          {
            "name": "biological_function_interpretation_gate",
            "role": "gate",
            "field": "$.biological_function_interpretation",
            "kind": "value",
            "op": "in",
            "expected": [
              "efferocytosis",
              "phagocytosis"
            ]
          },
          {
            "name": "myeloid_compartment_abundance_gate",
            "role": "gate",
            "field": "$.myeloid_compartment_abundance",
            "kind": "value",
            "op": "in",
            "expected": [
              "expands_in_covid"
            ]
          },
          {
            "name": "myeloid_expansion_localization_gate",
            "role": "gate",
            "field": "$.myeloid_expansion_localization",
            "kind": "value",
            "op": "in",
            "expected": [
              "recruited_driven_resident_dysfunction"
            ]
          },
          {
            "name": "dominant_signature_gate",
            "role": "gate",
            "field": "$.dominant_signature",
            "kind": "value",
            "op": "in",
            "expected": [
              "technical_or_agonal_artifact"
            ]
          },
          {
            "name": "dominant_signature_direction_gate",
            "role": "gate",
            "field": "$.dominant_signature_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_in_covid"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-sol-pi",
      "modelName": "gpt-5.6-sol",
      "harness": "pi",
      "task": "snrnaseq_per_donor_composition_shifts_lethal_covid_lung",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 1,
      "passed": true,
      "score": null,
      "durationS": 1734.912516,
      "cost": 2.303298,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dict_match",
            "role": "gate",
            "field": "cell_type_direction_map",
            "kind": "dict",
            "expected": {
              "Myeloid": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "Epithelia": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Fibroblasts": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "T_NK": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "B_plasma": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Endothelial": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Mast": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "APC": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Neuronal": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              }
            }
          },
          {
            "name": "over_merge_immune_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Immune",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_leukocyte_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Leukocyte",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_lymphoid_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Lymphoid",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_stromal_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Stromal",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-sol-pi",
      "modelName": "gpt-5.6-sol",
      "harness": "pi",
      "task": "snrnaseq_per_donor_composition_shifts_lethal_covid_lung",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 514.414633,
      "cost": 0.9846130000000001,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dict_match",
            "role": "gate",
            "field": "cell_type_direction_map",
            "kind": "dict",
            "expected": {
              "Myeloid": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "Epithelia": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Fibroblasts": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "T_NK": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "B_plasma": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Endothelial": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Mast": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "APC": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Neuronal": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              }
            }
          },
          {
            "name": "over_merge_immune_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Immune",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_leukocyte_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Leukocyte",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_lymphoid_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Lymphoid",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_stromal_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Stromal",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-sol-pi",
      "modelName": "gpt-5.6-sol",
      "harness": "pi",
      "task": "snrnaseq_per_donor_composition_shifts_lethal_covid_lung",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1175.185965,
      "cost": 1.0093165,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dict_match",
            "role": "gate",
            "field": "cell_type_direction_map",
            "kind": "dict",
            "expected": {
              "Myeloid": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "Epithelia": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Fibroblasts": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "T_NK": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "B_plasma": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Endothelial": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Mast": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "APC": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Neuronal": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              }
            }
          },
          {
            "name": "over_merge_immune_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Immune",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_leukocyte_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Leukocyte",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_lymphoid_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Lymphoid",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_stromal_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Stromal",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-sol-pi",
      "modelName": "gpt-5.6-sol",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_enriched_apc_states_v2",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 1589.769138,
      "cost": 2.664124749999999,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "top_3_cluster_enriched_apc_states",
            "kind": "list",
            "matchKey": "apc_state",
            "k": 3,
            "expected": [
              {
                "key": "plasmacytoid_DC",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "T_cell_stimulatory_role_supported",
                      "role_direction_unresolved"
                    ],
                    "canonical": "T_cell_stimulatory_role_supported"
                  }
                }
              },
              {
                "key": "C1qhigh_lipid_associated_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "C1qhigh_inflammatory_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "plasma_cell",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "cohort_significant_per_patient_inconsistent"
                    ],
                    "canonical": "cohort_significant_per_patient_inconsistent"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CD80_CD86": {
                "op": "in",
                "value": [
                  "mreg_DC",
                  "shared_across_APC_states"
                ]
              },
              "PD_L1_in_APC_clusters": {
                "op": "in",
                "value": [
                  "C1qhigh_macrophage_either_subtype",
                  "shared_across_APC_states",
                  "mreg_DC"
                ]
              },
              "type_I_IFN": {
                "op": "in",
                "value": [
                  "plasmacytoid_DC",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "cf_mreg_dc_claimed_enriched",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "mreg_DC",
              "mature_DC_enriched_in_immunoregulatory_molecules"
            ]
          },
          {
            "name": "cf_framework_offcompartment_or_generic_substitution",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "cDC1",
              "Maier_mregDC",
              "Cheng_C1QC_TAM",
              "Cheng_SPP1_TAM",
              "Cheng_FOLR2_TAM",
              "Villani_pDC",
              "Zhang_inflammatory_monocyte",
              "LAMP3_pos_DC",
              "myeloid_derived_suppressor_cell",
              "MDSC",
              "tumor_associated_neutrophil",
              "mast_cell",
              "APC",
              "antigen_presenting_cell",
              "macrophage",
              "dendritic_cell",
              "tumor_associated_macrophage",
              "TAM"
            ]
          },
          {
            "name": "cf_direction_commit_on_unresolvable_slot",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_not_resolvable_hedge",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].functional_attribution_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_type_I_IFN_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.type_I_IFN",
            "kind": "value",
            "op": "in",
            "expected": [
              "mreg_DC",
              "C1qhigh_macrophage_either_subtype"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-sol-pi",
      "modelName": "gpt-5.6-sol",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_enriched_apc_states_v2",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 3635.172751,
      "cost": 3.8214699999999997,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "top_3_cluster_enriched_apc_states",
            "kind": "list",
            "matchKey": "apc_state",
            "k": 3,
            "expected": [
              {
                "key": "plasmacytoid_DC",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "T_cell_stimulatory_role_supported",
                      "role_direction_unresolved"
                    ],
                    "canonical": "T_cell_stimulatory_role_supported"
                  }
                }
              },
              {
                "key": "C1qhigh_lipid_associated_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "C1qhigh_inflammatory_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "plasma_cell",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "cohort_significant_per_patient_inconsistent"
                    ],
                    "canonical": "cohort_significant_per_patient_inconsistent"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CD80_CD86": {
                "op": "in",
                "value": [
                  "mreg_DC",
                  "shared_across_APC_states"
                ]
              },
              "PD_L1_in_APC_clusters": {
                "op": "in",
                "value": [
                  "C1qhigh_macrophage_either_subtype",
                  "shared_across_APC_states",
                  "mreg_DC"
                ]
              },
              "type_I_IFN": {
                "op": "in",
                "value": [
                  "plasmacytoid_DC",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "cf_mreg_dc_claimed_enriched",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "mreg_DC",
              "mature_DC_enriched_in_immunoregulatory_molecules"
            ]
          },
          {
            "name": "cf_framework_offcompartment_or_generic_substitution",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "cDC1",
              "Maier_mregDC",
              "Cheng_C1QC_TAM",
              "Cheng_SPP1_TAM",
              "Cheng_FOLR2_TAM",
              "Villani_pDC",
              "Zhang_inflammatory_monocyte",
              "LAMP3_pos_DC",
              "myeloid_derived_suppressor_cell",
              "MDSC",
              "tumor_associated_neutrophil",
              "mast_cell",
              "APC",
              "antigen_presenting_cell",
              "macrophage",
              "dendritic_cell",
              "tumor_associated_macrophage",
              "TAM"
            ]
          },
          {
            "name": "cf_direction_commit_on_unresolvable_slot",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_not_resolvable_hedge",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].functional_attribution_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_type_I_IFN_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.type_I_IFN",
            "kind": "value",
            "op": "in",
            "expected": [
              "mreg_DC",
              "C1qhigh_macrophage_either_subtype"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-sol-pi",
      "modelName": "gpt-5.6-sol",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_enriched_apc_states_v2",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1858.231427,
      "cost": 4.38869,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "top_3_cluster_enriched_apc_states",
            "kind": "list",
            "matchKey": "apc_state",
            "k": 3,
            "expected": [
              {
                "key": "plasmacytoid_DC",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "T_cell_stimulatory_role_supported",
                      "role_direction_unresolved"
                    ],
                    "canonical": "T_cell_stimulatory_role_supported"
                  }
                }
              },
              {
                "key": "C1qhigh_lipid_associated_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "C1qhigh_inflammatory_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "plasma_cell",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "cohort_significant_per_patient_inconsistent"
                    ],
                    "canonical": "cohort_significant_per_patient_inconsistent"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CD80_CD86": {
                "op": "in",
                "value": [
                  "mreg_DC",
                  "shared_across_APC_states"
                ]
              },
              "PD_L1_in_APC_clusters": {
                "op": "in",
                "value": [
                  "C1qhigh_macrophage_either_subtype",
                  "shared_across_APC_states",
                  "mreg_DC"
                ]
              },
              "type_I_IFN": {
                "op": "in",
                "value": [
                  "plasmacytoid_DC",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "cf_mreg_dc_claimed_enriched",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "mreg_DC",
              "mature_DC_enriched_in_immunoregulatory_molecules"
            ]
          },
          {
            "name": "cf_framework_offcompartment_or_generic_substitution",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "cDC1",
              "Maier_mregDC",
              "Cheng_C1QC_TAM",
              "Cheng_SPP1_TAM",
              "Cheng_FOLR2_TAM",
              "Villani_pDC",
              "Zhang_inflammatory_monocyte",
              "LAMP3_pos_DC",
              "myeloid_derived_suppressor_cell",
              "MDSC",
              "tumor_associated_neutrophil",
              "mast_cell",
              "APC",
              "antigen_presenting_cell",
              "macrophage",
              "dendritic_cell",
              "tumor_associated_macrophage",
              "TAM"
            ]
          },
          {
            "name": "cf_direction_commit_on_unresolvable_slot",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_not_resolvable_hedge",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].functional_attribution_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_type_I_IFN_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.type_I_IFN",
            "kind": "value",
            "op": "in",
            "expected": [
              "mreg_DC",
              "C1qhigh_macrophage_either_subtype"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-sol-pi",
      "modelName": "gpt-5.6-sol",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_tumor_state_causation_v5",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 3503.262882,
      "cost": 2.941752499999999,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "enriched_tumor_states",
            "kind": "list",
            "matchKey": "tumor_state",
            "k": 3,
            "expected": [
              {
                "key": "immune_response",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "direction_unresolved_cross_sectional"
                    ],
                    "canonical": "direction_unresolved_cross_sectional"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "IFN_response_axis",
                      "broad_antigen_presentation_MHC"
                    ],
                    "canonical": "IFN_response_axis"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "enrichment_diminished_by_decontamination",
                      "threshold_sensitive"
                    ],
                    "canonical": "enrichment_diminished_by_decontamination"
                  }
                }
              },
              {
                "key": "stress_hypoxia",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "data_supports_cell_intrinsic_cause"
                    ],
                    "canonical": "data_supports_cell_intrinsic_cause"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "HIF_hypoxia_program_specific"
                    ],
                    "canonical": "HIF_hypoxia_program_specific"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "robust_across_methods",
                      "robust_after_decontamination"
                    ],
                    "canonical": "robust_across_methods"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CXCL9_CXCL10": {
                "op": "in",
                "value": [
                  "immune_response_tumor_state"
                ]
              },
              "CCL5": {
                "op": "in",
                "value": [
                  "shared_across_tumor_states",
                  "partner_lymphocyte_derived",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "gate_conjugate_gate_confounder",
            "role": "gate",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "partner_lymphocyte_capture"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "antigen_presentation_decomposition",
            "kind": "dict",
            "expected": {
              "MHC_class_I": {
                "op": "in",
                "value": [
                  "co_regulated_with_dominant_program",
                  "tumor_intrinsic_enriched_independent"
                ]
              },
              "MHC_class_II": {
                "op": "in",
                "value": [
                  "partner_cell_attributable",
                  "flat_not_enriched",
                  "method_sensitive_no_robust_call",
                  "co_regulated_with_dominant_program"
                ]
              }
            }
          },
          {
            "name": "cf_non_robust_state_claimed_enriched",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "transitory_melanocytic",
              "intermediate_melanocytic",
              "melanocytic",
              "MITF_high",
              "neural_crest_like",
              "mitotic",
              "stress_p53",
              "undifferentiated",
              "dedifferentiated",
              "mesenchymal_tumor",
              "AXL_high_tumor",
              "invasive_tumor",
              "senescence_like",
              "EMT_transition",
              "cancer_stem_cell_like",
              "oxidative_stress_tumor"
            ]
          },
          {
            "name": "cf_framework_generic_or_offtissue_substitution",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "Tsoi_undifferentiated",
              "Tsoi_neural_crest_like",
              "Tsoi_transitory",
              "Tsoi_melanocytic",
              "Rambow_NCSC",
              "Wouters_mesenchymal",
              "Pozniak_TCF4_dependent",
              "tumor_cell",
              "melanoma_cell",
              "malignant_cell",
              "tumor_subpopulation",
              "keratinocyte",
              "hepatocyte",
              "lung_adenocarcinoma_cell",
              "colon_adenocarcinoma_cell",
              "Tirosh_cell_cycle_program",
              "Jerby_Arnon_program",
              "glioblastoma_cell",
              "breast_carcinoma_cell"
            ]
          },
          {
            "name": "cf_immune_response_direction_committed",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_immune_robustness_recall",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_ambiguous_hedge",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].causation_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_stress_generic_stress_response_evidence",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_conjugate_confounder_contamination_denied",
            "role": "hard_fail",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected"
            ]
          },
          {
            "name": "cf_ccl5_recall_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CCL5",
            "kind": "avoid",
            "expected": [
              "immune_response_tumor_state"
            ]
          },
          {
            "name": "cf_cxcl9_cxcl10_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CXCL9_CXCL10",
            "kind": "avoid",
            "expected": [
              "stress_hypoxia_tumor_state"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-sol-pi",
      "modelName": "gpt-5.6-sol",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_tumor_state_causation_v5",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 2181.744202,
      "cost": 2.5416805000000005,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "enriched_tumor_states",
            "kind": "list",
            "matchKey": "tumor_state",
            "k": 3,
            "expected": [
              {
                "key": "immune_response",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "direction_unresolved_cross_sectional"
                    ],
                    "canonical": "direction_unresolved_cross_sectional"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "IFN_response_axis",
                      "broad_antigen_presentation_MHC"
                    ],
                    "canonical": "IFN_response_axis"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "enrichment_diminished_by_decontamination",
                      "threshold_sensitive"
                    ],
                    "canonical": "enrichment_diminished_by_decontamination"
                  }
                }
              },
              {
                "key": "stress_hypoxia",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "data_supports_cell_intrinsic_cause"
                    ],
                    "canonical": "data_supports_cell_intrinsic_cause"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "HIF_hypoxia_program_specific"
                    ],
                    "canonical": "HIF_hypoxia_program_specific"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "robust_across_methods",
                      "robust_after_decontamination"
                    ],
                    "canonical": "robust_across_methods"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CXCL9_CXCL10": {
                "op": "in",
                "value": [
                  "immune_response_tumor_state"
                ]
              },
              "CCL5": {
                "op": "in",
                "value": [
                  "shared_across_tumor_states",
                  "partner_lymphocyte_derived",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "gate_conjugate_gate_confounder",
            "role": "gate",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "partner_lymphocyte_capture"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "antigen_presentation_decomposition",
            "kind": "dict",
            "expected": {
              "MHC_class_I": {
                "op": "in",
                "value": [
                  "co_regulated_with_dominant_program",
                  "tumor_intrinsic_enriched_independent"
                ]
              },
              "MHC_class_II": {
                "op": "in",
                "value": [
                  "partner_cell_attributable",
                  "flat_not_enriched",
                  "method_sensitive_no_robust_call",
                  "co_regulated_with_dominant_program"
                ]
              }
            }
          },
          {
            "name": "cf_non_robust_state_claimed_enriched",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "transitory_melanocytic",
              "intermediate_melanocytic",
              "melanocytic",
              "MITF_high",
              "neural_crest_like",
              "mitotic",
              "stress_p53",
              "undifferentiated",
              "dedifferentiated",
              "mesenchymal_tumor",
              "AXL_high_tumor",
              "invasive_tumor",
              "senescence_like",
              "EMT_transition",
              "cancer_stem_cell_like",
              "oxidative_stress_tumor"
            ]
          },
          {
            "name": "cf_framework_generic_or_offtissue_substitution",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "Tsoi_undifferentiated",
              "Tsoi_neural_crest_like",
              "Tsoi_transitory",
              "Tsoi_melanocytic",
              "Rambow_NCSC",
              "Wouters_mesenchymal",
              "Pozniak_TCF4_dependent",
              "tumor_cell",
              "melanoma_cell",
              "malignant_cell",
              "tumor_subpopulation",
              "keratinocyte",
              "hepatocyte",
              "lung_adenocarcinoma_cell",
              "colon_adenocarcinoma_cell",
              "Tirosh_cell_cycle_program",
              "Jerby_Arnon_program",
              "glioblastoma_cell",
              "breast_carcinoma_cell"
            ]
          },
          {
            "name": "cf_immune_response_direction_committed",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_immune_robustness_recall",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_ambiguous_hedge",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].causation_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_stress_generic_stress_response_evidence",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_conjugate_confounder_contamination_denied",
            "role": "hard_fail",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected"
            ]
          },
          {
            "name": "cf_ccl5_recall_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CCL5",
            "kind": "avoid",
            "expected": [
              "immune_response_tumor_state"
            ]
          },
          {
            "name": "cf_cxcl9_cxcl10_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CXCL9_CXCL10",
            "kind": "avoid",
            "expected": [
              "stress_hypoxia_tumor_state"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-sol-pi",
      "modelName": "gpt-5.6-sol",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_tumor_state_causation_v5",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 6302.112306,
      "cost": 4.423678749999999,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "enriched_tumor_states",
            "kind": "list",
            "matchKey": "tumor_state",
            "k": 3,
            "expected": [
              {
                "key": "immune_response",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "direction_unresolved_cross_sectional"
                    ],
                    "canonical": "direction_unresolved_cross_sectional"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "IFN_response_axis",
                      "broad_antigen_presentation_MHC"
                    ],
                    "canonical": "IFN_response_axis"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "enrichment_diminished_by_decontamination",
                      "threshold_sensitive"
                    ],
                    "canonical": "enrichment_diminished_by_decontamination"
                  }
                }
              },
              {
                "key": "stress_hypoxia",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "data_supports_cell_intrinsic_cause"
                    ],
                    "canonical": "data_supports_cell_intrinsic_cause"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "HIF_hypoxia_program_specific"
                    ],
                    "canonical": "HIF_hypoxia_program_specific"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "robust_across_methods",
                      "robust_after_decontamination"
                    ],
                    "canonical": "robust_across_methods"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CXCL9_CXCL10": {
                "op": "in",
                "value": [
                  "immune_response_tumor_state"
                ]
              },
              "CCL5": {
                "op": "in",
                "value": [
                  "shared_across_tumor_states",
                  "partner_lymphocyte_derived",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "gate_conjugate_gate_confounder",
            "role": "gate",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "partner_lymphocyte_capture"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "antigen_presentation_decomposition",
            "kind": "dict",
            "expected": {
              "MHC_class_I": {
                "op": "in",
                "value": [
                  "co_regulated_with_dominant_program",
                  "tumor_intrinsic_enriched_independent"
                ]
              },
              "MHC_class_II": {
                "op": "in",
                "value": [
                  "partner_cell_attributable",
                  "flat_not_enriched",
                  "method_sensitive_no_robust_call",
                  "co_regulated_with_dominant_program"
                ]
              }
            }
          },
          {
            "name": "cf_non_robust_state_claimed_enriched",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "transitory_melanocytic",
              "intermediate_melanocytic",
              "melanocytic",
              "MITF_high",
              "neural_crest_like",
              "mitotic",
              "stress_p53",
              "undifferentiated",
              "dedifferentiated",
              "mesenchymal_tumor",
              "AXL_high_tumor",
              "invasive_tumor",
              "senescence_like",
              "EMT_transition",
              "cancer_stem_cell_like",
              "oxidative_stress_tumor"
            ]
          },
          {
            "name": "cf_framework_generic_or_offtissue_substitution",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "Tsoi_undifferentiated",
              "Tsoi_neural_crest_like",
              "Tsoi_transitory",
              "Tsoi_melanocytic",
              "Rambow_NCSC",
              "Wouters_mesenchymal",
              "Pozniak_TCF4_dependent",
              "tumor_cell",
              "melanoma_cell",
              "malignant_cell",
              "tumor_subpopulation",
              "keratinocyte",
              "hepatocyte",
              "lung_adenocarcinoma_cell",
              "colon_adenocarcinoma_cell",
              "Tirosh_cell_cycle_program",
              "Jerby_Arnon_program",
              "glioblastoma_cell",
              "breast_carcinoma_cell"
            ]
          },
          {
            "name": "cf_immune_response_direction_committed",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_immune_robustness_recall",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_ambiguous_hedge",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].causation_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_stress_generic_stress_response_evidence",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_conjugate_confounder_contamination_denied",
            "role": "hard_fail",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected"
            ]
          },
          {
            "name": "cf_ccl5_recall_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CCL5",
            "kind": "avoid",
            "expected": [
              "immune_response_tumor_state"
            ]
          },
          {
            "name": "cf_cxcl9_cxcl10_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CXCL9_CXCL10",
            "kind": "avoid",
            "expected": [
              "stress_hypoxia_tumor_state"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-sol-pi",
      "modelName": "gpt-5.6-sol",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_vs_cd39_tumor_reactivity_dominance",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 860.114342,
      "cost": 1.9328950000000003,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "population_findings",
            "kind": "list",
            "matchKey": "capture_gate",
            "k": 4,
            "expected": [
              {
                "key": "tumor_cell_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "apc_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "combined_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "unselected_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "naive",
                      "central_memory",
                      "effector_memory"
                    ],
                    "canonical": "naive"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "low"
                    ],
                    "canonical": "low"
                  },
                  "purity_caveat": {
                    "op": "equals",
                    "values": [
                      "pure_T_cell"
                    ],
                    "canonical": "pure_T_cell"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "no_reactivity_evidence"
                    ],
                    "canonical": "no_reactivity_evidence"
                  }
                }
              },
              {
                "key": "marker_sorted_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "equals",
                    "values": [
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "terminal_exhausted_LAG3_high"
                  },
                  "clonal_expansion_level": {
                    "op": "in",
                    "values": [
                      "high",
                      "intermediate"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "marker_on_non_T_lineages",
                      "single_cell_gating_loses_target"
                    ],
                    "canonical": "marker_on_non_T_lineages"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "marker_or_exhaustion_defined_only"
                    ],
                    "canonical": "marker_or_exhaustion_defined_only"
                  }
                }
              }
            ]
          },
          {
            "name": "best_capture_approach_gate",
            "role": "gate",
            "field": "best_capture_approach",
            "kind": "value",
            "expected": "heterotypic_cell_conjugate"
          },
          {
            "name": "best_selection_marker_gate",
            "role": "gate",
            "field": "best_selection_marker",
            "kind": "value",
            "expected": "none"
          },
          {
            "name": "reactivity_criterion_gate",
            "role": "gate",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "clonal_expansion_and_physical_engagement"
          },
          {
            "name": "cross_patient_robustness_gate",
            "role": "gate",
            "field": "cross_patient_robustness",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_across_patients",
              "marginal_in_some_patients"
            ]
          },
          {
            "name": "marker_selection_consequence_gate",
            "role": "gate",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "discards_clonally_expanded_reactive_cells"
          },
          {
            "name": "superiority_claim_basis_gate",
            "role": "gate",
            "field": "superiority_claim_basis",
            "kind": "value",
            "expected": "requires_functional_reactivity_assay"
          },
          {
            "name": "conjugate_enriched_state_gate",
            "role": "gate",
            "field": "conjugate_enriched_state",
            "kind": "value",
            "expected": "progenitor_exhausted_TCF7_pos_stem_like"
          },
          {
            "name": "cf_circular_exhaustion_criterion",
            "role": "hard_fail",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "exhaustion_signature_intensity"
          },
          {
            "name": "cf_marker_sort_prior",
            "role": "hard_fail",
            "field": "best_selection_marker",
            "kind": "value",
            "op": "in",
            "expected": [
              "CD39",
              "PD1",
              "PD1_high",
              "TIM3",
              "PD1_CD39",
              "CD39_CD103"
            ]
          },
          {
            "name": "cf_single_cell_capture_prior",
            "role": "hard_fail",
            "field": "best_capture_approach",
            "kind": "value",
            "op": "in",
            "expected": [
              "dissociated_single_cell",
              "dissociated_bulk_population"
            ]
          },
          {
            "name": "cf_off_data_capture_gate",
            "role": "hard_fail",
            "field": "$.population_findings[*].capture_gate",
            "kind": "avoid",
            "expected": [
              "dissociated_bulk",
              "matched_peripheral_blood",
              "matched_lymph_node",
              "post_hoc_expanded_clonotype_group"
            ]
          },
          {
            "name": "cf_conjugate_mischaracterized",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_marker_singlet_called_engaged",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "contains",
            "matchKey": "capture_gate",
            "entity": "marker_sorted_singlet",
            "valueField": "reactivity_evidence",
            "expected": [
              "engagement_and_clonal_expansion"
            ]
          },
          {
            "name": "cf_no_engaged_reactive_population",
            "role": "hard_fail",
            "field": "$.population_findings[*].reactivity_evidence",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_marker_captures_all_reactive",
            "role": "hard_fail",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "captures_full_reactive_repertoire"
          },
          {
            "name": "cf_superiority_overclaimed_from_phenotype",
            "role": "hard_fail",
            "field": "superiority_claim_basis",
            "kind": "value",
            "op": "in",
            "expected": [
              "established_by_scrna_phenotype",
              "established_by_reactivity_proxy_scores"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-sol-pi",
      "modelName": "gpt-5.6-sol",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_vs_cd39_tumor_reactivity_dominance",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 830.206802,
      "cost": 1.6921625000000002,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "population_findings",
            "kind": "list",
            "matchKey": "capture_gate",
            "k": 4,
            "expected": [
              {
                "key": "tumor_cell_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "apc_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "combined_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "unselected_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "naive",
                      "central_memory",
                      "effector_memory"
                    ],
                    "canonical": "naive"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "low"
                    ],
                    "canonical": "low"
                  },
                  "purity_caveat": {
                    "op": "equals",
                    "values": [
                      "pure_T_cell"
                    ],
                    "canonical": "pure_T_cell"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "no_reactivity_evidence"
                    ],
                    "canonical": "no_reactivity_evidence"
                  }
                }
              },
              {
                "key": "marker_sorted_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "equals",
                    "values": [
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "terminal_exhausted_LAG3_high"
                  },
                  "clonal_expansion_level": {
                    "op": "in",
                    "values": [
                      "high",
                      "intermediate"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "marker_on_non_T_lineages",
                      "single_cell_gating_loses_target"
                    ],
                    "canonical": "marker_on_non_T_lineages"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "marker_or_exhaustion_defined_only"
                    ],
                    "canonical": "marker_or_exhaustion_defined_only"
                  }
                }
              }
            ]
          },
          {
            "name": "best_capture_approach_gate",
            "role": "gate",
            "field": "best_capture_approach",
            "kind": "value",
            "expected": "heterotypic_cell_conjugate"
          },
          {
            "name": "best_selection_marker_gate",
            "role": "gate",
            "field": "best_selection_marker",
            "kind": "value",
            "expected": "none"
          },
          {
            "name": "reactivity_criterion_gate",
            "role": "gate",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "clonal_expansion_and_physical_engagement"
          },
          {
            "name": "cross_patient_robustness_gate",
            "role": "gate",
            "field": "cross_patient_robustness",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_across_patients",
              "marginal_in_some_patients"
            ]
          },
          {
            "name": "marker_selection_consequence_gate",
            "role": "gate",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "discards_clonally_expanded_reactive_cells"
          },
          {
            "name": "superiority_claim_basis_gate",
            "role": "gate",
            "field": "superiority_claim_basis",
            "kind": "value",
            "expected": "requires_functional_reactivity_assay"
          },
          {
            "name": "conjugate_enriched_state_gate",
            "role": "gate",
            "field": "conjugate_enriched_state",
            "kind": "value",
            "expected": "progenitor_exhausted_TCF7_pos_stem_like"
          },
          {
            "name": "cf_circular_exhaustion_criterion",
            "role": "hard_fail",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "exhaustion_signature_intensity"
          },
          {
            "name": "cf_marker_sort_prior",
            "role": "hard_fail",
            "field": "best_selection_marker",
            "kind": "value",
            "op": "in",
            "expected": [
              "CD39",
              "PD1",
              "PD1_high",
              "TIM3",
              "PD1_CD39",
              "CD39_CD103"
            ]
          },
          {
            "name": "cf_single_cell_capture_prior",
            "role": "hard_fail",
            "field": "best_capture_approach",
            "kind": "value",
            "op": "in",
            "expected": [
              "dissociated_single_cell",
              "dissociated_bulk_population"
            ]
          },
          {
            "name": "cf_off_data_capture_gate",
            "role": "hard_fail",
            "field": "$.population_findings[*].capture_gate",
            "kind": "avoid",
            "expected": [
              "dissociated_bulk",
              "matched_peripheral_blood",
              "matched_lymph_node",
              "post_hoc_expanded_clonotype_group"
            ]
          },
          {
            "name": "cf_conjugate_mischaracterized",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_marker_singlet_called_engaged",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "contains",
            "matchKey": "capture_gate",
            "entity": "marker_sorted_singlet",
            "valueField": "reactivity_evidence",
            "expected": [
              "engagement_and_clonal_expansion"
            ]
          },
          {
            "name": "cf_no_engaged_reactive_population",
            "role": "hard_fail",
            "field": "$.population_findings[*].reactivity_evidence",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_marker_captures_all_reactive",
            "role": "hard_fail",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "captures_full_reactive_repertoire"
          },
          {
            "name": "cf_superiority_overclaimed_from_phenotype",
            "role": "hard_fail",
            "field": "superiority_claim_basis",
            "kind": "value",
            "op": "in",
            "expected": [
              "established_by_scrna_phenotype",
              "established_by_reactivity_proxy_scores"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-sol-pi",
      "modelName": "gpt-5.6-sol",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_vs_cd39_tumor_reactivity_dominance",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1556.748243,
      "cost": 1.5398317499999996,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "population_findings",
            "kind": "list",
            "matchKey": "capture_gate",
            "k": 4,
            "expected": [
              {
                "key": "tumor_cell_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "apc_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "combined_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "unselected_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "naive",
                      "central_memory",
                      "effector_memory"
                    ],
                    "canonical": "naive"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "low"
                    ],
                    "canonical": "low"
                  },
                  "purity_caveat": {
                    "op": "equals",
                    "values": [
                      "pure_T_cell"
                    ],
                    "canonical": "pure_T_cell"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "no_reactivity_evidence"
                    ],
                    "canonical": "no_reactivity_evidence"
                  }
                }
              },
              {
                "key": "marker_sorted_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "equals",
                    "values": [
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "terminal_exhausted_LAG3_high"
                  },
                  "clonal_expansion_level": {
                    "op": "in",
                    "values": [
                      "high",
                      "intermediate"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "marker_on_non_T_lineages",
                      "single_cell_gating_loses_target"
                    ],
                    "canonical": "marker_on_non_T_lineages"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "marker_or_exhaustion_defined_only"
                    ],
                    "canonical": "marker_or_exhaustion_defined_only"
                  }
                }
              }
            ]
          },
          {
            "name": "best_capture_approach_gate",
            "role": "gate",
            "field": "best_capture_approach",
            "kind": "value",
            "expected": "heterotypic_cell_conjugate"
          },
          {
            "name": "best_selection_marker_gate",
            "role": "gate",
            "field": "best_selection_marker",
            "kind": "value",
            "expected": "none"
          },
          {
            "name": "reactivity_criterion_gate",
            "role": "gate",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "clonal_expansion_and_physical_engagement"
          },
          {
            "name": "cross_patient_robustness_gate",
            "role": "gate",
            "field": "cross_patient_robustness",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_across_patients",
              "marginal_in_some_patients"
            ]
          },
          {
            "name": "marker_selection_consequence_gate",
            "role": "gate",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "discards_clonally_expanded_reactive_cells"
          },
          {
            "name": "superiority_claim_basis_gate",
            "role": "gate",
            "field": "superiority_claim_basis",
            "kind": "value",
            "expected": "requires_functional_reactivity_assay"
          },
          {
            "name": "conjugate_enriched_state_gate",
            "role": "gate",
            "field": "conjugate_enriched_state",
            "kind": "value",
            "expected": "progenitor_exhausted_TCF7_pos_stem_like"
          },
          {
            "name": "cf_circular_exhaustion_criterion",
            "role": "hard_fail",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "exhaustion_signature_intensity"
          },
          {
            "name": "cf_marker_sort_prior",
            "role": "hard_fail",
            "field": "best_selection_marker",
            "kind": "value",
            "op": "in",
            "expected": [
              "CD39",
              "PD1",
              "PD1_high",
              "TIM3",
              "PD1_CD39",
              "CD39_CD103"
            ]
          },
          {
            "name": "cf_single_cell_capture_prior",
            "role": "hard_fail",
            "field": "best_capture_approach",
            "kind": "value",
            "op": "in",
            "expected": [
              "dissociated_single_cell",
              "dissociated_bulk_population"
            ]
          },
          {
            "name": "cf_off_data_capture_gate",
            "role": "hard_fail",
            "field": "$.population_findings[*].capture_gate",
            "kind": "avoid",
            "expected": [
              "dissociated_bulk",
              "matched_peripheral_blood",
              "matched_lymph_node",
              "post_hoc_expanded_clonotype_group"
            ]
          },
          {
            "name": "cf_conjugate_mischaracterized",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_marker_singlet_called_engaged",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "contains",
            "matchKey": "capture_gate",
            "entity": "marker_sorted_singlet",
            "valueField": "reactivity_evidence",
            "expected": [
              "engagement_and_clonal_expansion"
            ]
          },
          {
            "name": "cf_no_engaged_reactive_population",
            "role": "hard_fail",
            "field": "$.population_findings[*].reactivity_evidence",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_marker_captures_all_reactive",
            "role": "hard_fail",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "captures_full_reactive_repertoire"
          },
          {
            "name": "cf_superiority_overclaimed_from_phenotype",
            "role": "hard_fail",
            "field": "superiority_claim_basis",
            "kind": "value",
            "op": "in",
            "expected": [
              "established_by_scrna_phenotype",
              "established_by_reactivity_proxy_scores"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-sol-pi",
      "modelName": "gpt-5.6-sol",
      "harness": "pi",
      "task": "10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 1,
      "passed": true,
      "score": null,
      "durationS": 2243.652772,
      "cost": 3.35292575,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_singlet_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "TOXhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "LAG3high_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "GZMKhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "intermediate_exhausted_gzmk"
                    ],
                    "canonical": "intermediate_exhausted_gzmk"
                  }
                }
              },
              {
                "key": "TCF7_stem_like_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "stem_like_exhausted"
                    ],
                    "canonical": "stem_like_exhausted"
                  }
                }
              },
              {
                "key": "MKI67_pos_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              },
              {
                "key": "MKI67high_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_tumor_cluster_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "Tn",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "Tn_Tmem",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "early_Tem_GZMK",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              },
              {
                "key": "Tem_GZMH",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              }
            ]
          },
          {
            "name": "clonotype_expansion_direction_data_true",
            "role": "gate",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_substantially_more_expanded_than_cluster",
              "singlet_modestly_more_expanded_than_cluster"
            ]
          },
          {
            "name": "biological_inference_data_true",
            "role": "gate",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_antigen_experienced_cluster_bystander"
            ]
          },
          {
            "name": "singlet_top3_contains_naive_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Tn",
              "Tn_Tmem",
              "early_Tem_GZMK",
              "Tem_GZMH"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "TOXhigh_Tex",
              "LAG3high_Tex",
              "GZMKhigh_Tex",
              "TCF7_stem_like_Tex",
              "MKI67_pos_Tex_Tprol",
              "MKI67high_Tex_Tprol"
            ]
          },
          {
            "name": "singlet_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "TOXhigh_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "TCF7_stem_like_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "MKI67_state_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "Tn_or_Tn_Tmem_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_stemlike_precursor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_reported_canonical_expansion_direction",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_substantially_more_expanded_than_singlet",
              "cluster_modestly_more_expanded_than_singlet"
            ]
          },
          {
            "name": "cf_expansion_lazy_defer",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "expansion_pattern_not_detectable"
            ]
          },
          {
            "name": "cf_reported_canonical_expected_biology",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_antigen_experienced_singlet_bystander"
            ]
          },
          {
            "name": "cf_biological_inference_null_artifact_or_lineage",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "differences_reflect_dissociation_or_sorting_artifact",
              "cluster_and_singlet_capture_same_biology",
              "cluster_and_singlet_capture_distinct_cd8_lineages"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "biological_relations",
            "kind": "dict",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_greater_than",
                  "greater_than"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "less_than",
                  "much_less_than"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than",
                  "approximately_equal"
                ]
              },
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal"
                ]
              },
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal",
                  "greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_exhaustion_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_naive_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_proliferation_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_expansion_richness_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_multichain_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_intermediate_gzmk_overgeneralized",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_melanocytic_conjugate_overcall",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "values": [
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_relations_mass_hedge",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-sol-pi",
      "modelName": "gpt-5.6-sol",
      "harness": "pi",
      "task": "10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 2,
      "passed": true,
      "score": null,
      "durationS": 1470.892179,
      "cost": 2.51703575,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_singlet_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "TOXhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "LAG3high_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "GZMKhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "intermediate_exhausted_gzmk"
                    ],
                    "canonical": "intermediate_exhausted_gzmk"
                  }
                }
              },
              {
                "key": "TCF7_stem_like_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "stem_like_exhausted"
                    ],
                    "canonical": "stem_like_exhausted"
                  }
                }
              },
              {
                "key": "MKI67_pos_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              },
              {
                "key": "MKI67high_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_tumor_cluster_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "Tn",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "Tn_Tmem",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "early_Tem_GZMK",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              },
              {
                "key": "Tem_GZMH",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              }
            ]
          },
          {
            "name": "clonotype_expansion_direction_data_true",
            "role": "gate",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_substantially_more_expanded_than_cluster",
              "singlet_modestly_more_expanded_than_cluster"
            ]
          },
          {
            "name": "biological_inference_data_true",
            "role": "gate",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_antigen_experienced_cluster_bystander"
            ]
          },
          {
            "name": "singlet_top3_contains_naive_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Tn",
              "Tn_Tmem",
              "early_Tem_GZMK",
              "Tem_GZMH"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "TOXhigh_Tex",
              "LAG3high_Tex",
              "GZMKhigh_Tex",
              "TCF7_stem_like_Tex",
              "MKI67_pos_Tex_Tprol",
              "MKI67high_Tex_Tprol"
            ]
          },
          {
            "name": "singlet_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "TOXhigh_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "TCF7_stem_like_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "MKI67_state_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "Tn_or_Tn_Tmem_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_stemlike_precursor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_reported_canonical_expansion_direction",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_substantially_more_expanded_than_singlet",
              "cluster_modestly_more_expanded_than_singlet"
            ]
          },
          {
            "name": "cf_expansion_lazy_defer",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "expansion_pattern_not_detectable"
            ]
          },
          {
            "name": "cf_reported_canonical_expected_biology",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_antigen_experienced_singlet_bystander"
            ]
          },
          {
            "name": "cf_biological_inference_null_artifact_or_lineage",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "differences_reflect_dissociation_or_sorting_artifact",
              "cluster_and_singlet_capture_same_biology",
              "cluster_and_singlet_capture_distinct_cd8_lineages"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "biological_relations",
            "kind": "dict",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_greater_than",
                  "greater_than"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "less_than",
                  "much_less_than"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than",
                  "approximately_equal"
                ]
              },
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal"
                ]
              },
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal",
                  "greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_exhaustion_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_naive_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_proliferation_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_expansion_richness_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_multichain_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_intermediate_gzmk_overgeneralized",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_melanocytic_conjugate_overcall",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "values": [
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_relations_mass_hedge",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-sol-pi",
      "modelName": "gpt-5.6-sol",
      "harness": "pi",
      "task": "10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 2026.893155,
      "cost": 2.01928975,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_singlet_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "TOXhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "LAG3high_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "GZMKhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "intermediate_exhausted_gzmk"
                    ],
                    "canonical": "intermediate_exhausted_gzmk"
                  }
                }
              },
              {
                "key": "TCF7_stem_like_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "stem_like_exhausted"
                    ],
                    "canonical": "stem_like_exhausted"
                  }
                }
              },
              {
                "key": "MKI67_pos_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              },
              {
                "key": "MKI67high_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_tumor_cluster_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "Tn",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "Tn_Tmem",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "early_Tem_GZMK",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              },
              {
                "key": "Tem_GZMH",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              }
            ]
          },
          {
            "name": "clonotype_expansion_direction_data_true",
            "role": "gate",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_substantially_more_expanded_than_cluster",
              "singlet_modestly_more_expanded_than_cluster"
            ]
          },
          {
            "name": "biological_inference_data_true",
            "role": "gate",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_antigen_experienced_cluster_bystander"
            ]
          },
          {
            "name": "singlet_top3_contains_naive_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Tn",
              "Tn_Tmem",
              "early_Tem_GZMK",
              "Tem_GZMH"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "TOXhigh_Tex",
              "LAG3high_Tex",
              "GZMKhigh_Tex",
              "TCF7_stem_like_Tex",
              "MKI67_pos_Tex_Tprol",
              "MKI67high_Tex_Tprol"
            ]
          },
          {
            "name": "singlet_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "TOXhigh_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "TCF7_stem_like_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "MKI67_state_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "Tn_or_Tn_Tmem_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_stemlike_precursor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_reported_canonical_expansion_direction",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_substantially_more_expanded_than_singlet",
              "cluster_modestly_more_expanded_than_singlet"
            ]
          },
          {
            "name": "cf_expansion_lazy_defer",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "expansion_pattern_not_detectable"
            ]
          },
          {
            "name": "cf_reported_canonical_expected_biology",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_antigen_experienced_singlet_bystander"
            ]
          },
          {
            "name": "cf_biological_inference_null_artifact_or_lineage",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "differences_reflect_dissociation_or_sorting_artifact",
              "cluster_and_singlet_capture_same_biology",
              "cluster_and_singlet_capture_distinct_cd8_lineages"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "biological_relations",
            "kind": "dict",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_greater_than",
                  "greater_than"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "less_than",
                  "much_less_than"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than",
                  "approximately_equal"
                ]
              },
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal"
                ]
              },
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal",
                  "greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_exhaustion_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_naive_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_proliferation_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_expansion_richness_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_multichain_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_intermediate_gzmk_overgeneralized",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_melanocytic_conjugate_overcall",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "values": [
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_relations_mass_hedge",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-sol-pi",
      "modelName": "gpt-5.6-sol",
      "harness": "pi",
      "task": "10x_5p_melanoma_tcr_matched_apc_vs_tumor_exhaustion",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 1,
      "passed": true,
      "score": null,
      "durationS": 525.270879,
      "cost": 2.1673857499999998,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "state_discriminating_genes",
            "kind": "list",
            "matchKey": "gene",
            "k": 3,
            "expected": [
              {
                "key": "CTLA4",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TOX",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "ENTPD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "HAVCR2",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TIGIT",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "PDCD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "difference_attributable_to",
            "role": "gate",
            "field": "difference_attributable_to",
            "kind": "value",
            "expected": "covaries_with_apc_conjugate_state"
          },
          {
            "name": "causal_status",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "direction_unresolved_selection_confound"
          },
          {
            "name": "functional_interpretation",
            "role": "gate",
            "field": "functional_interpretation",
            "kind": "value",
            "expected": "costimulation_signature_function_unresolved"
          },
          {
            "name": "cf_contamination_gene_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "MLANA",
              "PMEL",
              "DCT",
              "TYR",
              "TYRP1",
              "SOX10",
              "MITF",
              "MAGEA4"
            ]
          },
          {
            "name": "cf_apc_partner_contamination_gene",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "HLA-DRA",
              "CD68",
              "LYZ",
              "CD14",
              "C1QA",
              "ITGAX",
              "CD1C"
            ]
          },
          {
            "name": "cf_ligand_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "CD274",
              "PDCD1LG2",
              "CD80",
              "CD86"
            ]
          },
          {
            "name": "cf_attribution_causal_or_baseline",
            "role": "hard_fail",
            "field": "$.difference_attributable_to",
            "kind": "avoid",
            "expected": [
              "apc_engagement",
              "tumor_cell_engagement",
              "clonotype_intrinsic_baseline"
            ]
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "engagement_induces_exhausted_state",
              "pre_exhausted_cells_select_into_apc_conjugates"
            ]
          },
          {
            "name": "cf_causal_fabrication",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "resolved_by_pseudotemporal_ordering",
              "resolved_by_perturbation_experiment"
            ]
          },
          {
            "name": "cf_functional_overclaim",
            "role": "hard_fail",
            "field": "$.functional_interpretation",
            "kind": "avoid",
            "expected": [
              "costimulation_licensed_reinvigoration_competent",
              "costimulation_licensed_checkpoint_responsive",
              "tumor_driven_terminal_exhaustion"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-sol-pi",
      "modelName": "gpt-5.6-sol",
      "harness": "pi",
      "task": "10x_5p_melanoma_tcr_matched_apc_vs_tumor_exhaustion",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 2,
      "passed": true,
      "score": null,
      "durationS": 933.855601,
      "cost": 2.8951082499999994,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "state_discriminating_genes",
            "kind": "list",
            "matchKey": "gene",
            "k": 3,
            "expected": [
              {
                "key": "CTLA4",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TOX",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "ENTPD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "HAVCR2",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TIGIT",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "PDCD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "difference_attributable_to",
            "role": "gate",
            "field": "difference_attributable_to",
            "kind": "value",
            "expected": "covaries_with_apc_conjugate_state"
          },
          {
            "name": "causal_status",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "direction_unresolved_selection_confound"
          },
          {
            "name": "functional_interpretation",
            "role": "gate",
            "field": "functional_interpretation",
            "kind": "value",
            "expected": "costimulation_signature_function_unresolved"
          },
          {
            "name": "cf_contamination_gene_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "MLANA",
              "PMEL",
              "DCT",
              "TYR",
              "TYRP1",
              "SOX10",
              "MITF",
              "MAGEA4"
            ]
          },
          {
            "name": "cf_apc_partner_contamination_gene",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "HLA-DRA",
              "CD68",
              "LYZ",
              "CD14",
              "C1QA",
              "ITGAX",
              "CD1C"
            ]
          },
          {
            "name": "cf_ligand_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "CD274",
              "PDCD1LG2",
              "CD80",
              "CD86"
            ]
          },
          {
            "name": "cf_attribution_causal_or_baseline",
            "role": "hard_fail",
            "field": "$.difference_attributable_to",
            "kind": "avoid",
            "expected": [
              "apc_engagement",
              "tumor_cell_engagement",
              "clonotype_intrinsic_baseline"
            ]
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "engagement_induces_exhausted_state",
              "pre_exhausted_cells_select_into_apc_conjugates"
            ]
          },
          {
            "name": "cf_causal_fabrication",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "resolved_by_pseudotemporal_ordering",
              "resolved_by_perturbation_experiment"
            ]
          },
          {
            "name": "cf_functional_overclaim",
            "role": "hard_fail",
            "field": "$.functional_interpretation",
            "kind": "avoid",
            "expected": [
              "costimulation_licensed_reinvigoration_competent",
              "costimulation_licensed_checkpoint_responsive",
              "tumor_driven_terminal_exhaustion"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-sol-pi",
      "modelName": "gpt-5.6-sol",
      "harness": "pi",
      "task": "10x_5p_melanoma_tcr_matched_apc_vs_tumor_exhaustion",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 591.280571,
      "cost": 1.2652165000000002,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "state_discriminating_genes",
            "kind": "list",
            "matchKey": "gene",
            "k": 3,
            "expected": [
              {
                "key": "CTLA4",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TOX",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "ENTPD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "HAVCR2",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TIGIT",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "PDCD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "difference_attributable_to",
            "role": "gate",
            "field": "difference_attributable_to",
            "kind": "value",
            "expected": "covaries_with_apc_conjugate_state"
          },
          {
            "name": "causal_status",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "direction_unresolved_selection_confound"
          },
          {
            "name": "functional_interpretation",
            "role": "gate",
            "field": "functional_interpretation",
            "kind": "value",
            "expected": "costimulation_signature_function_unresolved"
          },
          {
            "name": "cf_contamination_gene_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "MLANA",
              "PMEL",
              "DCT",
              "TYR",
              "TYRP1",
              "SOX10",
              "MITF",
              "MAGEA4"
            ]
          },
          {
            "name": "cf_apc_partner_contamination_gene",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "HLA-DRA",
              "CD68",
              "LYZ",
              "CD14",
              "C1QA",
              "ITGAX",
              "CD1C"
            ]
          },
          {
            "name": "cf_ligand_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "CD274",
              "PDCD1LG2",
              "CD80",
              "CD86"
            ]
          },
          {
            "name": "cf_attribution_causal_or_baseline",
            "role": "hard_fail",
            "field": "$.difference_attributable_to",
            "kind": "avoid",
            "expected": [
              "apc_engagement",
              "tumor_cell_engagement",
              "clonotype_intrinsic_baseline"
            ]
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "engagement_induces_exhausted_state",
              "pre_exhausted_cells_select_into_apc_conjugates"
            ]
          },
          {
            "name": "cf_causal_fabrication",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "resolved_by_pseudotemporal_ordering",
              "resolved_by_perturbation_experiment"
            ]
          },
          {
            "name": "cf_functional_overclaim",
            "role": "hard_fail",
            "field": "$.functional_interpretation",
            "kind": "avoid",
            "expected": [
              "costimulation_licensed_reinvigoration_competent",
              "costimulation_licensed_checkpoint_responsive",
              "tumor_driven_terminal_exhaustion"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-sol-pi",
      "modelName": "gpt-5.6-sol",
      "harness": "pi",
      "task": "green_2_tme_context_dependence",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 1871.463884,
      "cost": 5.4726047499999995,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "program_map",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "sig_includes_itgae",
            "role": "gate",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Itgae"
            ]
          },
          {
            "name": "declared_a_known_confounder",
            "role": "gate",
            "field": "$.excluded_confounders[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "cf_signature_gut_contaminant",
            "role": "hard_fail",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "generic_signature_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd103_program_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "<"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "per_model_regulatory_state",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "poised_unfired"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "late_activation"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "closing_committed"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_tuples",
            "kind": "list",
            "matchKey": "regulator",
            "k": 1,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "role": {
                    "op": "equals",
                    "values": [
                      "repressor"
                    ]
                  },
                  "gating_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  },
                  "evidence_basis": {
                    "op": "equals",
                    "values": [
                      "target_program_behavior"
                    ]
                  },
                  "expr_vs_activity": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "locus_differential_gate",
            "role": "gate",
            "field": "locus_differential",
            "kind": "value",
            "expected": "uniform_across_models"
          },
          {
            "name": "formation_limited_by_gate",
            "role": "gate",
            "field": "formation_limited_by",
            "kind": "value",
            "expected": "absent_induction"
          },
          {
            "name": "locus_competent_in_naive_gate",
            "role": "gate",
            "field": "locus_competent_in_naive",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "ready_silent_fraction_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 3,
            "expected": [
              {
                "key": "B16_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_KPC_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "activator_chromatin_footprint_gate",
            "role": "gate",
            "field": "activator_chromatin_footprint",
            "kind": "value",
            "expected": "inconsistent_across_formers"
          },
          {
            "name": "exhaustion_comparable_gate",
            "role": "gate",
            "field": "exhaustion_comparable_across_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_nondeterminant_gate",
            "role": "gate",
            "field": "exhaustion_explains_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_separable_program_gate",
            "role": "gate",
            "field": "exhaustion_separable_program",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "exhaustion_counterexample_gate",
            "role": "gate",
            "field": "exhaustion_counterexample_model",
            "kind": "value",
            "expected": "KPC_GP"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "exhaustion_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "cf_inconsistent_present_without_activator",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "program",
            "entity": "present",
            "valueField": "activator_state",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_no_block_but_absent",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "none",
            "valueField": "program",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_late_block_without_poised_locus",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "late_activation",
            "valueField": "locus_state",
            "expected": [
              "closing_committed",
              "active"
            ]
          },
          {
            "name": "cf_inconsistent_early_block_with_brake_released",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "early_release",
            "valueField": "brake_state",
            "expected": [
              "released"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-sol-pi",
      "modelName": "gpt-5.6-sol",
      "harness": "pi",
      "task": "green_2_tme_context_dependence",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 1527.886043,
      "cost": 8.20146875,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "program_map",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "sig_includes_itgae",
            "role": "gate",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Itgae"
            ]
          },
          {
            "name": "declared_a_known_confounder",
            "role": "gate",
            "field": "$.excluded_confounders[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "cf_signature_gut_contaminant",
            "role": "hard_fail",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "generic_signature_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd103_program_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "<"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "per_model_regulatory_state",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "poised_unfired"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "late_activation"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "closing_committed"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_tuples",
            "kind": "list",
            "matchKey": "regulator",
            "k": 1,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "role": {
                    "op": "equals",
                    "values": [
                      "repressor"
                    ]
                  },
                  "gating_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  },
                  "evidence_basis": {
                    "op": "equals",
                    "values": [
                      "target_program_behavior"
                    ]
                  },
                  "expr_vs_activity": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "locus_differential_gate",
            "role": "gate",
            "field": "locus_differential",
            "kind": "value",
            "expected": "uniform_across_models"
          },
          {
            "name": "formation_limited_by_gate",
            "role": "gate",
            "field": "formation_limited_by",
            "kind": "value",
            "expected": "absent_induction"
          },
          {
            "name": "locus_competent_in_naive_gate",
            "role": "gate",
            "field": "locus_competent_in_naive",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "ready_silent_fraction_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 3,
            "expected": [
              {
                "key": "B16_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_KPC_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "activator_chromatin_footprint_gate",
            "role": "gate",
            "field": "activator_chromatin_footprint",
            "kind": "value",
            "expected": "inconsistent_across_formers"
          },
          {
            "name": "exhaustion_comparable_gate",
            "role": "gate",
            "field": "exhaustion_comparable_across_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_nondeterminant_gate",
            "role": "gate",
            "field": "exhaustion_explains_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_separable_program_gate",
            "role": "gate",
            "field": "exhaustion_separable_program",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "exhaustion_counterexample_gate",
            "role": "gate",
            "field": "exhaustion_counterexample_model",
            "kind": "value",
            "expected": "KPC_GP"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "exhaustion_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "cf_inconsistent_present_without_activator",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "program",
            "entity": "present",
            "valueField": "activator_state",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_no_block_but_absent",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "none",
            "valueField": "program",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_late_block_without_poised_locus",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "late_activation",
            "valueField": "locus_state",
            "expected": [
              "closing_committed",
              "active"
            ]
          },
          {
            "name": "cf_inconsistent_early_block_with_brake_released",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "early_release",
            "valueField": "brake_state",
            "expected": [
              "released"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-sol-pi",
      "modelName": "gpt-5.6-sol",
      "harness": "pi",
      "task": "green_2_tme_context_dependence",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 2263.543708,
      "cost": 7.645146499999998,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "program_map",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "sig_includes_itgae",
            "role": "gate",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Itgae"
            ]
          },
          {
            "name": "declared_a_known_confounder",
            "role": "gate",
            "field": "$.excluded_confounders[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "cf_signature_gut_contaminant",
            "role": "hard_fail",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "generic_signature_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd103_program_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "<"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "per_model_regulatory_state",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "poised_unfired"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "late_activation"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "closing_committed"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_tuples",
            "kind": "list",
            "matchKey": "regulator",
            "k": 1,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "role": {
                    "op": "equals",
                    "values": [
                      "repressor"
                    ]
                  },
                  "gating_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  },
                  "evidence_basis": {
                    "op": "equals",
                    "values": [
                      "target_program_behavior"
                    ]
                  },
                  "expr_vs_activity": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "locus_differential_gate",
            "role": "gate",
            "field": "locus_differential",
            "kind": "value",
            "expected": "uniform_across_models"
          },
          {
            "name": "formation_limited_by_gate",
            "role": "gate",
            "field": "formation_limited_by",
            "kind": "value",
            "expected": "absent_induction"
          },
          {
            "name": "locus_competent_in_naive_gate",
            "role": "gate",
            "field": "locus_competent_in_naive",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "ready_silent_fraction_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 3,
            "expected": [
              {
                "key": "B16_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_KPC_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "activator_chromatin_footprint_gate",
            "role": "gate",
            "field": "activator_chromatin_footprint",
            "kind": "value",
            "expected": "inconsistent_across_formers"
          },
          {
            "name": "exhaustion_comparable_gate",
            "role": "gate",
            "field": "exhaustion_comparable_across_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_nondeterminant_gate",
            "role": "gate",
            "field": "exhaustion_explains_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_separable_program_gate",
            "role": "gate",
            "field": "exhaustion_separable_program",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "exhaustion_counterexample_gate",
            "role": "gate",
            "field": "exhaustion_counterexample_model",
            "kind": "value",
            "expected": "KPC_GP"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "exhaustion_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "cf_inconsistent_present_without_activator",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "program",
            "entity": "present",
            "valueField": "activator_state",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_no_block_but_absent",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "none",
            "valueField": "program",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_late_block_without_poised_locus",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "late_activation",
            "valueField": "locus_state",
            "expected": [
              "closing_committed",
              "active"
            ]
          },
          {
            "name": "cf_inconsistent_early_block_with_brake_released",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "early_release",
            "valueField": "brake_state",
            "expected": [
              "released"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-sol-pi",
      "modelName": "gpt-5.6-sol",
      "harness": "pi",
      "task": "green_klf2_hidden_repressor_of_trm_like_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 2061.02935,
      "cost": 3.97487075,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "regulator_gate",
            "role": "gate",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "KLF2",
              "Klf2"
            ]
          },
          {
            "name": "controlling_node_character_gate",
            "role": "gate",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "high_differential_expression"
          },
          {
            "name": "expression_vs_activity_gate",
            "role": "gate",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "expression_exceeds_activity"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "target_program_partition",
            "kind": "list",
            "matchKey": "gene",
            "k": 7,
            "expected": [
              {
                "key": "S1pr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "S1pr4",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Klrg1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Cx3cr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Itgae",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Hic1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "P2rx7",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              }
            ]
          },
          {
            "name": "causal_status_gate",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "association_only_sufficiency_requires_perturbation"
          },
          {
            "name": "residency_control_logic_gate",
            "role": "gate",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_release_necessary_not_sufficient_multi_regulon"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "trm_like_program_by_model",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_absent"
                    ],
                    "canonical": "niche_absent"
                  }
                }
              }
            ]
          },
          {
            "name": "cf_regulator_activator_or_wrong_node",
            "role": "hard_fail",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "RUNX3",
              "Runx3",
              "PRDM1",
              "Prdm1",
              "BLIMP1",
              "Blimp1",
              "BHLHE40",
              "Bhlhe40",
              "ZNF683",
              "Zfp683",
              "HOBIT",
              "Hobit",
              "AHR",
              "Ahr",
              "BATF",
              "Batf",
              "TOX",
              "Tox",
              "NR4A1",
              "Nr4a1",
              "NR4A2",
              "Nr4a2",
              "NR4A3",
              "Nr4a3",
              "TCF7",
              "Tcf7",
              "TCF1",
              "Tcf1",
              "BACH2",
              "Bach2",
              "TBX21",
              "Tbx21",
              "TBET",
              "Tbet",
              "FLI1",
              "Fli1",
              "NR3C1",
              "Nr3c1",
              "EOMES",
              "Eomes",
              "ZEB2",
              "Zeb2",
              "ID2",
              "Id2",
              "ID3",
              "Id3",
              "IRF4",
              "Irf4"
            ]
          },
          {
            "name": "cf_node_character_hidden_driver_recall",
            "role": "hard_fail",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "hidden_driver_activity_not_expression"
          },
          {
            "name": "cf_effect_relation_hidden_pattern",
            "role": "hard_fail",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "activity_exceeds_expression"
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "causal_status",
            "kind": "value",
            "op": "in",
            "expected": [
              "sufficiency_established",
              "necessity_established",
              "causal_established_in_dataset",
              "sufficiency_via_perturbation_in_dataset"
            ]
          },
          {
            "name": "cf_residency_logic_activator_prior",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "residency_activator_induction_primary"
          },
          {
            "name": "cf_residency_logic_single_regulon",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_controls_entire_residency_program"
          },
          {
            "name": "cf_decoy_called_target",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_distractor_model_reported",
            "role": "hard_fail",
            "field": "$.trm_like_program_by_model[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_chromatin_only_decoy",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-sol-pi",
      "modelName": "gpt-5.6-sol",
      "harness": "pi",
      "task": "green_klf2_hidden_repressor_of_trm_like_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 757.030619,
      "cost": 2.1598742500000006,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "regulator_gate",
            "role": "gate",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "KLF2",
              "Klf2"
            ]
          },
          {
            "name": "controlling_node_character_gate",
            "role": "gate",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "high_differential_expression"
          },
          {
            "name": "expression_vs_activity_gate",
            "role": "gate",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "expression_exceeds_activity"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "target_program_partition",
            "kind": "list",
            "matchKey": "gene",
            "k": 7,
            "expected": [
              {
                "key": "S1pr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "S1pr4",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Klrg1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Cx3cr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Itgae",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Hic1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "P2rx7",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              }
            ]
          },
          {
            "name": "causal_status_gate",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "association_only_sufficiency_requires_perturbation"
          },
          {
            "name": "residency_control_logic_gate",
            "role": "gate",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_release_necessary_not_sufficient_multi_regulon"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "trm_like_program_by_model",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_absent"
                    ],
                    "canonical": "niche_absent"
                  }
                }
              }
            ]
          },
          {
            "name": "cf_regulator_activator_or_wrong_node",
            "role": "hard_fail",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "RUNX3",
              "Runx3",
              "PRDM1",
              "Prdm1",
              "BLIMP1",
              "Blimp1",
              "BHLHE40",
              "Bhlhe40",
              "ZNF683",
              "Zfp683",
              "HOBIT",
              "Hobit",
              "AHR",
              "Ahr",
              "BATF",
              "Batf",
              "TOX",
              "Tox",
              "NR4A1",
              "Nr4a1",
              "NR4A2",
              "Nr4a2",
              "NR4A3",
              "Nr4a3",
              "TCF7",
              "Tcf7",
              "TCF1",
              "Tcf1",
              "BACH2",
              "Bach2",
              "TBX21",
              "Tbx21",
              "TBET",
              "Tbet",
              "FLI1",
              "Fli1",
              "NR3C1",
              "Nr3c1",
              "EOMES",
              "Eomes",
              "ZEB2",
              "Zeb2",
              "ID2",
              "Id2",
              "ID3",
              "Id3",
              "IRF4",
              "Irf4"
            ]
          },
          {
            "name": "cf_node_character_hidden_driver_recall",
            "role": "hard_fail",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "hidden_driver_activity_not_expression"
          },
          {
            "name": "cf_effect_relation_hidden_pattern",
            "role": "hard_fail",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "activity_exceeds_expression"
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "causal_status",
            "kind": "value",
            "op": "in",
            "expected": [
              "sufficiency_established",
              "necessity_established",
              "causal_established_in_dataset",
              "sufficiency_via_perturbation_in_dataset"
            ]
          },
          {
            "name": "cf_residency_logic_activator_prior",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "residency_activator_induction_primary"
          },
          {
            "name": "cf_residency_logic_single_regulon",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_controls_entire_residency_program"
          },
          {
            "name": "cf_decoy_called_target",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_distractor_model_reported",
            "role": "hard_fail",
            "field": "$.trm_like_program_by_model[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_chromatin_only_decoy",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-sol-pi",
      "modelName": "gpt-5.6-sol",
      "harness": "pi",
      "task": "green_klf2_hidden_repressor_of_trm_like_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1381.305385,
      "cost": 1.8189274999999998,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "regulator_gate",
            "role": "gate",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "KLF2",
              "Klf2"
            ]
          },
          {
            "name": "controlling_node_character_gate",
            "role": "gate",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "high_differential_expression"
          },
          {
            "name": "expression_vs_activity_gate",
            "role": "gate",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "expression_exceeds_activity"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "target_program_partition",
            "kind": "list",
            "matchKey": "gene",
            "k": 7,
            "expected": [
              {
                "key": "S1pr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "S1pr4",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Klrg1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Cx3cr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Itgae",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Hic1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "P2rx7",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              }
            ]
          },
          {
            "name": "causal_status_gate",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "association_only_sufficiency_requires_perturbation"
          },
          {
            "name": "residency_control_logic_gate",
            "role": "gate",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_release_necessary_not_sufficient_multi_regulon"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "trm_like_program_by_model",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_absent"
                    ],
                    "canonical": "niche_absent"
                  }
                }
              }
            ]
          },
          {
            "name": "cf_regulator_activator_or_wrong_node",
            "role": "hard_fail",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "RUNX3",
              "Runx3",
              "PRDM1",
              "Prdm1",
              "BLIMP1",
              "Blimp1",
              "BHLHE40",
              "Bhlhe40",
              "ZNF683",
              "Zfp683",
              "HOBIT",
              "Hobit",
              "AHR",
              "Ahr",
              "BATF",
              "Batf",
              "TOX",
              "Tox",
              "NR4A1",
              "Nr4a1",
              "NR4A2",
              "Nr4a2",
              "NR4A3",
              "Nr4a3",
              "TCF7",
              "Tcf7",
              "TCF1",
              "Tcf1",
              "BACH2",
              "Bach2",
              "TBX21",
              "Tbx21",
              "TBET",
              "Tbet",
              "FLI1",
              "Fli1",
              "NR3C1",
              "Nr3c1",
              "EOMES",
              "Eomes",
              "ZEB2",
              "Zeb2",
              "ID2",
              "Id2",
              "ID3",
              "Id3",
              "IRF4",
              "Irf4"
            ]
          },
          {
            "name": "cf_node_character_hidden_driver_recall",
            "role": "hard_fail",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "hidden_driver_activity_not_expression"
          },
          {
            "name": "cf_effect_relation_hidden_pattern",
            "role": "hard_fail",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "activity_exceeds_expression"
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "causal_status",
            "kind": "value",
            "op": "in",
            "expected": [
              "sufficiency_established",
              "necessity_established",
              "causal_established_in_dataset",
              "sufficiency_via_perturbation_in_dataset"
            ]
          },
          {
            "name": "cf_residency_logic_activator_prior",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "residency_activator_induction_primary"
          },
          {
            "name": "cf_residency_logic_single_regulon",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_controls_entire_residency_program"
          },
          {
            "name": "cf_decoy_called_target",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_distractor_model_reported",
            "role": "hard_fail",
            "field": "$.trm_like_program_by_model[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_chromatin_only_decoy",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-sol-pi",
      "modelName": "gpt-5.6-sol",
      "harness": "pi",
      "task": "green_shared_vs_distinct_regulators_trm_vs_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 1,
      "passed": true,
      "score": null,
      "durationS": 3986.814583,
      "cost": 4.280715,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_program_map",
            "kind": "list",
            "matchKey": "regulator",
            "k": 3,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "both"
                    ],
                    "canonical": "both"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "represses"
                    ],
                    "canonical": "represses"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Batf",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "trm_like_til_specific"
                    ],
                    "canonical": "trm_like_til_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "chromatin_over_rna"
                    ],
                    "canonical": "chromatin_over_rna"
                  }
                }
              },
              {
                "key": "Ahr",
                "fields": {
                  "program_specificity": {
                    "op": "in",
                    "values": [
                      "trm_specific",
                      "both"
                    ],
                    "canonical": "trm_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "rna_over_chromatin"
                    ],
                    "canonical": "rna_over_chromatin"
                  }
                }
              }
            ]
          },
          {
            "name": "regulator_set_no_extras_gate",
            "role": "gate",
            "field": "$.regulator_program_map[*]",
            "kind": "other",
            "expected": null
          },
          {
            "name": "program_identity_conclusion_gate",
            "role": "gate",
            "field": "program_identity_conclusion",
            "kind": "value",
            "expected": "distinct_state_shared_brake_divergent_activators"
          },
          {
            "name": "residency_install_logic_gate",
            "role": "gate",
            "field": "residency_install_logic",
            "kind": "value",
            "expected": "brake_release_permits_context_matched_activator"
          },
          {
            "name": "activator_context_dependence_gate",
            "role": "gate",
            "field": "activator_context_dependence",
            "kind": "value",
            "expected": "context_matched_crossover"
          },
          {
            "name": "evidence_ceiling_gate",
            "role": "gate",
            "field": "evidence_ceiling",
            "kind": "value",
            "expected": "specificity_dissociations_require_perturbation"
          },
          {
            "name": "cf_install_logic_flat_or_single_step",
            "role": "hard_fail",
            "field": "residency_install_logic",
            "kind": "value",
            "op": "in",
            "expected": [
              "flat_shared_residency_toolkit",
              "activator_induction_sufficient",
              "brake_release_alone_sufficient"
            ]
          },
          {
            "name": "cf_program_identity_lumped_or_wrong",
            "role": "hard_fail",
            "field": "program_identity_conclusion",
            "kind": "value",
            "op": "in",
            "expected": [
              "trm_like_til_is_tumor_trm",
              "trm_like_til_is_exhausted_subset",
              "trm_like_til_is_effector_state",
              "trm_like_til_is_transitional_intermediate",
              "trm_like_til_is_shared_program_distinct_niche"
            ]
          },
          {
            "name": "cf_ahr_assigned_to_til",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Ahr",
            "valueField": "program_specificity",
            "expected": [
              "trm_like_til_specific"
            ]
          },
          {
            "name": "cf_batf_assigned_to_trm_or_shared",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Batf",
            "valueField": "program_specificity",
            "expected": [
              "trm_specific",
              "both"
            ]
          },
          {
            "name": "cf_klf2_called_activator",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Klf2",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          },
          {
            "name": "cf_evidence_ceiling_overclaim",
            "role": "hard_fail",
            "field": "evidence_ceiling",
            "kind": "value",
            "op": "in",
            "expected": [
              "specificity_established_from_regulon_activity",
              "specificity_established_in_dataset",
              "causation_established_by_cross_sectional_concordance",
              "necessity_and_sufficiency_established"
            ]
          },
          {
            "name": "cf_activator_context_shared_or_single",
            "role": "hard_fail",
            "field": "activator_context_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "shared_activators_both_contexts",
              "single_global_activator",
              "redundant_activators"
            ]
          },
          {
            "name": "cf_single_modality_rule",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_hic1_motif_brake_mislabel",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_runx3_anticanonical_as_controller",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Runx3",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-sol-pi",
      "modelName": "gpt-5.6-sol",
      "harness": "pi",
      "task": "green_shared_vs_distinct_regulators_trm_vs_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 320.986123,
      "cost": 1.14976275,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_program_map",
            "kind": "list",
            "matchKey": "regulator",
            "k": 3,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "both"
                    ],
                    "canonical": "both"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "represses"
                    ],
                    "canonical": "represses"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Batf",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "trm_like_til_specific"
                    ],
                    "canonical": "trm_like_til_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "chromatin_over_rna"
                    ],
                    "canonical": "chromatin_over_rna"
                  }
                }
              },
              {
                "key": "Ahr",
                "fields": {
                  "program_specificity": {
                    "op": "in",
                    "values": [
                      "trm_specific",
                      "both"
                    ],
                    "canonical": "trm_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "rna_over_chromatin"
                    ],
                    "canonical": "rna_over_chromatin"
                  }
                }
              }
            ]
          },
          {
            "name": "regulator_set_no_extras_gate",
            "role": "gate",
            "field": "$.regulator_program_map[*]",
            "kind": "other",
            "expected": null
          },
          {
            "name": "program_identity_conclusion_gate",
            "role": "gate",
            "field": "program_identity_conclusion",
            "kind": "value",
            "expected": "distinct_state_shared_brake_divergent_activators"
          },
          {
            "name": "residency_install_logic_gate",
            "role": "gate",
            "field": "residency_install_logic",
            "kind": "value",
            "expected": "brake_release_permits_context_matched_activator"
          },
          {
            "name": "activator_context_dependence_gate",
            "role": "gate",
            "field": "activator_context_dependence",
            "kind": "value",
            "expected": "context_matched_crossover"
          },
          {
            "name": "evidence_ceiling_gate",
            "role": "gate",
            "field": "evidence_ceiling",
            "kind": "value",
            "expected": "specificity_dissociations_require_perturbation"
          },
          {
            "name": "cf_install_logic_flat_or_single_step",
            "role": "hard_fail",
            "field": "residency_install_logic",
            "kind": "value",
            "op": "in",
            "expected": [
              "flat_shared_residency_toolkit",
              "activator_induction_sufficient",
              "brake_release_alone_sufficient"
            ]
          },
          {
            "name": "cf_program_identity_lumped_or_wrong",
            "role": "hard_fail",
            "field": "program_identity_conclusion",
            "kind": "value",
            "op": "in",
            "expected": [
              "trm_like_til_is_tumor_trm",
              "trm_like_til_is_exhausted_subset",
              "trm_like_til_is_effector_state",
              "trm_like_til_is_transitional_intermediate",
              "trm_like_til_is_shared_program_distinct_niche"
            ]
          },
          {
            "name": "cf_ahr_assigned_to_til",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Ahr",
            "valueField": "program_specificity",
            "expected": [
              "trm_like_til_specific"
            ]
          },
          {
            "name": "cf_batf_assigned_to_trm_or_shared",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Batf",
            "valueField": "program_specificity",
            "expected": [
              "trm_specific",
              "both"
            ]
          },
          {
            "name": "cf_klf2_called_activator",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Klf2",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          },
          {
            "name": "cf_evidence_ceiling_overclaim",
            "role": "hard_fail",
            "field": "evidence_ceiling",
            "kind": "value",
            "op": "in",
            "expected": [
              "specificity_established_from_regulon_activity",
              "specificity_established_in_dataset",
              "causation_established_by_cross_sectional_concordance",
              "necessity_and_sufficiency_established"
            ]
          },
          {
            "name": "cf_activator_context_shared_or_single",
            "role": "hard_fail",
            "field": "activator_context_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "shared_activators_both_contexts",
              "single_global_activator",
              "redundant_activators"
            ]
          },
          {
            "name": "cf_single_modality_rule",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_hic1_motif_brake_mislabel",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_runx3_anticanonical_as_controller",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Runx3",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-sol-pi",
      "modelName": "gpt-5.6-sol",
      "harness": "pi",
      "task": "green_shared_vs_distinct_regulators_trm_vs_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1464.178755,
      "cost": 1.8583037500000001,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_program_map",
            "kind": "list",
            "matchKey": "regulator",
            "k": 3,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "both"
                    ],
                    "canonical": "both"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "represses"
                    ],
                    "canonical": "represses"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Batf",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "trm_like_til_specific"
                    ],
                    "canonical": "trm_like_til_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "chromatin_over_rna"
                    ],
                    "canonical": "chromatin_over_rna"
                  }
                }
              },
              {
                "key": "Ahr",
                "fields": {
                  "program_specificity": {
                    "op": "in",
                    "values": [
                      "trm_specific",
                      "both"
                    ],
                    "canonical": "trm_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "rna_over_chromatin"
                    ],
                    "canonical": "rna_over_chromatin"
                  }
                }
              }
            ]
          },
          {
            "name": "regulator_set_no_extras_gate",
            "role": "gate",
            "field": "$.regulator_program_map[*]",
            "kind": "other",
            "expected": null
          },
          {
            "name": "program_identity_conclusion_gate",
            "role": "gate",
            "field": "program_identity_conclusion",
            "kind": "value",
            "expected": "distinct_state_shared_brake_divergent_activators"
          },
          {
            "name": "residency_install_logic_gate",
            "role": "gate",
            "field": "residency_install_logic",
            "kind": "value",
            "expected": "brake_release_permits_context_matched_activator"
          },
          {
            "name": "activator_context_dependence_gate",
            "role": "gate",
            "field": "activator_context_dependence",
            "kind": "value",
            "expected": "context_matched_crossover"
          },
          {
            "name": "evidence_ceiling_gate",
            "role": "gate",
            "field": "evidence_ceiling",
            "kind": "value",
            "expected": "specificity_dissociations_require_perturbation"
          },
          {
            "name": "cf_install_logic_flat_or_single_step",
            "role": "hard_fail",
            "field": "residency_install_logic",
            "kind": "value",
            "op": "in",
            "expected": [
              "flat_shared_residency_toolkit",
              "activator_induction_sufficient",
              "brake_release_alone_sufficient"
            ]
          },
          {
            "name": "cf_program_identity_lumped_or_wrong",
            "role": "hard_fail",
            "field": "program_identity_conclusion",
            "kind": "value",
            "op": "in",
            "expected": [
              "trm_like_til_is_tumor_trm",
              "trm_like_til_is_exhausted_subset",
              "trm_like_til_is_effector_state",
              "trm_like_til_is_transitional_intermediate",
              "trm_like_til_is_shared_program_distinct_niche"
            ]
          },
          {
            "name": "cf_ahr_assigned_to_til",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Ahr",
            "valueField": "program_specificity",
            "expected": [
              "trm_like_til_specific"
            ]
          },
          {
            "name": "cf_batf_assigned_to_trm_or_shared",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Batf",
            "valueField": "program_specificity",
            "expected": [
              "trm_specific",
              "both"
            ]
          },
          {
            "name": "cf_klf2_called_activator",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Klf2",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          },
          {
            "name": "cf_evidence_ceiling_overclaim",
            "role": "hard_fail",
            "field": "evidence_ceiling",
            "kind": "value",
            "op": "in",
            "expected": [
              "specificity_established_from_regulon_activity",
              "specificity_established_in_dataset",
              "causation_established_by_cross_sectional_concordance",
              "necessity_and_sufficiency_established"
            ]
          },
          {
            "name": "cf_activator_context_shared_or_single",
            "role": "hard_fail",
            "field": "activator_context_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "shared_activators_both_contexts",
              "single_global_activator",
              "redundant_activators"
            ]
          },
          {
            "name": "cf_single_modality_rule",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_hic1_motif_brake_mislabel",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_runx3_anticanonical_as_controller",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Runx3",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-sol-pi",
      "modelName": "gpt-5.6-sol",
      "harness": "pi",
      "task": "gse297023_q2_senescence_not_mechanism",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 1,
      "passed": true,
      "score": null,
      "durationS": 1048.432835,
      "cost": 1.07828375,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "candidate_is_senescence",
            "role": "gate",
            "field": "$.candidate_mechanism",
            "kind": "value",
            "op": "in",
            "expected": [
              "cellular_senescence",
              "oncogene_induced_senescence",
              "replicative_senescence"
            ]
          },
          {
            "name": "arrest_core_not_engaged",
            "role": "gate",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_not_engaged"
            ]
          },
          {
            "name": "secretory_arm_is_inflammaging",
            "role": "gate",
            "field": "$.secretory_arm_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "acute_phase_inflammaging_response"
            ]
          },
          {
            "name": "secretory_arm_broadly_elevated",
            "role": "gate",
            "field": "$.secretory_arm_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "broadly_elevated"
            ]
          },
          {
            "name": "carries_aging_program",
            "role": "gate",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "carries_bona_fide_aging_program_distinct_from_nominated"
            ]
          },
          {
            "name": "conclusion_senescence_unlikely",
            "role": "gate",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "unlikely_major_contributor"
            ]
          },
          {
            "name": "cf_arrest_core_engaged_recall",
            "role": "hard_fail",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_engaged"
            ]
          },
          {
            "name": "cf_aging_program_conflation_or_explain_away",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "aging_program_indistinguishable_from_nominated_program",
              "aging_program_is_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_no_aging_program_sensitivity_failure",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_detectable_aging_program"
            ]
          },
          {
            "name": "cf_senescence_driver_conclusion",
            "role": "hard_fail",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "likely_primary_driver",
              "plausible_supported_contributor"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-sol-pi",
      "modelName": "gpt-5.6-sol",
      "harness": "pi",
      "task": "gse297023_q2_senescence_not_mechanism",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 2,
      "passed": true,
      "score": null,
      "durationS": 780.172852,
      "cost": 1.2778335000000003,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "candidate_is_senescence",
            "role": "gate",
            "field": "$.candidate_mechanism",
            "kind": "value",
            "op": "in",
            "expected": [
              "cellular_senescence",
              "oncogene_induced_senescence",
              "replicative_senescence"
            ]
          },
          {
            "name": "arrest_core_not_engaged",
            "role": "gate",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_not_engaged"
            ]
          },
          {
            "name": "secretory_arm_is_inflammaging",
            "role": "gate",
            "field": "$.secretory_arm_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "acute_phase_inflammaging_response"
            ]
          },
          {
            "name": "secretory_arm_broadly_elevated",
            "role": "gate",
            "field": "$.secretory_arm_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "broadly_elevated"
            ]
          },
          {
            "name": "carries_aging_program",
            "role": "gate",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "carries_bona_fide_aging_program_distinct_from_nominated"
            ]
          },
          {
            "name": "conclusion_senescence_unlikely",
            "role": "gate",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "unlikely_major_contributor"
            ]
          },
          {
            "name": "cf_arrest_core_engaged_recall",
            "role": "hard_fail",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_engaged"
            ]
          },
          {
            "name": "cf_aging_program_conflation_or_explain_away",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "aging_program_indistinguishable_from_nominated_program",
              "aging_program_is_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_no_aging_program_sensitivity_failure",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_detectable_aging_program"
            ]
          },
          {
            "name": "cf_senescence_driver_conclusion",
            "role": "hard_fail",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "likely_primary_driver",
              "plausible_supported_contributor"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-sol-pi",
      "modelName": "gpt-5.6-sol",
      "harness": "pi",
      "task": "gse297023_q2_senescence_not_mechanism",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 3,
      "passed": true,
      "score": null,
      "durationS": 2094.783294,
      "cost": 1.923076,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "candidate_is_senescence",
            "role": "gate",
            "field": "$.candidate_mechanism",
            "kind": "value",
            "op": "in",
            "expected": [
              "cellular_senescence",
              "oncogene_induced_senescence",
              "replicative_senescence"
            ]
          },
          {
            "name": "arrest_core_not_engaged",
            "role": "gate",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_not_engaged"
            ]
          },
          {
            "name": "secretory_arm_is_inflammaging",
            "role": "gate",
            "field": "$.secretory_arm_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "acute_phase_inflammaging_response"
            ]
          },
          {
            "name": "secretory_arm_broadly_elevated",
            "role": "gate",
            "field": "$.secretory_arm_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "broadly_elevated"
            ]
          },
          {
            "name": "carries_aging_program",
            "role": "gate",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "carries_bona_fide_aging_program_distinct_from_nominated"
            ]
          },
          {
            "name": "conclusion_senescence_unlikely",
            "role": "gate",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "unlikely_major_contributor"
            ]
          },
          {
            "name": "cf_arrest_core_engaged_recall",
            "role": "hard_fail",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_engaged"
            ]
          },
          {
            "name": "cf_aging_program_conflation_or_explain_away",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "aging_program_indistinguishable_from_nominated_program",
              "aging_program_is_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_no_aging_program_sensitivity_failure",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_detectable_aging_program"
            ]
          },
          {
            "name": "cf_senescence_driver_conclusion",
            "role": "hard_fail",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "likely_primary_driver",
              "plausible_supported_contributor"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-sol-pi",
      "modelName": "gpt-5.6-sol",
      "harness": "pi",
      "task": "gse297023_q3_age_retained_mapk_reduced",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 1,
      "passed": true,
      "score": null,
      "durationS": 1953.051146,
      "cost": 2.8912430000000002,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "age_retained",
            "role": "gate",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_retained"
            ]
          },
          {
            "name": "identity_is_aging",
            "role": "gate",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_aging_signatures"
            ]
          },
          {
            "name": "nature_state_shift",
            "role": "gate",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_state_expression_shift"
            ]
          },
          {
            "name": "pathway_is_mapk",
            "role": "gate",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "mapk"
            ]
          },
          {
            "name": "pathway_reduced_in_aged",
            "role": "gate",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "reduced_in_aged"
            ]
          },
          {
            "name": "cf_mapk_increased_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_in_aged"
            ]
          },
          {
            "name": "cf_age_erased",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_erased"
            ]
          },
          {
            "name": "cf_pervasive_overcall",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_pervasive_artifactual"
            ]
          },
          {
            "name": "cf_explain_away_aging",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_compositional_or_identity",
            "role": "hard_fail",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_proportion_shift",
              "cell_identity_shift",
              "composition_dominant_state_secondary",
              "both_state_and_composition_equally"
            ]
          },
          {
            "name": "cf_biggest_inflammatory_signal_not_mapk",
            "role": "hard_fail",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "jak_stat",
              "nfkb",
              "tnfa"
            ]
          },
          {
            "name": "cf_senescence_misattribution",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_senescence_signature_specifically"
            ]
          },
          {
            "name": "cf_dismiss_as_batch_artifact",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_batch_or_technical_artifact"
            ]
          },
          {
            "name": "cf_mapk_comparable_constitutive_kras_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "comparable_across_ages"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-sol-pi",
      "modelName": "gpt-5.6-sol",
      "harness": "pi",
      "task": "gse297023_q3_age_retained_mapk_reduced",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 2,
      "passed": true,
      "score": null,
      "durationS": 1734.263494,
      "cost": 2.3821705,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "age_retained",
            "role": "gate",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_retained"
            ]
          },
          {
            "name": "identity_is_aging",
            "role": "gate",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_aging_signatures"
            ]
          },
          {
            "name": "nature_state_shift",
            "role": "gate",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_state_expression_shift"
            ]
          },
          {
            "name": "pathway_is_mapk",
            "role": "gate",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "mapk"
            ]
          },
          {
            "name": "pathway_reduced_in_aged",
            "role": "gate",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "reduced_in_aged"
            ]
          },
          {
            "name": "cf_mapk_increased_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_in_aged"
            ]
          },
          {
            "name": "cf_age_erased",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_erased"
            ]
          },
          {
            "name": "cf_pervasive_overcall",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_pervasive_artifactual"
            ]
          },
          {
            "name": "cf_explain_away_aging",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_compositional_or_identity",
            "role": "hard_fail",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_proportion_shift",
              "cell_identity_shift",
              "composition_dominant_state_secondary",
              "both_state_and_composition_equally"
            ]
          },
          {
            "name": "cf_biggest_inflammatory_signal_not_mapk",
            "role": "hard_fail",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "jak_stat",
              "nfkb",
              "tnfa"
            ]
          },
          {
            "name": "cf_senescence_misattribution",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_senescence_signature_specifically"
            ]
          },
          {
            "name": "cf_dismiss_as_batch_artifact",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_batch_or_technical_artifact"
            ]
          },
          {
            "name": "cf_mapk_comparable_constitutive_kras_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "comparable_across_ages"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-sol-pi",
      "modelName": "gpt-5.6-sol",
      "harness": "pi",
      "task": "gse297023_q3_age_retained_mapk_reduced",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 2201.034995,
      "cost": 4.01873375,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "age_retained",
            "role": "gate",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_retained"
            ]
          },
          {
            "name": "identity_is_aging",
            "role": "gate",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_aging_signatures"
            ]
          },
          {
            "name": "nature_state_shift",
            "role": "gate",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_state_expression_shift"
            ]
          },
          {
            "name": "pathway_is_mapk",
            "role": "gate",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "mapk"
            ]
          },
          {
            "name": "pathway_reduced_in_aged",
            "role": "gate",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "reduced_in_aged"
            ]
          },
          {
            "name": "cf_mapk_increased_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_in_aged"
            ]
          },
          {
            "name": "cf_age_erased",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_erased"
            ]
          },
          {
            "name": "cf_pervasive_overcall",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_pervasive_artifactual"
            ]
          },
          {
            "name": "cf_explain_away_aging",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_compositional_or_identity",
            "role": "hard_fail",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_proportion_shift",
              "cell_identity_shift",
              "composition_dominant_state_secondary",
              "both_state_and_composition_equally"
            ]
          },
          {
            "name": "cf_biggest_inflammatory_signal_not_mapk",
            "role": "hard_fail",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "jak_stat",
              "nfkb",
              "tnfa"
            ]
          },
          {
            "name": "cf_senescence_misattribution",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_senescence_signature_specifically"
            ]
          },
          {
            "name": "cf_dismiss_as_batch_artifact",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_batch_or_technical_artifact"
            ]
          },
          {
            "name": "cf_mapk_comparable_constitutive_kras_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "comparable_across_ages"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-sol-pi",
      "modelName": "gpt-5.6-sol",
      "harness": "pi",
      "task": "gse297023_q5_tme_age_dependent_remodeling",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 1,
      "passed": true,
      "score": null,
      "durationS": 2689.246061,
      "cost": 2.9200702500000006,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "perturbation_is_pten",
            "role": "gate",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "pten"
            ]
          },
          {
            "name": "tme_non_cell_autonomous",
            "role": "gate",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "substantial_non_cell_autonomous_remodeling"
            ]
          },
          {
            "name": "tme_state_not_composition",
            "role": "gate",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "tme_age_dependent",
            "role": "gate",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_dependent"
            ]
          },
          {
            "name": "tme_response_broader_in_aged",
            "role": "gate",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_aged"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "most_responsive_noncancer_cell_types",
            "kind": "list",
            "matchKey": "cell_type",
            "k": 2,
            "expected": [
              {
                "key": "alveolar_macrophage",
                "fields": {}
              },
              {
                "key": "endothelial",
                "fields": {}
              }
            ]
          },
          {
            "name": "cf_cell_intrinsic_only",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "negligible_cell_intrinsic_only_effect",
              "remodeling_unresolved",
              "modest_remodeling_few_cell_types"
            ]
          },
          {
            "name": "cf_remodeling_is_contamination",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "apparent_remodeling_reflects_cancer_contamination"
            ]
          },
          {
            "name": "cf_perturbation_design_misread",
            "role": "hard_fail",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "multiple_tumor_suppressors_pooled",
              "no_additional_gene_difference_kras_only",
              "rb1_cdkn2a_double",
              "pcna_essential_gene_control",
              "difference_is_oncogene_not_suppressor"
            ]
          },
          {
            "name": "cf_age_independent_tme",
            "role": "hard_fail",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_independent"
            ]
          },
          {
            "name": "cf_compositional_shift",
            "role": "hard_fail",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_frequency_shift",
              "compositional_immune_infiltration",
              "composition_dominant_state_secondary",
              "both_state_and_composition"
            ]
          },
          {
            "name": "cf_epithelial_or_cancer_substitution",
            "role": "hard_fail",
            "field": "$.most_responsive_noncancer_cell_types[*].cell_type",
            "kind": "avoid",
            "expected": [
              "AT2_cell",
              "AT1_cell",
              "cancer_cell",
              "ciliated",
              "secretory",
              "club_cell",
              "basal_cell",
              "goblet_cell"
            ]
          },
          {
            "name": "cf_tme_breadth_broader_in_young",
            "role": "hard_fail",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_young"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-sol-pi",
      "modelName": "gpt-5.6-sol",
      "harness": "pi",
      "task": "gse297023_q5_tme_age_dependent_remodeling",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 2,
      "passed": true,
      "score": null,
      "durationS": 3992.133024,
      "cost": 2.9775720000000003,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "perturbation_is_pten",
            "role": "gate",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "pten"
            ]
          },
          {
            "name": "tme_non_cell_autonomous",
            "role": "gate",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "substantial_non_cell_autonomous_remodeling"
            ]
          },
          {
            "name": "tme_state_not_composition",
            "role": "gate",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "tme_age_dependent",
            "role": "gate",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_dependent"
            ]
          },
          {
            "name": "tme_response_broader_in_aged",
            "role": "gate",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_aged"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "most_responsive_noncancer_cell_types",
            "kind": "list",
            "matchKey": "cell_type",
            "k": 2,
            "expected": [
              {
                "key": "alveolar_macrophage",
                "fields": {}
              },
              {
                "key": "endothelial",
                "fields": {}
              }
            ]
          },
          {
            "name": "cf_cell_intrinsic_only",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "negligible_cell_intrinsic_only_effect",
              "remodeling_unresolved",
              "modest_remodeling_few_cell_types"
            ]
          },
          {
            "name": "cf_remodeling_is_contamination",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "apparent_remodeling_reflects_cancer_contamination"
            ]
          },
          {
            "name": "cf_perturbation_design_misread",
            "role": "hard_fail",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "multiple_tumor_suppressors_pooled",
              "no_additional_gene_difference_kras_only",
              "rb1_cdkn2a_double",
              "pcna_essential_gene_control",
              "difference_is_oncogene_not_suppressor"
            ]
          },
          {
            "name": "cf_age_independent_tme",
            "role": "hard_fail",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_independent"
            ]
          },
          {
            "name": "cf_compositional_shift",
            "role": "hard_fail",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_frequency_shift",
              "compositional_immune_infiltration",
              "composition_dominant_state_secondary",
              "both_state_and_composition"
            ]
          },
          {
            "name": "cf_epithelial_or_cancer_substitution",
            "role": "hard_fail",
            "field": "$.most_responsive_noncancer_cell_types[*].cell_type",
            "kind": "avoid",
            "expected": [
              "AT2_cell",
              "AT1_cell",
              "cancer_cell",
              "ciliated",
              "secretory",
              "club_cell",
              "basal_cell",
              "goblet_cell"
            ]
          },
          {
            "name": "cf_tme_breadth_broader_in_young",
            "role": "hard_fail",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_young"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-sol-pi",
      "modelName": "gpt-5.6-sol",
      "harness": "pi",
      "task": "gse297023_q5_tme_age_dependent_remodeling",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 3,
      "passed": true,
      "score": null,
      "durationS": 1131.957989,
      "cost": 2.57379375,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "perturbation_is_pten",
            "role": "gate",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "pten"
            ]
          },
          {
            "name": "tme_non_cell_autonomous",
            "role": "gate",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "substantial_non_cell_autonomous_remodeling"
            ]
          },
          {
            "name": "tme_state_not_composition",
            "role": "gate",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "tme_age_dependent",
            "role": "gate",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_dependent"
            ]
          },
          {
            "name": "tme_response_broader_in_aged",
            "role": "gate",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_aged"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "most_responsive_noncancer_cell_types",
            "kind": "list",
            "matchKey": "cell_type",
            "k": 2,
            "expected": [
              {
                "key": "alveolar_macrophage",
                "fields": {}
              },
              {
                "key": "endothelial",
                "fields": {}
              }
            ]
          },
          {
            "name": "cf_cell_intrinsic_only",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "negligible_cell_intrinsic_only_effect",
              "remodeling_unresolved",
              "modest_remodeling_few_cell_types"
            ]
          },
          {
            "name": "cf_remodeling_is_contamination",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "apparent_remodeling_reflects_cancer_contamination"
            ]
          },
          {
            "name": "cf_perturbation_design_misread",
            "role": "hard_fail",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "multiple_tumor_suppressors_pooled",
              "no_additional_gene_difference_kras_only",
              "rb1_cdkn2a_double",
              "pcna_essential_gene_control",
              "difference_is_oncogene_not_suppressor"
            ]
          },
          {
            "name": "cf_age_independent_tme",
            "role": "hard_fail",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_independent"
            ]
          },
          {
            "name": "cf_compositional_shift",
            "role": "hard_fail",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_frequency_shift",
              "compositional_immune_infiltration",
              "composition_dominant_state_secondary",
              "both_state_and_composition"
            ]
          },
          {
            "name": "cf_epithelial_or_cancer_substitution",
            "role": "hard_fail",
            "field": "$.most_responsive_noncancer_cell_types[*].cell_type",
            "kind": "avoid",
            "expected": [
              "AT2_cell",
              "AT1_cell",
              "cancer_cell",
              "ciliated",
              "secretory",
              "club_cell",
              "basal_cell",
              "goblet_cell"
            ]
          },
          {
            "name": "cf_tme_breadth_broader_in_young",
            "role": "hard_fail",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_young"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-sol-pi",
      "modelName": "gpt-5.6-sol",
      "harness": "pi",
      "task": "gse297023_q6_age_immune_infiltration",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 1,
      "passed": true,
      "score": null,
      "durationS": 1635.582761,
      "cost": 2.8346325000000006,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "composition_stable_with_age",
            "role": "gate",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "frequencies_largely_stable"
            ]
          },
          {
            "name": "age_signal_is_state_not_composition",
            "role": "gate",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "frequencies_uninformative_for_infiltration",
            "role": "gate",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_cannot_establish_infiltration"
            ]
          },
          {
            "name": "infiltration_increases_with_age",
            "role": "gate",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_with_age"
            ]
          },
          {
            "name": "infiltration_strength_calibrated",
            "role": "gate",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "significant_increase_strongest_markers_only",
              "consistent_increase_not_statistically_robust"
            ]
          },
          {
            "name": "genotype_marker_fraction_lower_in_pten",
            "role": "gate",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "lower_in_pten_deficient"
            ]
          },
          {
            "name": "cf_composition_shift_reading",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "major_compositional_shift_with_age",
              "increased_immune_fraction_with_age"
            ]
          },
          {
            "name": "cf_lineage_specific_composition_claim",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "myeloid_expansion_with_age",
              "lymphoid_expansion_with_age",
              "minor_shifts_in_some_cell_types"
            ]
          },
          {
            "name": "cf_immunosenescence_depletion_prior",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "immune_depletion_with_age"
            ]
          },
          {
            "name": "cf_age_modality_compositional",
            "role": "hard_fail",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_type_composition_shift",
              "both_equally",
              "composition_dominant_state_secondary"
            ]
          },
          {
            "name": "cf_overread_frequencies_as_infiltration",
            "role": "hard_fail",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_shows_no_infiltration_change",
              "single_cell_quantifies_infiltration",
              "frequencies_show_increased_infiltration",
              "frequencies_show_decreased_infiltration"
            ]
          },
          {
            "name": "cf_infiltration_decreases_immunosenescence",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "decreased_with_age"
            ]
          },
          {
            "name": "cf_infiltration_unchanged_scrna_overread",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "unchanged_with_age"
            ]
          },
          {
            "name": "cf_infiltration_indeterminate_unused_node",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "direction_indeterminate_from_data"
            ]
          },
          {
            "name": "cf_infiltration_mixed_direction",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "mixed_by_immune_cell_type"
            ]
          },
          {
            "name": "cf_infiltration_genotype_confusion",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_only_in_pten_deficient",
              "decreased_only_in_pten_deficient"
            ]
          },
          {
            "name": "cf_pten_infiltration_paper_text_recall",
            "role": "hard_fail",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "higher_in_pten_deficient",
              "higher_in_pten_deficient_only_in_young",
              "higher_in_pten_deficient_only_in_aged"
            ]
          },
          {
            "name": "cf_pseudoreplication_overclaim",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_increase_across_markers"
            ]
          },
          {
            "name": "cf_infiltration_strength_decrease_direction",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_decrease_across_markers",
              "significant_decrease_strongest_markers_only",
              "consistent_decrease_not_statistically_robust"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-sol-pi",
      "modelName": "gpt-5.6-sol",
      "harness": "pi",
      "task": "gse297023_q6_age_immune_infiltration",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 2,
      "passed": true,
      "score": null,
      "durationS": 3196.528947,
      "cost": 2.3883982500000003,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "composition_stable_with_age",
            "role": "gate",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "frequencies_largely_stable"
            ]
          },
          {
            "name": "age_signal_is_state_not_composition",
            "role": "gate",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "frequencies_uninformative_for_infiltration",
            "role": "gate",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_cannot_establish_infiltration"
            ]
          },
          {
            "name": "infiltration_increases_with_age",
            "role": "gate",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_with_age"
            ]
          },
          {
            "name": "infiltration_strength_calibrated",
            "role": "gate",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "significant_increase_strongest_markers_only",
              "consistent_increase_not_statistically_robust"
            ]
          },
          {
            "name": "genotype_marker_fraction_lower_in_pten",
            "role": "gate",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "lower_in_pten_deficient"
            ]
          },
          {
            "name": "cf_composition_shift_reading",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "major_compositional_shift_with_age",
              "increased_immune_fraction_with_age"
            ]
          },
          {
            "name": "cf_lineage_specific_composition_claim",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "myeloid_expansion_with_age",
              "lymphoid_expansion_with_age",
              "minor_shifts_in_some_cell_types"
            ]
          },
          {
            "name": "cf_immunosenescence_depletion_prior",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "immune_depletion_with_age"
            ]
          },
          {
            "name": "cf_age_modality_compositional",
            "role": "hard_fail",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_type_composition_shift",
              "both_equally",
              "composition_dominant_state_secondary"
            ]
          },
          {
            "name": "cf_overread_frequencies_as_infiltration",
            "role": "hard_fail",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_shows_no_infiltration_change",
              "single_cell_quantifies_infiltration",
              "frequencies_show_increased_infiltration",
              "frequencies_show_decreased_infiltration"
            ]
          },
          {
            "name": "cf_infiltration_decreases_immunosenescence",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "decreased_with_age"
            ]
          },
          {
            "name": "cf_infiltration_unchanged_scrna_overread",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "unchanged_with_age"
            ]
          },
          {
            "name": "cf_infiltration_indeterminate_unused_node",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "direction_indeterminate_from_data"
            ]
          },
          {
            "name": "cf_infiltration_mixed_direction",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "mixed_by_immune_cell_type"
            ]
          },
          {
            "name": "cf_infiltration_genotype_confusion",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_only_in_pten_deficient",
              "decreased_only_in_pten_deficient"
            ]
          },
          {
            "name": "cf_pten_infiltration_paper_text_recall",
            "role": "hard_fail",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "higher_in_pten_deficient",
              "higher_in_pten_deficient_only_in_young",
              "higher_in_pten_deficient_only_in_aged"
            ]
          },
          {
            "name": "cf_pseudoreplication_overclaim",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_increase_across_markers"
            ]
          },
          {
            "name": "cf_infiltration_strength_decrease_direction",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_decrease_across_markers",
              "significant_decrease_strongest_markers_only",
              "consistent_decrease_not_statistically_robust"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-sol-pi",
      "modelName": "gpt-5.6-sol",
      "harness": "pi",
      "task": "gse297023_q6_age_immune_infiltration",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 3,
      "passed": true,
      "score": null,
      "durationS": 2035.961534,
      "cost": 1.6919449999999998,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "composition_stable_with_age",
            "role": "gate",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "frequencies_largely_stable"
            ]
          },
          {
            "name": "age_signal_is_state_not_composition",
            "role": "gate",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "frequencies_uninformative_for_infiltration",
            "role": "gate",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_cannot_establish_infiltration"
            ]
          },
          {
            "name": "infiltration_increases_with_age",
            "role": "gate",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_with_age"
            ]
          },
          {
            "name": "infiltration_strength_calibrated",
            "role": "gate",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "significant_increase_strongest_markers_only",
              "consistent_increase_not_statistically_robust"
            ]
          },
          {
            "name": "genotype_marker_fraction_lower_in_pten",
            "role": "gate",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "lower_in_pten_deficient"
            ]
          },
          {
            "name": "cf_composition_shift_reading",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "major_compositional_shift_with_age",
              "increased_immune_fraction_with_age"
            ]
          },
          {
            "name": "cf_lineage_specific_composition_claim",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "myeloid_expansion_with_age",
              "lymphoid_expansion_with_age",
              "minor_shifts_in_some_cell_types"
            ]
          },
          {
            "name": "cf_immunosenescence_depletion_prior",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "immune_depletion_with_age"
            ]
          },
          {
            "name": "cf_age_modality_compositional",
            "role": "hard_fail",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_type_composition_shift",
              "both_equally",
              "composition_dominant_state_secondary"
            ]
          },
          {
            "name": "cf_overread_frequencies_as_infiltration",
            "role": "hard_fail",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_shows_no_infiltration_change",
              "single_cell_quantifies_infiltration",
              "frequencies_show_increased_infiltration",
              "frequencies_show_decreased_infiltration"
            ]
          },
          {
            "name": "cf_infiltration_decreases_immunosenescence",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "decreased_with_age"
            ]
          },
          {
            "name": "cf_infiltration_unchanged_scrna_overread",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "unchanged_with_age"
            ]
          },
          {
            "name": "cf_infiltration_indeterminate_unused_node",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "direction_indeterminate_from_data"
            ]
          },
          {
            "name": "cf_infiltration_mixed_direction",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "mixed_by_immune_cell_type"
            ]
          },
          {
            "name": "cf_infiltration_genotype_confusion",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_only_in_pten_deficient",
              "decreased_only_in_pten_deficient"
            ]
          },
          {
            "name": "cf_pten_infiltration_paper_text_recall",
            "role": "hard_fail",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "higher_in_pten_deficient",
              "higher_in_pten_deficient_only_in_young",
              "higher_in_pten_deficient_only_in_aged"
            ]
          },
          {
            "name": "cf_pseudoreplication_overclaim",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_increase_across_markers"
            ]
          },
          {
            "name": "cf_infiltration_strength_decrease_direction",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_decrease_across_markers",
              "significant_decrease_strongest_markers_only",
              "consistent_decrease_not_statistically_robust"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-terra-pi",
      "modelName": "gpt-5.6-terra",
      "harness": "pi",
      "task": "gse155381_q4_pseudotime_delay",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 530.996132,
      "cost": 1.01751625,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "pseudotime_direction_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "E17_chronological_match_weighted",
            "role": "additive",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pluripotency_exit_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pseudotime_direction_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_synchronous_with_monkey_EPI",
              "chimera_human_EPI_developmentally_advanced",
              "chimera_human_EPI_delayed_at_pre_implantation_only_resolved_post_implantation"
            ]
          },
          {
            "name": "E17_chronological_match_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "resembles_monkey_E17_Gast_EPI",
              "resembles_chimera_monkey_E17_EPI_specifically",
              "chimera_human_E17_EPI_more_advanced_than_monkey_E17_Gast"
            ]
          },
          {
            "name": "pluripotency_exit_paper_inverted_or_prior_driven",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_advances_pluripotency_exit_prematurely",
              "naive_pluripotency_program_specifically_engaged",
              "pluripotency_core_retained_via_KLF17_naive_marker_specifically"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-terra-pi",
      "modelName": "gpt-5.6-terra",
      "harness": "pi",
      "task": "gse155381_q4_pseudotime_delay",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 770.814694,
      "cost": 1.522175375,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "pseudotime_direction_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "E17_chronological_match_weighted",
            "role": "additive",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pluripotency_exit_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pseudotime_direction_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_synchronous_with_monkey_EPI",
              "chimera_human_EPI_developmentally_advanced",
              "chimera_human_EPI_delayed_at_pre_implantation_only_resolved_post_implantation"
            ]
          },
          {
            "name": "E17_chronological_match_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "resembles_monkey_E17_Gast_EPI",
              "resembles_chimera_monkey_E17_EPI_specifically",
              "chimera_human_E17_EPI_more_advanced_than_monkey_E17_Gast"
            ]
          },
          {
            "name": "pluripotency_exit_paper_inverted_or_prior_driven",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_advances_pluripotency_exit_prematurely",
              "naive_pluripotency_program_specifically_engaged",
              "pluripotency_core_retained_via_KLF17_naive_marker_specifically"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-terra-pi",
      "modelName": "gpt-5.6-terra",
      "harness": "pi",
      "task": "gse155381_q4_pseudotime_delay",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 902.149448,
      "cost": 1.4402320000000002,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "pseudotime_direction_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "E17_chronological_match_weighted",
            "role": "additive",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pluripotency_exit_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pseudotime_direction_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_synchronous_with_monkey_EPI",
              "chimera_human_EPI_developmentally_advanced",
              "chimera_human_EPI_delayed_at_pre_implantation_only_resolved_post_implantation"
            ]
          },
          {
            "name": "E17_chronological_match_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "resembles_monkey_E17_Gast_EPI",
              "resembles_chimera_monkey_E17_EPI_specifically",
              "chimera_human_E17_EPI_more_advanced_than_monkey_E17_Gast"
            ]
          },
          {
            "name": "pluripotency_exit_paper_inverted_or_prior_driven",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_advances_pluripotency_exit_prematurely",
              "naive_pluripotency_program_specifically_engaged",
              "pluripotency_core_retained_via_KLF17_naive_marker_specifically"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-terra-pi",
      "modelName": "gpt-5.6-terra",
      "harness": "pi",
      "task": "gse155381_q5_chimera_monkey_perturbation",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 1049.766648,
      "cost": 0.766550875,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "deg_direction_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "comparison_subset_choice_weighted",
            "role": "additive",
            "field": "$.comparison_subset_choice",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "panel_resolved_strong_target_ccn1_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN1",
              "CYR61"
            ]
          },
          {
            "name": "panel_resolved_strong_target_ccn2_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN2",
              "CTGF"
            ]
          },
          {
            "name": "hippo_pathway_weighted",
            "role": "additive",
            "field": "$.hippo_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "tgf_beta_pathway_weighted",
            "role": "additive",
            "field": "$.tgf_beta_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "deg_direction_batch_effect_uncorrected",
            "role": "hard_fail",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_dominated_global",
              "up_dominated_in_chosen_subset"
            ]
          },
          {
            "name": "silent_panel_dropout_paired_with_directional_pathway_claim",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "hippo_pathway_direction": {
                "op": "in",
                "values": [
                  "hippo_pathway_upregulated",
                  "hippo_pathway_unchanged",
                  "wnt_pathway_specifically_perturbed",
                  "stress_response_pathway_dominantly_engaged",
                  "no_pathway_signal_above_confound",
                  "yap_tead_direct_target_panel_downregulated",
                  "yap_tead_panel_dependent_direction",
                  "hippo_pathway_downregulated"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-terra-pi",
      "modelName": "gpt-5.6-terra",
      "harness": "pi",
      "task": "gse155381_q5_chimera_monkey_perturbation",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 560.682921,
      "cost": 0.888713875,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "deg_direction_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "comparison_subset_choice_weighted",
            "role": "additive",
            "field": "$.comparison_subset_choice",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "panel_resolved_strong_target_ccn1_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN1",
              "CYR61"
            ]
          },
          {
            "name": "panel_resolved_strong_target_ccn2_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN2",
              "CTGF"
            ]
          },
          {
            "name": "hippo_pathway_weighted",
            "role": "additive",
            "field": "$.hippo_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "tgf_beta_pathway_weighted",
            "role": "additive",
            "field": "$.tgf_beta_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "deg_direction_batch_effect_uncorrected",
            "role": "hard_fail",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_dominated_global",
              "up_dominated_in_chosen_subset"
            ]
          },
          {
            "name": "silent_panel_dropout_paired_with_directional_pathway_claim",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "hippo_pathway_direction": {
                "op": "in",
                "values": [
                  "hippo_pathway_upregulated",
                  "hippo_pathway_unchanged",
                  "wnt_pathway_specifically_perturbed",
                  "stress_response_pathway_dominantly_engaged",
                  "no_pathway_signal_above_confound",
                  "yap_tead_direct_target_panel_downregulated",
                  "yap_tead_panel_dependent_direction",
                  "hippo_pathway_downregulated"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-terra-pi",
      "modelName": "gpt-5.6-terra",
      "harness": "pi",
      "task": "gse155381_q5_chimera_monkey_perturbation",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 374.275881,
      "cost": 1.112091625,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "deg_direction_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "comparison_subset_choice_weighted",
            "role": "additive",
            "field": "$.comparison_subset_choice",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "panel_resolved_strong_target_ccn1_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN1",
              "CYR61"
            ]
          },
          {
            "name": "panel_resolved_strong_target_ccn2_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN2",
              "CTGF"
            ]
          },
          {
            "name": "hippo_pathway_weighted",
            "role": "additive",
            "field": "$.hippo_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "tgf_beta_pathway_weighted",
            "role": "additive",
            "field": "$.tgf_beta_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "deg_direction_batch_effect_uncorrected",
            "role": "hard_fail",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_dominated_global",
              "up_dominated_in_chosen_subset"
            ]
          },
          {
            "name": "silent_panel_dropout_paired_with_directional_pathway_claim",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "hippo_pathway_direction": {
                "op": "in",
                "values": [
                  "hippo_pathway_upregulated",
                  "hippo_pathway_unchanged",
                  "wnt_pathway_specifically_perturbed",
                  "stress_response_pathway_dominantly_engaged",
                  "no_pathway_signal_above_confound",
                  "yap_tead_direct_target_panel_downregulated",
                  "yap_tead_panel_dependent_direction",
                  "hippo_pathway_downregulated"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-terra-pi",
      "modelName": "gpt-5.6-terra",
      "harness": "pi",
      "task": "gse155381_q7_xspecies_epi_communication",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 1246.741147,
      "cost": 1.1569632499999998,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "richness_weighted",
            "role": "additive",
            "field": "$.xspecies_signaling_richness",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_weighted",
            "role": "additive",
            "field": "$.dominant_signaling_axis",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_wnt_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "WNT_FZD_LRP_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_fgf_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "FGF_FGFR_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_notch_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "NOTCH_DLL_JAG_family"
                ]
              }
            }
          },
          {
            "name": "signaling_directionality_weighted",
            "role": "additive",
            "field": "$.signaling_directionality",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "top_signal_interaction_class_weighted",
            "role": "additive",
            "field": "$.top_signal_interaction_class",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-terra-pi",
      "modelName": "gpt-5.6-terra",
      "harness": "pi",
      "task": "gse155381_q7_xspecies_epi_communication",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 874.072296,
      "cost": 1.0725083750000002,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "richness_weighted",
            "role": "additive",
            "field": "$.xspecies_signaling_richness",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_weighted",
            "role": "additive",
            "field": "$.dominant_signaling_axis",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_wnt_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "WNT_FZD_LRP_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_fgf_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "FGF_FGFR_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_notch_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "NOTCH_DLL_JAG_family"
                ]
              }
            }
          },
          {
            "name": "signaling_directionality_weighted",
            "role": "additive",
            "field": "$.signaling_directionality",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "top_signal_interaction_class_weighted",
            "role": "additive",
            "field": "$.top_signal_interaction_class",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-terra-pi",
      "modelName": "gpt-5.6-terra",
      "harness": "pi",
      "task": "gse155381_q7_xspecies_epi_communication",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 555.911947,
      "cost": 1.62562225,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "richness_weighted",
            "role": "additive",
            "field": "$.xspecies_signaling_richness",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_weighted",
            "role": "additive",
            "field": "$.dominant_signaling_axis",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_wnt_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "WNT_FZD_LRP_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_fgf_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "FGF_FGFR_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_notch_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "NOTCH_DLL_JAG_family"
                ]
              }
            }
          },
          {
            "name": "signaling_directionality_weighted",
            "role": "additive",
            "field": "$.signaling_directionality",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "top_signal_interaction_class_weighted",
            "role": "additive",
            "field": "$.top_signal_interaction_class",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-terra-pi",
      "modelName": "gpt-5.6-terra",
      "harness": "pi",
      "task": "gse155381_q8_bidirectional_convergence",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 1368.501424,
      "cost": 4.510892874999999,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "chimera_human_identity_shift_target_weighted",
            "role": "additive",
            "field": "$.chimera_human_identity_shift_target",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_in_vivo_state_assignment_weighted",
            "role": "additive",
            "field": "$.chimera_human_in_vivo_state_assignment",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_shift_magnitude_weighted",
            "role": "additive",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_monkey_identity_shift_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "both_arms_null_contradicted_by_calibration",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "chimera_human_identity_shift_target": {
                "op": "equals",
                "values": [
                  "chimera_human_maintains_origin_identity_no_measurable_shift"
                ]
              },
              "chimera_monkey_identity_shift": {
                "op": "equals",
                "values": [
                  "chimera_monkey_unchanged_relative_to_control"
                ]
              }
            }
          },
          {
            "name": "chimera_monkey_paper_claim_on_confounded_input",
            "role": "hard_fail",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "value",
            "expected": "chimera_monkey_shifts_toward_chimera_human_identity"
          },
          {
            "name": "chimera_human_magnitude_wrong_attribution_signature",
            "role": "hard_fail",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "value",
            "op": "in",
            "expected": [
              "dramatic_complete_replacement_of_origin_identity",
              "magnitude_dominated_by_culture_or_metabolic_axis",
              "magnitude_dominated_by_developmental_stage_axis",
              "magnitude_dominated_by_cell_cycle_axis",
              "magnitude_dominated_by_batch_effect_axis"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-terra-pi",
      "modelName": "gpt-5.6-terra",
      "harness": "pi",
      "task": "gse155381_q8_bidirectional_convergence",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 2247.259767,
      "cost": 2.162842125,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "chimera_human_identity_shift_target_weighted",
            "role": "additive",
            "field": "$.chimera_human_identity_shift_target",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_in_vivo_state_assignment_weighted",
            "role": "additive",
            "field": "$.chimera_human_in_vivo_state_assignment",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_shift_magnitude_weighted",
            "role": "additive",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_monkey_identity_shift_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "both_arms_null_contradicted_by_calibration",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "chimera_human_identity_shift_target": {
                "op": "equals",
                "values": [
                  "chimera_human_maintains_origin_identity_no_measurable_shift"
                ]
              },
              "chimera_monkey_identity_shift": {
                "op": "equals",
                "values": [
                  "chimera_monkey_unchanged_relative_to_control"
                ]
              }
            }
          },
          {
            "name": "chimera_monkey_paper_claim_on_confounded_input",
            "role": "hard_fail",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "value",
            "expected": "chimera_monkey_shifts_toward_chimera_human_identity"
          },
          {
            "name": "chimera_human_magnitude_wrong_attribution_signature",
            "role": "hard_fail",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "value",
            "op": "in",
            "expected": [
              "dramatic_complete_replacement_of_origin_identity",
              "magnitude_dominated_by_culture_or_metabolic_axis",
              "magnitude_dominated_by_developmental_stage_axis",
              "magnitude_dominated_by_cell_cycle_axis",
              "magnitude_dominated_by_batch_effect_axis"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-terra-pi",
      "modelName": "gpt-5.6-terra",
      "harness": "pi",
      "task": "gse155381_q8_bidirectional_convergence",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 898.471754,
      "cost": 2.83213575,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "chimera_human_identity_shift_target_weighted",
            "role": "additive",
            "field": "$.chimera_human_identity_shift_target",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_in_vivo_state_assignment_weighted",
            "role": "additive",
            "field": "$.chimera_human_in_vivo_state_assignment",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_shift_magnitude_weighted",
            "role": "additive",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_monkey_identity_shift_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "both_arms_null_contradicted_by_calibration",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "chimera_human_identity_shift_target": {
                "op": "equals",
                "values": [
                  "chimera_human_maintains_origin_identity_no_measurable_shift"
                ]
              },
              "chimera_monkey_identity_shift": {
                "op": "equals",
                "values": [
                  "chimera_monkey_unchanged_relative_to_control"
                ]
              }
            }
          },
          {
            "name": "chimera_monkey_paper_claim_on_confounded_input",
            "role": "hard_fail",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "value",
            "expected": "chimera_monkey_shifts_toward_chimera_human_identity"
          },
          {
            "name": "chimera_human_magnitude_wrong_attribution_signature",
            "role": "hard_fail",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "value",
            "op": "in",
            "expected": [
              "dramatic_complete_replacement_of_origin_identity",
              "magnitude_dominated_by_culture_or_metabolic_axis",
              "magnitude_dominated_by_developmental_stage_axis",
              "magnitude_dominated_by_cell_cycle_axis",
              "magnitude_dominated_by_batch_effect_axis"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-terra-pi",
      "modelName": "gpt-5.6-terra",
      "harness": "pi",
      "task": "gse171524_ectopic_airway_rare_cell",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 1,
      "passed": true,
      "score": null,
      "durationS": 934.560106,
      "cost": 1.381638375,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "focus_population_identity_gate",
            "role": "gate",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "chemosensory_marker_expressing_population_alpha"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "focus_population_marker_calls",
            "kind": "list",
            "matchKey": "marker",
            "k": 3,
            "expected": [
              {
                "key": "POU2F3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "CHAT",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "TRPM5",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "LRMP",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNAT3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "AVIL",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNG13",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              }
            ]
          },
          {
            "name": "focus_population_condition_distribution_gate",
            "role": "gate",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_enriched_with_substantial_control_presence",
              "covid_predominant",
              "covid_only",
              "covid_enriched_trend_not_significant"
            ]
          },
          {
            "name": "focus_population_location_gate",
            "role": "gate",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "ectopic_parenchyma_in_covid",
              "parenchyma_only",
              "bronchiolized_or_peribronchial_parenchyma",
              "airway_and_parenchyma"
            ]
          },
          {
            "name": "focus_population_recovery_mode_gate",
            "role": "gate",
            "field": "$.focus_population_recovery_mode",
            "kind": "value",
            "op": "in",
            "expected": [
              "continuum_not_discrete_cluster"
            ]
          },
          {
            "name": "focus_population_identity_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "ionocyte_like_beta",
              "neuroendocrine_like_gamma",
              "basal_stem_progenitor_like_delta",
              "aberrant_basaloid_transitional_like_epsilon",
              "neuronal_or_neural_gene_minor_population_zeta",
              "deuterosomal_ciliated_progenitor_like_eta",
              "hillock_transitional_like_theta",
              "low_quality_doublet_or_debris",
              "no_rare_cell_recovered",
              "not_assessed"
            ]
          },
          {
            "name": "cf_wrong_lineage_marker_called_positive",
            "role": "hard_fail",
            "field": "$.focus_population_marker_calls[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "focus_population_condition_distribution_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "equally_distributed",
              "control_predominant",
              "not_assessed"
            ]
          },
          {
            "name": "focus_population_location_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "airway_only",
              "not_assessed"
            ]
          },
          {
            "name": "cross_axis_canonical_identity_with_airway_only_location_incoherence",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "focus_population_identity": {
                "op": "in",
                "values": [
                  "chemosensory_marker_expressing_population_alpha"
                ]
              },
              "focus_population_location": {
                "op": "in",
                "values": [
                  "airway_only"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-terra-pi",
      "modelName": "gpt-5.6-terra",
      "harness": "pi",
      "task": "gse171524_ectopic_airway_rare_cell",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 2,
      "passed": true,
      "score": null,
      "durationS": null,
      "cost": null,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "focus_population_identity_gate",
            "role": "gate",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "chemosensory_marker_expressing_population_alpha"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "focus_population_marker_calls",
            "kind": "list",
            "matchKey": "marker",
            "k": 3,
            "expected": [
              {
                "key": "POU2F3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "CHAT",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "TRPM5",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "LRMP",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNAT3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "AVIL",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNG13",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              }
            ]
          },
          {
            "name": "focus_population_condition_distribution_gate",
            "role": "gate",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_enriched_with_substantial_control_presence",
              "covid_predominant",
              "covid_only",
              "covid_enriched_trend_not_significant"
            ]
          },
          {
            "name": "focus_population_location_gate",
            "role": "gate",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "ectopic_parenchyma_in_covid",
              "parenchyma_only",
              "bronchiolized_or_peribronchial_parenchyma",
              "airway_and_parenchyma"
            ]
          },
          {
            "name": "focus_population_recovery_mode_gate",
            "role": "gate",
            "field": "$.focus_population_recovery_mode",
            "kind": "value",
            "op": "in",
            "expected": [
              "continuum_not_discrete_cluster"
            ]
          },
          {
            "name": "focus_population_identity_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "ionocyte_like_beta",
              "neuroendocrine_like_gamma",
              "basal_stem_progenitor_like_delta",
              "aberrant_basaloid_transitional_like_epsilon",
              "neuronal_or_neural_gene_minor_population_zeta",
              "deuterosomal_ciliated_progenitor_like_eta",
              "hillock_transitional_like_theta",
              "low_quality_doublet_or_debris",
              "no_rare_cell_recovered",
              "not_assessed"
            ]
          },
          {
            "name": "cf_wrong_lineage_marker_called_positive",
            "role": "hard_fail",
            "field": "$.focus_population_marker_calls[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "focus_population_condition_distribution_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "equally_distributed",
              "control_predominant",
              "not_assessed"
            ]
          },
          {
            "name": "focus_population_location_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "airway_only",
              "not_assessed"
            ]
          },
          {
            "name": "cross_axis_canonical_identity_with_airway_only_location_incoherence",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "focus_population_identity": {
                "op": "in",
                "values": [
                  "chemosensory_marker_expressing_population_alpha"
                ]
              },
              "focus_population_location": {
                "op": "in",
                "values": [
                  "airway_only"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-terra-pi",
      "modelName": "gpt-5.6-terra",
      "harness": "pi",
      "task": "gse171524_ectopic_airway_rare_cell",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1622.518767,
      "cost": 3.369135875,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "focus_population_identity_gate",
            "role": "gate",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "chemosensory_marker_expressing_population_alpha"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "focus_population_marker_calls",
            "kind": "list",
            "matchKey": "marker",
            "k": 3,
            "expected": [
              {
                "key": "POU2F3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "CHAT",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "TRPM5",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "LRMP",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNAT3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "AVIL",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNG13",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              }
            ]
          },
          {
            "name": "focus_population_condition_distribution_gate",
            "role": "gate",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_enriched_with_substantial_control_presence",
              "covid_predominant",
              "covid_only",
              "covid_enriched_trend_not_significant"
            ]
          },
          {
            "name": "focus_population_location_gate",
            "role": "gate",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "ectopic_parenchyma_in_covid",
              "parenchyma_only",
              "bronchiolized_or_peribronchial_parenchyma",
              "airway_and_parenchyma"
            ]
          },
          {
            "name": "focus_population_recovery_mode_gate",
            "role": "gate",
            "field": "$.focus_population_recovery_mode",
            "kind": "value",
            "op": "in",
            "expected": [
              "continuum_not_discrete_cluster"
            ]
          },
          {
            "name": "focus_population_identity_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "ionocyte_like_beta",
              "neuroendocrine_like_gamma",
              "basal_stem_progenitor_like_delta",
              "aberrant_basaloid_transitional_like_epsilon",
              "neuronal_or_neural_gene_minor_population_zeta",
              "deuterosomal_ciliated_progenitor_like_eta",
              "hillock_transitional_like_theta",
              "low_quality_doublet_or_debris",
              "no_rare_cell_recovered",
              "not_assessed"
            ]
          },
          {
            "name": "cf_wrong_lineage_marker_called_positive",
            "role": "hard_fail",
            "field": "$.focus_population_marker_calls[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "focus_population_condition_distribution_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "equally_distributed",
              "control_predominant",
              "not_assessed"
            ]
          },
          {
            "name": "focus_population_location_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "airway_only",
              "not_assessed"
            ]
          },
          {
            "name": "cross_axis_canonical_identity_with_airway_only_location_incoherence",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "focus_population_identity": {
                "op": "in",
                "values": [
                  "chemosensory_marker_expressing_population_alpha"
                ]
              },
              "focus_population_location": {
                "op": "in",
                "values": [
                  "airway_only"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-terra-pi",
      "modelName": "gpt-5.6-terra",
      "harness": "pi",
      "task": "gse171524_pathological_fibroblast_master_regulator_druggable_targets",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 1,
      "passed": true,
      "score": null,
      "durationS": 5216.524838,
      "cost": 1.2085363750000002,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "disease_associated_fibroblast_subtype_weighted",
            "role": "additive",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "mr_tf_junb_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_alpha"
            ]
          },
          {
            "name": "mr_tf_jund_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_beta"
            ]
          },
          {
            "name": "drug_mmp14_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_alpha"
            ]
          },
          {
            "name": "drug_stat3_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_beta"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_weighted",
            "role": "additive",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "disease_associated_fibroblast_subtype_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "value",
            "op": "in",
            "expected": [
              "subtype_lambda",
              "subtype_epsilon",
              "subtype_iota",
              "subtype_beta",
              "subtype_eta",
              "subtype_zeta",
              "subtype_not_assessed"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "simple_de_ranking_by_log_fc",
              "marker_based_de_only",
              "not_computed"
            ]
          },
          {
            "name": "top_master_regulator_transcription_factors_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_not_computed"
            ]
          },
          {
            "name": "top_druggable_proteins_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_not_assessed"
            ]
          },
          {
            "name": "cross_axis_circular_marker_in_druggable_proteins",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_gamma",
              "drug_target_delta",
              "drug_target_eta"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-terra-pi",
      "modelName": "gpt-5.6-terra",
      "harness": "pi",
      "task": "gse171524_pathological_fibroblast_master_regulator_druggable_targets",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 2,
      "passed": true,
      "score": null,
      "durationS": 2086.605691,
      "cost": 1.0582843749999997,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "disease_associated_fibroblast_subtype_weighted",
            "role": "additive",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "mr_tf_junb_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_alpha"
            ]
          },
          {
            "name": "mr_tf_jund_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_beta"
            ]
          },
          {
            "name": "drug_mmp14_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_alpha"
            ]
          },
          {
            "name": "drug_stat3_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_beta"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_weighted",
            "role": "additive",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "disease_associated_fibroblast_subtype_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "value",
            "op": "in",
            "expected": [
              "subtype_lambda",
              "subtype_epsilon",
              "subtype_iota",
              "subtype_beta",
              "subtype_eta",
              "subtype_zeta",
              "subtype_not_assessed"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "simple_de_ranking_by_log_fc",
              "marker_based_de_only",
              "not_computed"
            ]
          },
          {
            "name": "top_master_regulator_transcription_factors_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_not_computed"
            ]
          },
          {
            "name": "top_druggable_proteins_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_not_assessed"
            ]
          },
          {
            "name": "cross_axis_circular_marker_in_druggable_proteins",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_gamma",
              "drug_target_delta",
              "drug_target_eta"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-terra-pi",
      "modelName": "gpt-5.6-terra",
      "harness": "pi",
      "task": "gse171524_pathological_fibroblast_master_regulator_druggable_targets",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 3,
      "passed": true,
      "score": null,
      "durationS": 1637.875726,
      "cost": 1.1229136250000002,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "disease_associated_fibroblast_subtype_weighted",
            "role": "additive",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "mr_tf_junb_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_alpha"
            ]
          },
          {
            "name": "mr_tf_jund_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_beta"
            ]
          },
          {
            "name": "drug_mmp14_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_alpha"
            ]
          },
          {
            "name": "drug_stat3_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_beta"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_weighted",
            "role": "additive",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "disease_associated_fibroblast_subtype_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "value",
            "op": "in",
            "expected": [
              "subtype_lambda",
              "subtype_epsilon",
              "subtype_iota",
              "subtype_beta",
              "subtype_eta",
              "subtype_zeta",
              "subtype_not_assessed"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "simple_de_ranking_by_log_fc",
              "marker_based_de_only",
              "not_computed"
            ]
          },
          {
            "name": "top_master_regulator_transcription_factors_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_not_computed"
            ]
          },
          {
            "name": "top_druggable_proteins_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_not_assessed"
            ]
          },
          {
            "name": "cross_axis_circular_marker_in_druggable_proteins",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_gamma",
              "drug_target_delta",
              "drug_target_eta"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-terra-pi",
      "modelName": "gpt-5.6-terra",
      "harness": "pi",
      "task": "gse171524_plasma_bcr_neutralizing_clonotype",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 163.727391,
      "cost": 0.396906625,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dominant_pairing_heavy_gate",
            "role": "gate",
            "field": "$.dominant_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV4-34"
            ]
          },
          {
            "name": "dominant_pairing_light_gate",
            "role": "gate",
            "field": "$.dominant_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGLV3-19"
            ]
          },
          {
            "name": "dominant_neutralizer_match_gate",
            "role": "gate",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_known_neutralizer_match"
            ]
          },
          {
            "name": "neutralizer_pairing_heavy_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV1-18"
            ]
          },
          {
            "name": "neutralizer_pairing_light_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGKV3-20"
            ]
          },
          {
            "name": "neutralizer_recurrence_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "observed_in_2_to_5_donors"
            ]
          },
          {
            "name": "neutralizer_rank_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "top_5_pairing",
              "top_6_to_20_pairing"
            ]
          },
          {
            "name": "dominant_clone_isotype_gate",
            "role": "gate",
            "field": "$.dominant_clone_isotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "IgG"
            ]
          },
          {
            "name": "blineage_class_switch_contrast_gate",
            "role": "gate",
            "field": "$.blineage_class_switch_contrast",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_class_switched_control_higher_unswitched",
              "class_switched_in_both_cohorts"
            ]
          },
          {
            "name": "dominant_match_narrative_trap_or_null",
            "role": "hard_fail",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_rbd_neutralizer",
              "matches_known_autoreactive_clone",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_heavy",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_light",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_recurrence_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_rank_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "cross_axis_found_dominant_but_failed_to_scan_for_embedded_neutralizer",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_pairing_neutralizer_match": {
                "op": "in",
                "values": [
                  "no_known_neutralizer_match"
                ]
              },
              "neutralizer_pairing_heavy": {
                "op": "in",
                "values": [
                  "none_detected"
                ]
              }
            }
          },
          {
            "name": "neutralizer_pairing_not_exactly_the_s309_pair",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "op": "not",
            "expected": {
              "op": "and",
              "args": [
                {
                  "op": "field",
                  "name": "neutralizer_pairing_heavy",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGHV1-18"
                    ]
                  }
                },
                {
                  "op": "field",
                  "name": "neutralizer_pairing_light",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGKV3-20"
                    ]
                  }
                }
              ]
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-terra-pi",
      "modelName": "gpt-5.6-terra",
      "harness": "pi",
      "task": "gse171524_plasma_bcr_neutralizing_clonotype",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 266.594388,
      "cost": 0.5975171250000001,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dominant_pairing_heavy_gate",
            "role": "gate",
            "field": "$.dominant_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV4-34"
            ]
          },
          {
            "name": "dominant_pairing_light_gate",
            "role": "gate",
            "field": "$.dominant_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGLV3-19"
            ]
          },
          {
            "name": "dominant_neutralizer_match_gate",
            "role": "gate",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_known_neutralizer_match"
            ]
          },
          {
            "name": "neutralizer_pairing_heavy_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV1-18"
            ]
          },
          {
            "name": "neutralizer_pairing_light_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGKV3-20"
            ]
          },
          {
            "name": "neutralizer_recurrence_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "observed_in_2_to_5_donors"
            ]
          },
          {
            "name": "neutralizer_rank_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "top_5_pairing",
              "top_6_to_20_pairing"
            ]
          },
          {
            "name": "dominant_clone_isotype_gate",
            "role": "gate",
            "field": "$.dominant_clone_isotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "IgG"
            ]
          },
          {
            "name": "blineage_class_switch_contrast_gate",
            "role": "gate",
            "field": "$.blineage_class_switch_contrast",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_class_switched_control_higher_unswitched",
              "class_switched_in_both_cohorts"
            ]
          },
          {
            "name": "dominant_match_narrative_trap_or_null",
            "role": "hard_fail",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_rbd_neutralizer",
              "matches_known_autoreactive_clone",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_heavy",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_light",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_recurrence_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_rank_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "cross_axis_found_dominant_but_failed_to_scan_for_embedded_neutralizer",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_pairing_neutralizer_match": {
                "op": "in",
                "values": [
                  "no_known_neutralizer_match"
                ]
              },
              "neutralizer_pairing_heavy": {
                "op": "in",
                "values": [
                  "none_detected"
                ]
              }
            }
          },
          {
            "name": "neutralizer_pairing_not_exactly_the_s309_pair",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "op": "not",
            "expected": {
              "op": "and",
              "args": [
                {
                  "op": "field",
                  "name": "neutralizer_pairing_heavy",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGHV1-18"
                    ]
                  }
                },
                {
                  "op": "field",
                  "name": "neutralizer_pairing_light",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGKV3-20"
                    ]
                  }
                }
              ]
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-terra-pi",
      "modelName": "gpt-5.6-terra",
      "harness": "pi",
      "task": "gse171524_plasma_bcr_neutralizing_clonotype",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 206.787918,
      "cost": 0.438475375,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dominant_pairing_heavy_gate",
            "role": "gate",
            "field": "$.dominant_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV4-34"
            ]
          },
          {
            "name": "dominant_pairing_light_gate",
            "role": "gate",
            "field": "$.dominant_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGLV3-19"
            ]
          },
          {
            "name": "dominant_neutralizer_match_gate",
            "role": "gate",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_known_neutralizer_match"
            ]
          },
          {
            "name": "neutralizer_pairing_heavy_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV1-18"
            ]
          },
          {
            "name": "neutralizer_pairing_light_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGKV3-20"
            ]
          },
          {
            "name": "neutralizer_recurrence_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "observed_in_2_to_5_donors"
            ]
          },
          {
            "name": "neutralizer_rank_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "top_5_pairing",
              "top_6_to_20_pairing"
            ]
          },
          {
            "name": "dominant_clone_isotype_gate",
            "role": "gate",
            "field": "$.dominant_clone_isotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "IgG"
            ]
          },
          {
            "name": "blineage_class_switch_contrast_gate",
            "role": "gate",
            "field": "$.blineage_class_switch_contrast",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_class_switched_control_higher_unswitched",
              "class_switched_in_both_cohorts"
            ]
          },
          {
            "name": "dominant_match_narrative_trap_or_null",
            "role": "hard_fail",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_rbd_neutralizer",
              "matches_known_autoreactive_clone",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_heavy",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_light",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_recurrence_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_rank_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "cross_axis_found_dominant_but_failed_to_scan_for_embedded_neutralizer",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_pairing_neutralizer_match": {
                "op": "in",
                "values": [
                  "no_known_neutralizer_match"
                ]
              },
              "neutralizer_pairing_heavy": {
                "op": "in",
                "values": [
                  "none_detected"
                ]
              }
            }
          },
          {
            "name": "neutralizer_pairing_not_exactly_the_s309_pair",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "op": "not",
            "expected": {
              "op": "and",
              "args": [
                {
                  "op": "field",
                  "name": "neutralizer_pairing_heavy",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGHV1-18"
                    ]
                  }
                },
                {
                  "op": "field",
                  "name": "neutralizer_pairing_light",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGKV3-20"
                    ]
                  }
                }
              ]
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-terra-pi",
      "modelName": "gpt-5.6-terra",
      "harness": "pi",
      "task": "q6_myeloid_subset_restricted_de",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 1,
      "passed": true,
      "score": null,
      "durationS": 723.920513,
      "cost": 0.569670625,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "myeloid_subset_with_strongest_change_gate",
            "role": "gate",
            "field": "$.myeloid_subset_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "subset_gamma"
            ]
          },
          {
            "name": "gene_with_strongest_change_gate",
            "role": "gate",
            "field": "$.gene_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "AXL",
              "PROS1"
            ]
          },
          {
            "name": "direction_of_change_gate",
            "role": "gate",
            "field": "$.direction_of_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "down_in_covid"
            ]
          },
          {
            "name": "biological_function_interpretation_gate",
            "role": "gate",
            "field": "$.biological_function_interpretation",
            "kind": "value",
            "op": "in",
            "expected": [
              "efferocytosis",
              "phagocytosis"
            ]
          },
          {
            "name": "myeloid_compartment_abundance_gate",
            "role": "gate",
            "field": "$.myeloid_compartment_abundance",
            "kind": "value",
            "op": "in",
            "expected": [
              "expands_in_covid"
            ]
          },
          {
            "name": "myeloid_expansion_localization_gate",
            "role": "gate",
            "field": "$.myeloid_expansion_localization",
            "kind": "value",
            "op": "in",
            "expected": [
              "recruited_driven_resident_dysfunction"
            ]
          },
          {
            "name": "dominant_signature_gate",
            "role": "gate",
            "field": "$.dominant_signature",
            "kind": "value",
            "op": "in",
            "expected": [
              "technical_or_agonal_artifact"
            ]
          },
          {
            "name": "dominant_signature_direction_gate",
            "role": "gate",
            "field": "$.dominant_signature_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_in_covid"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-terra-pi",
      "modelName": "gpt-5.6-terra",
      "harness": "pi",
      "task": "q6_myeloid_subset_restricted_de",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": null,
      "cost": null,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "myeloid_subset_with_strongest_change_gate",
            "role": "gate",
            "field": "$.myeloid_subset_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "subset_gamma"
            ]
          },
          {
            "name": "gene_with_strongest_change_gate",
            "role": "gate",
            "field": "$.gene_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "AXL",
              "PROS1"
            ]
          },
          {
            "name": "direction_of_change_gate",
            "role": "gate",
            "field": "$.direction_of_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "down_in_covid"
            ]
          },
          {
            "name": "biological_function_interpretation_gate",
            "role": "gate",
            "field": "$.biological_function_interpretation",
            "kind": "value",
            "op": "in",
            "expected": [
              "efferocytosis",
              "phagocytosis"
            ]
          },
          {
            "name": "myeloid_compartment_abundance_gate",
            "role": "gate",
            "field": "$.myeloid_compartment_abundance",
            "kind": "value",
            "op": "in",
            "expected": [
              "expands_in_covid"
            ]
          },
          {
            "name": "myeloid_expansion_localization_gate",
            "role": "gate",
            "field": "$.myeloid_expansion_localization",
            "kind": "value",
            "op": "in",
            "expected": [
              "recruited_driven_resident_dysfunction"
            ]
          },
          {
            "name": "dominant_signature_gate",
            "role": "gate",
            "field": "$.dominant_signature",
            "kind": "value",
            "op": "in",
            "expected": [
              "technical_or_agonal_artifact"
            ]
          },
          {
            "name": "dominant_signature_direction_gate",
            "role": "gate",
            "field": "$.dominant_signature_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_in_covid"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-terra-pi",
      "modelName": "gpt-5.6-terra",
      "harness": "pi",
      "task": "q6_myeloid_subset_restricted_de",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": null,
      "cost": null,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "myeloid_subset_with_strongest_change_gate",
            "role": "gate",
            "field": "$.myeloid_subset_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "subset_gamma"
            ]
          },
          {
            "name": "gene_with_strongest_change_gate",
            "role": "gate",
            "field": "$.gene_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "AXL",
              "PROS1"
            ]
          },
          {
            "name": "direction_of_change_gate",
            "role": "gate",
            "field": "$.direction_of_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "down_in_covid"
            ]
          },
          {
            "name": "biological_function_interpretation_gate",
            "role": "gate",
            "field": "$.biological_function_interpretation",
            "kind": "value",
            "op": "in",
            "expected": [
              "efferocytosis",
              "phagocytosis"
            ]
          },
          {
            "name": "myeloid_compartment_abundance_gate",
            "role": "gate",
            "field": "$.myeloid_compartment_abundance",
            "kind": "value",
            "op": "in",
            "expected": [
              "expands_in_covid"
            ]
          },
          {
            "name": "myeloid_expansion_localization_gate",
            "role": "gate",
            "field": "$.myeloid_expansion_localization",
            "kind": "value",
            "op": "in",
            "expected": [
              "recruited_driven_resident_dysfunction"
            ]
          },
          {
            "name": "dominant_signature_gate",
            "role": "gate",
            "field": "$.dominant_signature",
            "kind": "value",
            "op": "in",
            "expected": [
              "technical_or_agonal_artifact"
            ]
          },
          {
            "name": "dominant_signature_direction_gate",
            "role": "gate",
            "field": "$.dominant_signature_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_in_covid"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-terra-pi",
      "modelName": "gpt-5.6-terra",
      "harness": "pi",
      "task": "snrnaseq_per_donor_composition_shifts_lethal_covid_lung",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 1,
      "passed": true,
      "score": null,
      "durationS": 802.003407,
      "cost": 0.684227625,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dict_match",
            "role": "gate",
            "field": "cell_type_direction_map",
            "kind": "dict",
            "expected": {
              "Myeloid": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "Epithelia": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Fibroblasts": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "T_NK": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "B_plasma": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Endothelial": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Mast": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "APC": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Neuronal": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              }
            }
          },
          {
            "name": "over_merge_immune_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Immune",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_leukocyte_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Leukocyte",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_lymphoid_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Lymphoid",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_stromal_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Stromal",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-terra-pi",
      "modelName": "gpt-5.6-terra",
      "harness": "pi",
      "task": "snrnaseq_per_donor_composition_shifts_lethal_covid_lung",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 3556.815859,
      "cost": 0.8806353750000001,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dict_match",
            "role": "gate",
            "field": "cell_type_direction_map",
            "kind": "dict",
            "expected": {
              "Myeloid": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "Epithelia": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Fibroblasts": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "T_NK": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "B_plasma": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Endothelial": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Mast": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "APC": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Neuronal": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              }
            }
          },
          {
            "name": "over_merge_immune_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Immune",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_leukocyte_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Leukocyte",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_lymphoid_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Lymphoid",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_stromal_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Stromal",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-terra-pi",
      "modelName": "gpt-5.6-terra",
      "harness": "pi",
      "task": "snrnaseq_per_donor_composition_shifts_lethal_covid_lung",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 3548.696637,
      "cost": 1.2790139999999999,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dict_match",
            "role": "gate",
            "field": "cell_type_direction_map",
            "kind": "dict",
            "expected": {
              "Myeloid": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "Epithelia": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Fibroblasts": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "T_NK": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "B_plasma": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Endothelial": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Mast": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "APC": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Neuronal": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              }
            }
          },
          {
            "name": "over_merge_immune_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Immune",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_leukocyte_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Leukocyte",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_lymphoid_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Lymphoid",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_stromal_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Stromal",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-terra-pi",
      "modelName": "gpt-5.6-terra",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_enriched_apc_states_v2",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 1249.894457,
      "cost": 1.037812625,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "top_3_cluster_enriched_apc_states",
            "kind": "list",
            "matchKey": "apc_state",
            "k": 3,
            "expected": [
              {
                "key": "plasmacytoid_DC",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "T_cell_stimulatory_role_supported",
                      "role_direction_unresolved"
                    ],
                    "canonical": "T_cell_stimulatory_role_supported"
                  }
                }
              },
              {
                "key": "C1qhigh_lipid_associated_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "C1qhigh_inflammatory_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "plasma_cell",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "cohort_significant_per_patient_inconsistent"
                    ],
                    "canonical": "cohort_significant_per_patient_inconsistent"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CD80_CD86": {
                "op": "in",
                "value": [
                  "mreg_DC",
                  "shared_across_APC_states"
                ]
              },
              "PD_L1_in_APC_clusters": {
                "op": "in",
                "value": [
                  "C1qhigh_macrophage_either_subtype",
                  "shared_across_APC_states",
                  "mreg_DC"
                ]
              },
              "type_I_IFN": {
                "op": "in",
                "value": [
                  "plasmacytoid_DC",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "cf_mreg_dc_claimed_enriched",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "mreg_DC",
              "mature_DC_enriched_in_immunoregulatory_molecules"
            ]
          },
          {
            "name": "cf_framework_offcompartment_or_generic_substitution",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "cDC1",
              "Maier_mregDC",
              "Cheng_C1QC_TAM",
              "Cheng_SPP1_TAM",
              "Cheng_FOLR2_TAM",
              "Villani_pDC",
              "Zhang_inflammatory_monocyte",
              "LAMP3_pos_DC",
              "myeloid_derived_suppressor_cell",
              "MDSC",
              "tumor_associated_neutrophil",
              "mast_cell",
              "APC",
              "antigen_presenting_cell",
              "macrophage",
              "dendritic_cell",
              "tumor_associated_macrophage",
              "TAM"
            ]
          },
          {
            "name": "cf_direction_commit_on_unresolvable_slot",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_not_resolvable_hedge",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].functional_attribution_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_type_I_IFN_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.type_I_IFN",
            "kind": "value",
            "op": "in",
            "expected": [
              "mreg_DC",
              "C1qhigh_macrophage_either_subtype"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-terra-pi",
      "modelName": "gpt-5.6-terra",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_enriched_apc_states_v2",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 1821.376031,
      "cost": 2.930628875,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "top_3_cluster_enriched_apc_states",
            "kind": "list",
            "matchKey": "apc_state",
            "k": 3,
            "expected": [
              {
                "key": "plasmacytoid_DC",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "T_cell_stimulatory_role_supported",
                      "role_direction_unresolved"
                    ],
                    "canonical": "T_cell_stimulatory_role_supported"
                  }
                }
              },
              {
                "key": "C1qhigh_lipid_associated_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "C1qhigh_inflammatory_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "plasma_cell",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "cohort_significant_per_patient_inconsistent"
                    ],
                    "canonical": "cohort_significant_per_patient_inconsistent"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CD80_CD86": {
                "op": "in",
                "value": [
                  "mreg_DC",
                  "shared_across_APC_states"
                ]
              },
              "PD_L1_in_APC_clusters": {
                "op": "in",
                "value": [
                  "C1qhigh_macrophage_either_subtype",
                  "shared_across_APC_states",
                  "mreg_DC"
                ]
              },
              "type_I_IFN": {
                "op": "in",
                "value": [
                  "plasmacytoid_DC",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "cf_mreg_dc_claimed_enriched",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "mreg_DC",
              "mature_DC_enriched_in_immunoregulatory_molecules"
            ]
          },
          {
            "name": "cf_framework_offcompartment_or_generic_substitution",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "cDC1",
              "Maier_mregDC",
              "Cheng_C1QC_TAM",
              "Cheng_SPP1_TAM",
              "Cheng_FOLR2_TAM",
              "Villani_pDC",
              "Zhang_inflammatory_monocyte",
              "LAMP3_pos_DC",
              "myeloid_derived_suppressor_cell",
              "MDSC",
              "tumor_associated_neutrophil",
              "mast_cell",
              "APC",
              "antigen_presenting_cell",
              "macrophage",
              "dendritic_cell",
              "tumor_associated_macrophage",
              "TAM"
            ]
          },
          {
            "name": "cf_direction_commit_on_unresolvable_slot",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_not_resolvable_hedge",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].functional_attribution_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_type_I_IFN_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.type_I_IFN",
            "kind": "value",
            "op": "in",
            "expected": [
              "mreg_DC",
              "C1qhigh_macrophage_either_subtype"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-terra-pi",
      "modelName": "gpt-5.6-terra",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_enriched_apc_states_v2",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1283.477702,
      "cost": 1.6284838750000001,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "top_3_cluster_enriched_apc_states",
            "kind": "list",
            "matchKey": "apc_state",
            "k": 3,
            "expected": [
              {
                "key": "plasmacytoid_DC",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "T_cell_stimulatory_role_supported",
                      "role_direction_unresolved"
                    ],
                    "canonical": "T_cell_stimulatory_role_supported"
                  }
                }
              },
              {
                "key": "C1qhigh_lipid_associated_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "C1qhigh_inflammatory_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "plasma_cell",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "cohort_significant_per_patient_inconsistent"
                    ],
                    "canonical": "cohort_significant_per_patient_inconsistent"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CD80_CD86": {
                "op": "in",
                "value": [
                  "mreg_DC",
                  "shared_across_APC_states"
                ]
              },
              "PD_L1_in_APC_clusters": {
                "op": "in",
                "value": [
                  "C1qhigh_macrophage_either_subtype",
                  "shared_across_APC_states",
                  "mreg_DC"
                ]
              },
              "type_I_IFN": {
                "op": "in",
                "value": [
                  "plasmacytoid_DC",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "cf_mreg_dc_claimed_enriched",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "mreg_DC",
              "mature_DC_enriched_in_immunoregulatory_molecules"
            ]
          },
          {
            "name": "cf_framework_offcompartment_or_generic_substitution",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "cDC1",
              "Maier_mregDC",
              "Cheng_C1QC_TAM",
              "Cheng_SPP1_TAM",
              "Cheng_FOLR2_TAM",
              "Villani_pDC",
              "Zhang_inflammatory_monocyte",
              "LAMP3_pos_DC",
              "myeloid_derived_suppressor_cell",
              "MDSC",
              "tumor_associated_neutrophil",
              "mast_cell",
              "APC",
              "antigen_presenting_cell",
              "macrophage",
              "dendritic_cell",
              "tumor_associated_macrophage",
              "TAM"
            ]
          },
          {
            "name": "cf_direction_commit_on_unresolvable_slot",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_not_resolvable_hedge",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].functional_attribution_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_type_I_IFN_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.type_I_IFN",
            "kind": "value",
            "op": "in",
            "expected": [
              "mreg_DC",
              "C1qhigh_macrophage_either_subtype"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-terra-pi",
      "modelName": "gpt-5.6-terra",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_tumor_state_causation_v5",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 1386.630864,
      "cost": 2.0049171249999995,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "enriched_tumor_states",
            "kind": "list",
            "matchKey": "tumor_state",
            "k": 3,
            "expected": [
              {
                "key": "immune_response",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "direction_unresolved_cross_sectional"
                    ],
                    "canonical": "direction_unresolved_cross_sectional"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "IFN_response_axis",
                      "broad_antigen_presentation_MHC"
                    ],
                    "canonical": "IFN_response_axis"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "enrichment_diminished_by_decontamination",
                      "threshold_sensitive"
                    ],
                    "canonical": "enrichment_diminished_by_decontamination"
                  }
                }
              },
              {
                "key": "stress_hypoxia",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "data_supports_cell_intrinsic_cause"
                    ],
                    "canonical": "data_supports_cell_intrinsic_cause"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "HIF_hypoxia_program_specific"
                    ],
                    "canonical": "HIF_hypoxia_program_specific"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "robust_across_methods",
                      "robust_after_decontamination"
                    ],
                    "canonical": "robust_across_methods"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CXCL9_CXCL10": {
                "op": "in",
                "value": [
                  "immune_response_tumor_state"
                ]
              },
              "CCL5": {
                "op": "in",
                "value": [
                  "shared_across_tumor_states",
                  "partner_lymphocyte_derived",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "gate_conjugate_gate_confounder",
            "role": "gate",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "partner_lymphocyte_capture"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "antigen_presentation_decomposition",
            "kind": "dict",
            "expected": {
              "MHC_class_I": {
                "op": "in",
                "value": [
                  "co_regulated_with_dominant_program",
                  "tumor_intrinsic_enriched_independent"
                ]
              },
              "MHC_class_II": {
                "op": "in",
                "value": [
                  "partner_cell_attributable",
                  "flat_not_enriched",
                  "method_sensitive_no_robust_call",
                  "co_regulated_with_dominant_program"
                ]
              }
            }
          },
          {
            "name": "cf_non_robust_state_claimed_enriched",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "transitory_melanocytic",
              "intermediate_melanocytic",
              "melanocytic",
              "MITF_high",
              "neural_crest_like",
              "mitotic",
              "stress_p53",
              "undifferentiated",
              "dedifferentiated",
              "mesenchymal_tumor",
              "AXL_high_tumor",
              "invasive_tumor",
              "senescence_like",
              "EMT_transition",
              "cancer_stem_cell_like",
              "oxidative_stress_tumor"
            ]
          },
          {
            "name": "cf_framework_generic_or_offtissue_substitution",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "Tsoi_undifferentiated",
              "Tsoi_neural_crest_like",
              "Tsoi_transitory",
              "Tsoi_melanocytic",
              "Rambow_NCSC",
              "Wouters_mesenchymal",
              "Pozniak_TCF4_dependent",
              "tumor_cell",
              "melanoma_cell",
              "malignant_cell",
              "tumor_subpopulation",
              "keratinocyte",
              "hepatocyte",
              "lung_adenocarcinoma_cell",
              "colon_adenocarcinoma_cell",
              "Tirosh_cell_cycle_program",
              "Jerby_Arnon_program",
              "glioblastoma_cell",
              "breast_carcinoma_cell"
            ]
          },
          {
            "name": "cf_immune_response_direction_committed",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_immune_robustness_recall",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_ambiguous_hedge",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].causation_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_stress_generic_stress_response_evidence",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_conjugate_confounder_contamination_denied",
            "role": "hard_fail",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected"
            ]
          },
          {
            "name": "cf_ccl5_recall_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CCL5",
            "kind": "avoid",
            "expected": [
              "immune_response_tumor_state"
            ]
          },
          {
            "name": "cf_cxcl9_cxcl10_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CXCL9_CXCL10",
            "kind": "avoid",
            "expected": [
              "stress_hypoxia_tumor_state"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-terra-pi",
      "modelName": "gpt-5.6-terra",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_tumor_state_causation_v5",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 808.239956,
      "cost": 1.4282425000000003,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "enriched_tumor_states",
            "kind": "list",
            "matchKey": "tumor_state",
            "k": 3,
            "expected": [
              {
                "key": "immune_response",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "direction_unresolved_cross_sectional"
                    ],
                    "canonical": "direction_unresolved_cross_sectional"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "IFN_response_axis",
                      "broad_antigen_presentation_MHC"
                    ],
                    "canonical": "IFN_response_axis"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "enrichment_diminished_by_decontamination",
                      "threshold_sensitive"
                    ],
                    "canonical": "enrichment_diminished_by_decontamination"
                  }
                }
              },
              {
                "key": "stress_hypoxia",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "data_supports_cell_intrinsic_cause"
                    ],
                    "canonical": "data_supports_cell_intrinsic_cause"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "HIF_hypoxia_program_specific"
                    ],
                    "canonical": "HIF_hypoxia_program_specific"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "robust_across_methods",
                      "robust_after_decontamination"
                    ],
                    "canonical": "robust_across_methods"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CXCL9_CXCL10": {
                "op": "in",
                "value": [
                  "immune_response_tumor_state"
                ]
              },
              "CCL5": {
                "op": "in",
                "value": [
                  "shared_across_tumor_states",
                  "partner_lymphocyte_derived",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "gate_conjugate_gate_confounder",
            "role": "gate",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "partner_lymphocyte_capture"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "antigen_presentation_decomposition",
            "kind": "dict",
            "expected": {
              "MHC_class_I": {
                "op": "in",
                "value": [
                  "co_regulated_with_dominant_program",
                  "tumor_intrinsic_enriched_independent"
                ]
              },
              "MHC_class_II": {
                "op": "in",
                "value": [
                  "partner_cell_attributable",
                  "flat_not_enriched",
                  "method_sensitive_no_robust_call",
                  "co_regulated_with_dominant_program"
                ]
              }
            }
          },
          {
            "name": "cf_non_robust_state_claimed_enriched",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "transitory_melanocytic",
              "intermediate_melanocytic",
              "melanocytic",
              "MITF_high",
              "neural_crest_like",
              "mitotic",
              "stress_p53",
              "undifferentiated",
              "dedifferentiated",
              "mesenchymal_tumor",
              "AXL_high_tumor",
              "invasive_tumor",
              "senescence_like",
              "EMT_transition",
              "cancer_stem_cell_like",
              "oxidative_stress_tumor"
            ]
          },
          {
            "name": "cf_framework_generic_or_offtissue_substitution",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "Tsoi_undifferentiated",
              "Tsoi_neural_crest_like",
              "Tsoi_transitory",
              "Tsoi_melanocytic",
              "Rambow_NCSC",
              "Wouters_mesenchymal",
              "Pozniak_TCF4_dependent",
              "tumor_cell",
              "melanoma_cell",
              "malignant_cell",
              "tumor_subpopulation",
              "keratinocyte",
              "hepatocyte",
              "lung_adenocarcinoma_cell",
              "colon_adenocarcinoma_cell",
              "Tirosh_cell_cycle_program",
              "Jerby_Arnon_program",
              "glioblastoma_cell",
              "breast_carcinoma_cell"
            ]
          },
          {
            "name": "cf_immune_response_direction_committed",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_immune_robustness_recall",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_ambiguous_hedge",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].causation_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_stress_generic_stress_response_evidence",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_conjugate_confounder_contamination_denied",
            "role": "hard_fail",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected"
            ]
          },
          {
            "name": "cf_ccl5_recall_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CCL5",
            "kind": "avoid",
            "expected": [
              "immune_response_tumor_state"
            ]
          },
          {
            "name": "cf_cxcl9_cxcl10_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CXCL9_CXCL10",
            "kind": "avoid",
            "expected": [
              "stress_hypoxia_tumor_state"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-terra-pi",
      "modelName": "gpt-5.6-terra",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_tumor_state_causation_v5",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1495.90373,
      "cost": 1.2119979999999997,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "enriched_tumor_states",
            "kind": "list",
            "matchKey": "tumor_state",
            "k": 3,
            "expected": [
              {
                "key": "immune_response",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "direction_unresolved_cross_sectional"
                    ],
                    "canonical": "direction_unresolved_cross_sectional"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "IFN_response_axis",
                      "broad_antigen_presentation_MHC"
                    ],
                    "canonical": "IFN_response_axis"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "enrichment_diminished_by_decontamination",
                      "threshold_sensitive"
                    ],
                    "canonical": "enrichment_diminished_by_decontamination"
                  }
                }
              },
              {
                "key": "stress_hypoxia",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "data_supports_cell_intrinsic_cause"
                    ],
                    "canonical": "data_supports_cell_intrinsic_cause"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "HIF_hypoxia_program_specific"
                    ],
                    "canonical": "HIF_hypoxia_program_specific"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "robust_across_methods",
                      "robust_after_decontamination"
                    ],
                    "canonical": "robust_across_methods"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CXCL9_CXCL10": {
                "op": "in",
                "value": [
                  "immune_response_tumor_state"
                ]
              },
              "CCL5": {
                "op": "in",
                "value": [
                  "shared_across_tumor_states",
                  "partner_lymphocyte_derived",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "gate_conjugate_gate_confounder",
            "role": "gate",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "partner_lymphocyte_capture"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "antigen_presentation_decomposition",
            "kind": "dict",
            "expected": {
              "MHC_class_I": {
                "op": "in",
                "value": [
                  "co_regulated_with_dominant_program",
                  "tumor_intrinsic_enriched_independent"
                ]
              },
              "MHC_class_II": {
                "op": "in",
                "value": [
                  "partner_cell_attributable",
                  "flat_not_enriched",
                  "method_sensitive_no_robust_call",
                  "co_regulated_with_dominant_program"
                ]
              }
            }
          },
          {
            "name": "cf_non_robust_state_claimed_enriched",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "transitory_melanocytic",
              "intermediate_melanocytic",
              "melanocytic",
              "MITF_high",
              "neural_crest_like",
              "mitotic",
              "stress_p53",
              "undifferentiated",
              "dedifferentiated",
              "mesenchymal_tumor",
              "AXL_high_tumor",
              "invasive_tumor",
              "senescence_like",
              "EMT_transition",
              "cancer_stem_cell_like",
              "oxidative_stress_tumor"
            ]
          },
          {
            "name": "cf_framework_generic_or_offtissue_substitution",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "Tsoi_undifferentiated",
              "Tsoi_neural_crest_like",
              "Tsoi_transitory",
              "Tsoi_melanocytic",
              "Rambow_NCSC",
              "Wouters_mesenchymal",
              "Pozniak_TCF4_dependent",
              "tumor_cell",
              "melanoma_cell",
              "malignant_cell",
              "tumor_subpopulation",
              "keratinocyte",
              "hepatocyte",
              "lung_adenocarcinoma_cell",
              "colon_adenocarcinoma_cell",
              "Tirosh_cell_cycle_program",
              "Jerby_Arnon_program",
              "glioblastoma_cell",
              "breast_carcinoma_cell"
            ]
          },
          {
            "name": "cf_immune_response_direction_committed",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_immune_robustness_recall",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_ambiguous_hedge",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].causation_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_stress_generic_stress_response_evidence",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_conjugate_confounder_contamination_denied",
            "role": "hard_fail",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected"
            ]
          },
          {
            "name": "cf_ccl5_recall_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CCL5",
            "kind": "avoid",
            "expected": [
              "immune_response_tumor_state"
            ]
          },
          {
            "name": "cf_cxcl9_cxcl10_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CXCL9_CXCL10",
            "kind": "avoid",
            "expected": [
              "stress_hypoxia_tumor_state"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-terra-pi",
      "modelName": "gpt-5.6-terra",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_vs_cd39_tumor_reactivity_dominance",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 814.192847,
      "cost": 0.9459267499999999,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "population_findings",
            "kind": "list",
            "matchKey": "capture_gate",
            "k": 4,
            "expected": [
              {
                "key": "tumor_cell_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "apc_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "combined_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "unselected_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "naive",
                      "central_memory",
                      "effector_memory"
                    ],
                    "canonical": "naive"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "low"
                    ],
                    "canonical": "low"
                  },
                  "purity_caveat": {
                    "op": "equals",
                    "values": [
                      "pure_T_cell"
                    ],
                    "canonical": "pure_T_cell"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "no_reactivity_evidence"
                    ],
                    "canonical": "no_reactivity_evidence"
                  }
                }
              },
              {
                "key": "marker_sorted_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "equals",
                    "values": [
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "terminal_exhausted_LAG3_high"
                  },
                  "clonal_expansion_level": {
                    "op": "in",
                    "values": [
                      "high",
                      "intermediate"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "marker_on_non_T_lineages",
                      "single_cell_gating_loses_target"
                    ],
                    "canonical": "marker_on_non_T_lineages"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "marker_or_exhaustion_defined_only"
                    ],
                    "canonical": "marker_or_exhaustion_defined_only"
                  }
                }
              }
            ]
          },
          {
            "name": "best_capture_approach_gate",
            "role": "gate",
            "field": "best_capture_approach",
            "kind": "value",
            "expected": "heterotypic_cell_conjugate"
          },
          {
            "name": "best_selection_marker_gate",
            "role": "gate",
            "field": "best_selection_marker",
            "kind": "value",
            "expected": "none"
          },
          {
            "name": "reactivity_criterion_gate",
            "role": "gate",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "clonal_expansion_and_physical_engagement"
          },
          {
            "name": "cross_patient_robustness_gate",
            "role": "gate",
            "field": "cross_patient_robustness",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_across_patients",
              "marginal_in_some_patients"
            ]
          },
          {
            "name": "marker_selection_consequence_gate",
            "role": "gate",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "discards_clonally_expanded_reactive_cells"
          },
          {
            "name": "superiority_claim_basis_gate",
            "role": "gate",
            "field": "superiority_claim_basis",
            "kind": "value",
            "expected": "requires_functional_reactivity_assay"
          },
          {
            "name": "conjugate_enriched_state_gate",
            "role": "gate",
            "field": "conjugate_enriched_state",
            "kind": "value",
            "expected": "progenitor_exhausted_TCF7_pos_stem_like"
          },
          {
            "name": "cf_circular_exhaustion_criterion",
            "role": "hard_fail",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "exhaustion_signature_intensity"
          },
          {
            "name": "cf_marker_sort_prior",
            "role": "hard_fail",
            "field": "best_selection_marker",
            "kind": "value",
            "op": "in",
            "expected": [
              "CD39",
              "PD1",
              "PD1_high",
              "TIM3",
              "PD1_CD39",
              "CD39_CD103"
            ]
          },
          {
            "name": "cf_single_cell_capture_prior",
            "role": "hard_fail",
            "field": "best_capture_approach",
            "kind": "value",
            "op": "in",
            "expected": [
              "dissociated_single_cell",
              "dissociated_bulk_population"
            ]
          },
          {
            "name": "cf_off_data_capture_gate",
            "role": "hard_fail",
            "field": "$.population_findings[*].capture_gate",
            "kind": "avoid",
            "expected": [
              "dissociated_bulk",
              "matched_peripheral_blood",
              "matched_lymph_node",
              "post_hoc_expanded_clonotype_group"
            ]
          },
          {
            "name": "cf_conjugate_mischaracterized",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_marker_singlet_called_engaged",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "contains",
            "matchKey": "capture_gate",
            "entity": "marker_sorted_singlet",
            "valueField": "reactivity_evidence",
            "expected": [
              "engagement_and_clonal_expansion"
            ]
          },
          {
            "name": "cf_no_engaged_reactive_population",
            "role": "hard_fail",
            "field": "$.population_findings[*].reactivity_evidence",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_marker_captures_all_reactive",
            "role": "hard_fail",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "captures_full_reactive_repertoire"
          },
          {
            "name": "cf_superiority_overclaimed_from_phenotype",
            "role": "hard_fail",
            "field": "superiority_claim_basis",
            "kind": "value",
            "op": "in",
            "expected": [
              "established_by_scrna_phenotype",
              "established_by_reactivity_proxy_scores"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-terra-pi",
      "modelName": "gpt-5.6-terra",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_vs_cd39_tumor_reactivity_dominance",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 401.557416,
      "cost": 1.0028391250000002,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "population_findings",
            "kind": "list",
            "matchKey": "capture_gate",
            "k": 4,
            "expected": [
              {
                "key": "tumor_cell_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "apc_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "combined_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "unselected_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "naive",
                      "central_memory",
                      "effector_memory"
                    ],
                    "canonical": "naive"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "low"
                    ],
                    "canonical": "low"
                  },
                  "purity_caveat": {
                    "op": "equals",
                    "values": [
                      "pure_T_cell"
                    ],
                    "canonical": "pure_T_cell"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "no_reactivity_evidence"
                    ],
                    "canonical": "no_reactivity_evidence"
                  }
                }
              },
              {
                "key": "marker_sorted_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "equals",
                    "values": [
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "terminal_exhausted_LAG3_high"
                  },
                  "clonal_expansion_level": {
                    "op": "in",
                    "values": [
                      "high",
                      "intermediate"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "marker_on_non_T_lineages",
                      "single_cell_gating_loses_target"
                    ],
                    "canonical": "marker_on_non_T_lineages"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "marker_or_exhaustion_defined_only"
                    ],
                    "canonical": "marker_or_exhaustion_defined_only"
                  }
                }
              }
            ]
          },
          {
            "name": "best_capture_approach_gate",
            "role": "gate",
            "field": "best_capture_approach",
            "kind": "value",
            "expected": "heterotypic_cell_conjugate"
          },
          {
            "name": "best_selection_marker_gate",
            "role": "gate",
            "field": "best_selection_marker",
            "kind": "value",
            "expected": "none"
          },
          {
            "name": "reactivity_criterion_gate",
            "role": "gate",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "clonal_expansion_and_physical_engagement"
          },
          {
            "name": "cross_patient_robustness_gate",
            "role": "gate",
            "field": "cross_patient_robustness",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_across_patients",
              "marginal_in_some_patients"
            ]
          },
          {
            "name": "marker_selection_consequence_gate",
            "role": "gate",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "discards_clonally_expanded_reactive_cells"
          },
          {
            "name": "superiority_claim_basis_gate",
            "role": "gate",
            "field": "superiority_claim_basis",
            "kind": "value",
            "expected": "requires_functional_reactivity_assay"
          },
          {
            "name": "conjugate_enriched_state_gate",
            "role": "gate",
            "field": "conjugate_enriched_state",
            "kind": "value",
            "expected": "progenitor_exhausted_TCF7_pos_stem_like"
          },
          {
            "name": "cf_circular_exhaustion_criterion",
            "role": "hard_fail",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "exhaustion_signature_intensity"
          },
          {
            "name": "cf_marker_sort_prior",
            "role": "hard_fail",
            "field": "best_selection_marker",
            "kind": "value",
            "op": "in",
            "expected": [
              "CD39",
              "PD1",
              "PD1_high",
              "TIM3",
              "PD1_CD39",
              "CD39_CD103"
            ]
          },
          {
            "name": "cf_single_cell_capture_prior",
            "role": "hard_fail",
            "field": "best_capture_approach",
            "kind": "value",
            "op": "in",
            "expected": [
              "dissociated_single_cell",
              "dissociated_bulk_population"
            ]
          },
          {
            "name": "cf_off_data_capture_gate",
            "role": "hard_fail",
            "field": "$.population_findings[*].capture_gate",
            "kind": "avoid",
            "expected": [
              "dissociated_bulk",
              "matched_peripheral_blood",
              "matched_lymph_node",
              "post_hoc_expanded_clonotype_group"
            ]
          },
          {
            "name": "cf_conjugate_mischaracterized",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_marker_singlet_called_engaged",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "contains",
            "matchKey": "capture_gate",
            "entity": "marker_sorted_singlet",
            "valueField": "reactivity_evidence",
            "expected": [
              "engagement_and_clonal_expansion"
            ]
          },
          {
            "name": "cf_no_engaged_reactive_population",
            "role": "hard_fail",
            "field": "$.population_findings[*].reactivity_evidence",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_marker_captures_all_reactive",
            "role": "hard_fail",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "captures_full_reactive_repertoire"
          },
          {
            "name": "cf_superiority_overclaimed_from_phenotype",
            "role": "hard_fail",
            "field": "superiority_claim_basis",
            "kind": "value",
            "op": "in",
            "expected": [
              "established_by_scrna_phenotype",
              "established_by_reactivity_proxy_scores"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-terra-pi",
      "modelName": "gpt-5.6-terra",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_vs_cd39_tumor_reactivity_dominance",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 370.507246,
      "cost": 0.9665489999999999,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "population_findings",
            "kind": "list",
            "matchKey": "capture_gate",
            "k": 4,
            "expected": [
              {
                "key": "tumor_cell_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "apc_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "combined_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "unselected_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "naive",
                      "central_memory",
                      "effector_memory"
                    ],
                    "canonical": "naive"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "low"
                    ],
                    "canonical": "low"
                  },
                  "purity_caveat": {
                    "op": "equals",
                    "values": [
                      "pure_T_cell"
                    ],
                    "canonical": "pure_T_cell"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "no_reactivity_evidence"
                    ],
                    "canonical": "no_reactivity_evidence"
                  }
                }
              },
              {
                "key": "marker_sorted_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "equals",
                    "values": [
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "terminal_exhausted_LAG3_high"
                  },
                  "clonal_expansion_level": {
                    "op": "in",
                    "values": [
                      "high",
                      "intermediate"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "marker_on_non_T_lineages",
                      "single_cell_gating_loses_target"
                    ],
                    "canonical": "marker_on_non_T_lineages"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "marker_or_exhaustion_defined_only"
                    ],
                    "canonical": "marker_or_exhaustion_defined_only"
                  }
                }
              }
            ]
          },
          {
            "name": "best_capture_approach_gate",
            "role": "gate",
            "field": "best_capture_approach",
            "kind": "value",
            "expected": "heterotypic_cell_conjugate"
          },
          {
            "name": "best_selection_marker_gate",
            "role": "gate",
            "field": "best_selection_marker",
            "kind": "value",
            "expected": "none"
          },
          {
            "name": "reactivity_criterion_gate",
            "role": "gate",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "clonal_expansion_and_physical_engagement"
          },
          {
            "name": "cross_patient_robustness_gate",
            "role": "gate",
            "field": "cross_patient_robustness",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_across_patients",
              "marginal_in_some_patients"
            ]
          },
          {
            "name": "marker_selection_consequence_gate",
            "role": "gate",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "discards_clonally_expanded_reactive_cells"
          },
          {
            "name": "superiority_claim_basis_gate",
            "role": "gate",
            "field": "superiority_claim_basis",
            "kind": "value",
            "expected": "requires_functional_reactivity_assay"
          },
          {
            "name": "conjugate_enriched_state_gate",
            "role": "gate",
            "field": "conjugate_enriched_state",
            "kind": "value",
            "expected": "progenitor_exhausted_TCF7_pos_stem_like"
          },
          {
            "name": "cf_circular_exhaustion_criterion",
            "role": "hard_fail",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "exhaustion_signature_intensity"
          },
          {
            "name": "cf_marker_sort_prior",
            "role": "hard_fail",
            "field": "best_selection_marker",
            "kind": "value",
            "op": "in",
            "expected": [
              "CD39",
              "PD1",
              "PD1_high",
              "TIM3",
              "PD1_CD39",
              "CD39_CD103"
            ]
          },
          {
            "name": "cf_single_cell_capture_prior",
            "role": "hard_fail",
            "field": "best_capture_approach",
            "kind": "value",
            "op": "in",
            "expected": [
              "dissociated_single_cell",
              "dissociated_bulk_population"
            ]
          },
          {
            "name": "cf_off_data_capture_gate",
            "role": "hard_fail",
            "field": "$.population_findings[*].capture_gate",
            "kind": "avoid",
            "expected": [
              "dissociated_bulk",
              "matched_peripheral_blood",
              "matched_lymph_node",
              "post_hoc_expanded_clonotype_group"
            ]
          },
          {
            "name": "cf_conjugate_mischaracterized",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_marker_singlet_called_engaged",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "contains",
            "matchKey": "capture_gate",
            "entity": "marker_sorted_singlet",
            "valueField": "reactivity_evidence",
            "expected": [
              "engagement_and_clonal_expansion"
            ]
          },
          {
            "name": "cf_no_engaged_reactive_population",
            "role": "hard_fail",
            "field": "$.population_findings[*].reactivity_evidence",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_marker_captures_all_reactive",
            "role": "hard_fail",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "captures_full_reactive_repertoire"
          },
          {
            "name": "cf_superiority_overclaimed_from_phenotype",
            "role": "hard_fail",
            "field": "superiority_claim_basis",
            "kind": "value",
            "op": "in",
            "expected": [
              "established_by_scrna_phenotype",
              "established_by_reactivity_proxy_scores"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-terra-pi",
      "modelName": "gpt-5.6-terra",
      "harness": "pi",
      "task": "10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 902.087042,
      "cost": 1.069236125,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_singlet_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "TOXhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "LAG3high_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "GZMKhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "intermediate_exhausted_gzmk"
                    ],
                    "canonical": "intermediate_exhausted_gzmk"
                  }
                }
              },
              {
                "key": "TCF7_stem_like_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "stem_like_exhausted"
                    ],
                    "canonical": "stem_like_exhausted"
                  }
                }
              },
              {
                "key": "MKI67_pos_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              },
              {
                "key": "MKI67high_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_tumor_cluster_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "Tn",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "Tn_Tmem",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "early_Tem_GZMK",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              },
              {
                "key": "Tem_GZMH",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              }
            ]
          },
          {
            "name": "clonotype_expansion_direction_data_true",
            "role": "gate",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_substantially_more_expanded_than_cluster",
              "singlet_modestly_more_expanded_than_cluster"
            ]
          },
          {
            "name": "biological_inference_data_true",
            "role": "gate",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_antigen_experienced_cluster_bystander"
            ]
          },
          {
            "name": "singlet_top3_contains_naive_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Tn",
              "Tn_Tmem",
              "early_Tem_GZMK",
              "Tem_GZMH"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "TOXhigh_Tex",
              "LAG3high_Tex",
              "GZMKhigh_Tex",
              "TCF7_stem_like_Tex",
              "MKI67_pos_Tex_Tprol",
              "MKI67high_Tex_Tprol"
            ]
          },
          {
            "name": "singlet_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "TOXhigh_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "TCF7_stem_like_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "MKI67_state_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "Tn_or_Tn_Tmem_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_stemlike_precursor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_reported_canonical_expansion_direction",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_substantially_more_expanded_than_singlet",
              "cluster_modestly_more_expanded_than_singlet"
            ]
          },
          {
            "name": "cf_expansion_lazy_defer",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "expansion_pattern_not_detectable"
            ]
          },
          {
            "name": "cf_reported_canonical_expected_biology",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_antigen_experienced_singlet_bystander"
            ]
          },
          {
            "name": "cf_biological_inference_null_artifact_or_lineage",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "differences_reflect_dissociation_or_sorting_artifact",
              "cluster_and_singlet_capture_same_biology",
              "cluster_and_singlet_capture_distinct_cd8_lineages"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "biological_relations",
            "kind": "dict",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_greater_than",
                  "greater_than"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "less_than",
                  "much_less_than"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than",
                  "approximately_equal"
                ]
              },
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal"
                ]
              },
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal",
                  "greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_exhaustion_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_naive_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_proliferation_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_expansion_richness_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_multichain_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_intermediate_gzmk_overgeneralized",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_melanocytic_conjugate_overcall",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "values": [
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_relations_mass_hedge",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-terra-pi",
      "modelName": "gpt-5.6-terra",
      "harness": "pi",
      "task": "10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 1475.97265,
      "cost": 0.906843625,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_singlet_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "TOXhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "LAG3high_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "GZMKhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "intermediate_exhausted_gzmk"
                    ],
                    "canonical": "intermediate_exhausted_gzmk"
                  }
                }
              },
              {
                "key": "TCF7_stem_like_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "stem_like_exhausted"
                    ],
                    "canonical": "stem_like_exhausted"
                  }
                }
              },
              {
                "key": "MKI67_pos_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              },
              {
                "key": "MKI67high_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_tumor_cluster_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "Tn",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "Tn_Tmem",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "early_Tem_GZMK",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              },
              {
                "key": "Tem_GZMH",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              }
            ]
          },
          {
            "name": "clonotype_expansion_direction_data_true",
            "role": "gate",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_substantially_more_expanded_than_cluster",
              "singlet_modestly_more_expanded_than_cluster"
            ]
          },
          {
            "name": "biological_inference_data_true",
            "role": "gate",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_antigen_experienced_cluster_bystander"
            ]
          },
          {
            "name": "singlet_top3_contains_naive_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Tn",
              "Tn_Tmem",
              "early_Tem_GZMK",
              "Tem_GZMH"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "TOXhigh_Tex",
              "LAG3high_Tex",
              "GZMKhigh_Tex",
              "TCF7_stem_like_Tex",
              "MKI67_pos_Tex_Tprol",
              "MKI67high_Tex_Tprol"
            ]
          },
          {
            "name": "singlet_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "TOXhigh_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "TCF7_stem_like_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "MKI67_state_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "Tn_or_Tn_Tmem_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_stemlike_precursor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_reported_canonical_expansion_direction",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_substantially_more_expanded_than_singlet",
              "cluster_modestly_more_expanded_than_singlet"
            ]
          },
          {
            "name": "cf_expansion_lazy_defer",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "expansion_pattern_not_detectable"
            ]
          },
          {
            "name": "cf_reported_canonical_expected_biology",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_antigen_experienced_singlet_bystander"
            ]
          },
          {
            "name": "cf_biological_inference_null_artifact_or_lineage",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "differences_reflect_dissociation_or_sorting_artifact",
              "cluster_and_singlet_capture_same_biology",
              "cluster_and_singlet_capture_distinct_cd8_lineages"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "biological_relations",
            "kind": "dict",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_greater_than",
                  "greater_than"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "less_than",
                  "much_less_than"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than",
                  "approximately_equal"
                ]
              },
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal"
                ]
              },
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal",
                  "greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_exhaustion_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_naive_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_proliferation_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_expansion_richness_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_multichain_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_intermediate_gzmk_overgeneralized",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_melanocytic_conjugate_overcall",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "values": [
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_relations_mass_hedge",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-terra-pi",
      "modelName": "gpt-5.6-terra",
      "harness": "pi",
      "task": "10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1224.541359,
      "cost": 0.8304419999999999,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_singlet_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "TOXhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "LAG3high_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "GZMKhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "intermediate_exhausted_gzmk"
                    ],
                    "canonical": "intermediate_exhausted_gzmk"
                  }
                }
              },
              {
                "key": "TCF7_stem_like_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "stem_like_exhausted"
                    ],
                    "canonical": "stem_like_exhausted"
                  }
                }
              },
              {
                "key": "MKI67_pos_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              },
              {
                "key": "MKI67high_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_tumor_cluster_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "Tn",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "Tn_Tmem",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "early_Tem_GZMK",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              },
              {
                "key": "Tem_GZMH",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              }
            ]
          },
          {
            "name": "clonotype_expansion_direction_data_true",
            "role": "gate",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_substantially_more_expanded_than_cluster",
              "singlet_modestly_more_expanded_than_cluster"
            ]
          },
          {
            "name": "biological_inference_data_true",
            "role": "gate",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_antigen_experienced_cluster_bystander"
            ]
          },
          {
            "name": "singlet_top3_contains_naive_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Tn",
              "Tn_Tmem",
              "early_Tem_GZMK",
              "Tem_GZMH"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "TOXhigh_Tex",
              "LAG3high_Tex",
              "GZMKhigh_Tex",
              "TCF7_stem_like_Tex",
              "MKI67_pos_Tex_Tprol",
              "MKI67high_Tex_Tprol"
            ]
          },
          {
            "name": "singlet_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "TOXhigh_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "TCF7_stem_like_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "MKI67_state_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "Tn_or_Tn_Tmem_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_stemlike_precursor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_reported_canonical_expansion_direction",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_substantially_more_expanded_than_singlet",
              "cluster_modestly_more_expanded_than_singlet"
            ]
          },
          {
            "name": "cf_expansion_lazy_defer",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "expansion_pattern_not_detectable"
            ]
          },
          {
            "name": "cf_reported_canonical_expected_biology",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_antigen_experienced_singlet_bystander"
            ]
          },
          {
            "name": "cf_biological_inference_null_artifact_or_lineage",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "differences_reflect_dissociation_or_sorting_artifact",
              "cluster_and_singlet_capture_same_biology",
              "cluster_and_singlet_capture_distinct_cd8_lineages"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "biological_relations",
            "kind": "dict",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_greater_than",
                  "greater_than"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "less_than",
                  "much_less_than"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than",
                  "approximately_equal"
                ]
              },
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal"
                ]
              },
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal",
                  "greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_exhaustion_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_naive_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_proliferation_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_expansion_richness_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_multichain_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_intermediate_gzmk_overgeneralized",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_melanocytic_conjugate_overcall",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "values": [
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_relations_mass_hedge",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-terra-pi",
      "modelName": "gpt-5.6-terra",
      "harness": "pi",
      "task": "10x_5p_melanoma_tcr_matched_apc_vs_tumor_exhaustion",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 1,
      "passed": true,
      "score": null,
      "durationS": 357.876436,
      "cost": 0.9765810000000001,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "state_discriminating_genes",
            "kind": "list",
            "matchKey": "gene",
            "k": 3,
            "expected": [
              {
                "key": "CTLA4",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TOX",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "ENTPD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "HAVCR2",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TIGIT",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "PDCD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "difference_attributable_to",
            "role": "gate",
            "field": "difference_attributable_to",
            "kind": "value",
            "expected": "covaries_with_apc_conjugate_state"
          },
          {
            "name": "causal_status",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "direction_unresolved_selection_confound"
          },
          {
            "name": "functional_interpretation",
            "role": "gate",
            "field": "functional_interpretation",
            "kind": "value",
            "expected": "costimulation_signature_function_unresolved"
          },
          {
            "name": "cf_contamination_gene_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "MLANA",
              "PMEL",
              "DCT",
              "TYR",
              "TYRP1",
              "SOX10",
              "MITF",
              "MAGEA4"
            ]
          },
          {
            "name": "cf_apc_partner_contamination_gene",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "HLA-DRA",
              "CD68",
              "LYZ",
              "CD14",
              "C1QA",
              "ITGAX",
              "CD1C"
            ]
          },
          {
            "name": "cf_ligand_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "CD274",
              "PDCD1LG2",
              "CD80",
              "CD86"
            ]
          },
          {
            "name": "cf_attribution_causal_or_baseline",
            "role": "hard_fail",
            "field": "$.difference_attributable_to",
            "kind": "avoid",
            "expected": [
              "apc_engagement",
              "tumor_cell_engagement",
              "clonotype_intrinsic_baseline"
            ]
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "engagement_induces_exhausted_state",
              "pre_exhausted_cells_select_into_apc_conjugates"
            ]
          },
          {
            "name": "cf_causal_fabrication",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "resolved_by_pseudotemporal_ordering",
              "resolved_by_perturbation_experiment"
            ]
          },
          {
            "name": "cf_functional_overclaim",
            "role": "hard_fail",
            "field": "$.functional_interpretation",
            "kind": "avoid",
            "expected": [
              "costimulation_licensed_reinvigoration_competent",
              "costimulation_licensed_checkpoint_responsive",
              "tumor_driven_terminal_exhaustion"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-terra-pi",
      "modelName": "gpt-5.6-terra",
      "harness": "pi",
      "task": "10x_5p_melanoma_tcr_matched_apc_vs_tumor_exhaustion",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 2,
      "passed": true,
      "score": null,
      "durationS": 261.352442,
      "cost": 0.6585192499999999,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "state_discriminating_genes",
            "kind": "list",
            "matchKey": "gene",
            "k": 3,
            "expected": [
              {
                "key": "CTLA4",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TOX",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "ENTPD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "HAVCR2",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TIGIT",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "PDCD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "difference_attributable_to",
            "role": "gate",
            "field": "difference_attributable_to",
            "kind": "value",
            "expected": "covaries_with_apc_conjugate_state"
          },
          {
            "name": "causal_status",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "direction_unresolved_selection_confound"
          },
          {
            "name": "functional_interpretation",
            "role": "gate",
            "field": "functional_interpretation",
            "kind": "value",
            "expected": "costimulation_signature_function_unresolved"
          },
          {
            "name": "cf_contamination_gene_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "MLANA",
              "PMEL",
              "DCT",
              "TYR",
              "TYRP1",
              "SOX10",
              "MITF",
              "MAGEA4"
            ]
          },
          {
            "name": "cf_apc_partner_contamination_gene",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "HLA-DRA",
              "CD68",
              "LYZ",
              "CD14",
              "C1QA",
              "ITGAX",
              "CD1C"
            ]
          },
          {
            "name": "cf_ligand_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "CD274",
              "PDCD1LG2",
              "CD80",
              "CD86"
            ]
          },
          {
            "name": "cf_attribution_causal_or_baseline",
            "role": "hard_fail",
            "field": "$.difference_attributable_to",
            "kind": "avoid",
            "expected": [
              "apc_engagement",
              "tumor_cell_engagement",
              "clonotype_intrinsic_baseline"
            ]
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "engagement_induces_exhausted_state",
              "pre_exhausted_cells_select_into_apc_conjugates"
            ]
          },
          {
            "name": "cf_causal_fabrication",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "resolved_by_pseudotemporal_ordering",
              "resolved_by_perturbation_experiment"
            ]
          },
          {
            "name": "cf_functional_overclaim",
            "role": "hard_fail",
            "field": "$.functional_interpretation",
            "kind": "avoid",
            "expected": [
              "costimulation_licensed_reinvigoration_competent",
              "costimulation_licensed_checkpoint_responsive",
              "tumor_driven_terminal_exhaustion"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-terra-pi",
      "modelName": "gpt-5.6-terra",
      "harness": "pi",
      "task": "10x_5p_melanoma_tcr_matched_apc_vs_tumor_exhaustion",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": true,
      "score": null,
      "durationS": 529.832358,
      "cost": 0.60900625,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "state_discriminating_genes",
            "kind": "list",
            "matchKey": "gene",
            "k": 3,
            "expected": [
              {
                "key": "CTLA4",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TOX",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "ENTPD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "HAVCR2",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TIGIT",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "PDCD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "difference_attributable_to",
            "role": "gate",
            "field": "difference_attributable_to",
            "kind": "value",
            "expected": "covaries_with_apc_conjugate_state"
          },
          {
            "name": "causal_status",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "direction_unresolved_selection_confound"
          },
          {
            "name": "functional_interpretation",
            "role": "gate",
            "field": "functional_interpretation",
            "kind": "value",
            "expected": "costimulation_signature_function_unresolved"
          },
          {
            "name": "cf_contamination_gene_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "MLANA",
              "PMEL",
              "DCT",
              "TYR",
              "TYRP1",
              "SOX10",
              "MITF",
              "MAGEA4"
            ]
          },
          {
            "name": "cf_apc_partner_contamination_gene",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "HLA-DRA",
              "CD68",
              "LYZ",
              "CD14",
              "C1QA",
              "ITGAX",
              "CD1C"
            ]
          },
          {
            "name": "cf_ligand_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "CD274",
              "PDCD1LG2",
              "CD80",
              "CD86"
            ]
          },
          {
            "name": "cf_attribution_causal_or_baseline",
            "role": "hard_fail",
            "field": "$.difference_attributable_to",
            "kind": "avoid",
            "expected": [
              "apc_engagement",
              "tumor_cell_engagement",
              "clonotype_intrinsic_baseline"
            ]
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "engagement_induces_exhausted_state",
              "pre_exhausted_cells_select_into_apc_conjugates"
            ]
          },
          {
            "name": "cf_causal_fabrication",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "resolved_by_pseudotemporal_ordering",
              "resolved_by_perturbation_experiment"
            ]
          },
          {
            "name": "cf_functional_overclaim",
            "role": "hard_fail",
            "field": "$.functional_interpretation",
            "kind": "avoid",
            "expected": [
              "costimulation_licensed_reinvigoration_competent",
              "costimulation_licensed_checkpoint_responsive",
              "tumor_driven_terminal_exhaustion"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-terra-pi",
      "modelName": "gpt-5.6-terra",
      "harness": "pi",
      "task": "green_2_tme_context_dependence",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 1006.332464,
      "cost": 1.9441016249999998,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "program_map",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "sig_includes_itgae",
            "role": "gate",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Itgae"
            ]
          },
          {
            "name": "declared_a_known_confounder",
            "role": "gate",
            "field": "$.excluded_confounders[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "cf_signature_gut_contaminant",
            "role": "hard_fail",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "generic_signature_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd103_program_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "<"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "per_model_regulatory_state",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "poised_unfired"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "late_activation"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "closing_committed"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_tuples",
            "kind": "list",
            "matchKey": "regulator",
            "k": 1,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "role": {
                    "op": "equals",
                    "values": [
                      "repressor"
                    ]
                  },
                  "gating_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  },
                  "evidence_basis": {
                    "op": "equals",
                    "values": [
                      "target_program_behavior"
                    ]
                  },
                  "expr_vs_activity": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "locus_differential_gate",
            "role": "gate",
            "field": "locus_differential",
            "kind": "value",
            "expected": "uniform_across_models"
          },
          {
            "name": "formation_limited_by_gate",
            "role": "gate",
            "field": "formation_limited_by",
            "kind": "value",
            "expected": "absent_induction"
          },
          {
            "name": "locus_competent_in_naive_gate",
            "role": "gate",
            "field": "locus_competent_in_naive",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "ready_silent_fraction_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 3,
            "expected": [
              {
                "key": "B16_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_KPC_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "activator_chromatin_footprint_gate",
            "role": "gate",
            "field": "activator_chromatin_footprint",
            "kind": "value",
            "expected": "inconsistent_across_formers"
          },
          {
            "name": "exhaustion_comparable_gate",
            "role": "gate",
            "field": "exhaustion_comparable_across_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_nondeterminant_gate",
            "role": "gate",
            "field": "exhaustion_explains_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_separable_program_gate",
            "role": "gate",
            "field": "exhaustion_separable_program",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "exhaustion_counterexample_gate",
            "role": "gate",
            "field": "exhaustion_counterexample_model",
            "kind": "value",
            "expected": "KPC_GP"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "exhaustion_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "cf_inconsistent_present_without_activator",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "program",
            "entity": "present",
            "valueField": "activator_state",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_no_block_but_absent",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "none",
            "valueField": "program",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_late_block_without_poised_locus",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "late_activation",
            "valueField": "locus_state",
            "expected": [
              "closing_committed",
              "active"
            ]
          },
          {
            "name": "cf_inconsistent_early_block_with_brake_released",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "early_release",
            "valueField": "brake_state",
            "expected": [
              "released"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-terra-pi",
      "modelName": "gpt-5.6-terra",
      "harness": "pi",
      "task": "green_2_tme_context_dependence",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 973.689171,
      "cost": 3.240392,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "program_map",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "sig_includes_itgae",
            "role": "gate",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Itgae"
            ]
          },
          {
            "name": "declared_a_known_confounder",
            "role": "gate",
            "field": "$.excluded_confounders[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "cf_signature_gut_contaminant",
            "role": "hard_fail",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "generic_signature_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd103_program_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "<"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "per_model_regulatory_state",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "poised_unfired"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "late_activation"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "closing_committed"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_tuples",
            "kind": "list",
            "matchKey": "regulator",
            "k": 1,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "role": {
                    "op": "equals",
                    "values": [
                      "repressor"
                    ]
                  },
                  "gating_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  },
                  "evidence_basis": {
                    "op": "equals",
                    "values": [
                      "target_program_behavior"
                    ]
                  },
                  "expr_vs_activity": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "locus_differential_gate",
            "role": "gate",
            "field": "locus_differential",
            "kind": "value",
            "expected": "uniform_across_models"
          },
          {
            "name": "formation_limited_by_gate",
            "role": "gate",
            "field": "formation_limited_by",
            "kind": "value",
            "expected": "absent_induction"
          },
          {
            "name": "locus_competent_in_naive_gate",
            "role": "gate",
            "field": "locus_competent_in_naive",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "ready_silent_fraction_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 3,
            "expected": [
              {
                "key": "B16_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_KPC_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "activator_chromatin_footprint_gate",
            "role": "gate",
            "field": "activator_chromatin_footprint",
            "kind": "value",
            "expected": "inconsistent_across_formers"
          },
          {
            "name": "exhaustion_comparable_gate",
            "role": "gate",
            "field": "exhaustion_comparable_across_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_nondeterminant_gate",
            "role": "gate",
            "field": "exhaustion_explains_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_separable_program_gate",
            "role": "gate",
            "field": "exhaustion_separable_program",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "exhaustion_counterexample_gate",
            "role": "gate",
            "field": "exhaustion_counterexample_model",
            "kind": "value",
            "expected": "KPC_GP"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "exhaustion_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "cf_inconsistent_present_without_activator",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "program",
            "entity": "present",
            "valueField": "activator_state",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_no_block_but_absent",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "none",
            "valueField": "program",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_late_block_without_poised_locus",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "late_activation",
            "valueField": "locus_state",
            "expected": [
              "closing_committed",
              "active"
            ]
          },
          {
            "name": "cf_inconsistent_early_block_with_brake_released",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "early_release",
            "valueField": "brake_state",
            "expected": [
              "released"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-terra-pi",
      "modelName": "gpt-5.6-terra",
      "harness": "pi",
      "task": "green_2_tme_context_dependence",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1413.674779,
      "cost": 3.22698375,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "program_map",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "sig_includes_itgae",
            "role": "gate",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Itgae"
            ]
          },
          {
            "name": "declared_a_known_confounder",
            "role": "gate",
            "field": "$.excluded_confounders[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "cf_signature_gut_contaminant",
            "role": "hard_fail",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "generic_signature_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd103_program_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "<"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "per_model_regulatory_state",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "poised_unfired"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "late_activation"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "closing_committed"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_tuples",
            "kind": "list",
            "matchKey": "regulator",
            "k": 1,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "role": {
                    "op": "equals",
                    "values": [
                      "repressor"
                    ]
                  },
                  "gating_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  },
                  "evidence_basis": {
                    "op": "equals",
                    "values": [
                      "target_program_behavior"
                    ]
                  },
                  "expr_vs_activity": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "locus_differential_gate",
            "role": "gate",
            "field": "locus_differential",
            "kind": "value",
            "expected": "uniform_across_models"
          },
          {
            "name": "formation_limited_by_gate",
            "role": "gate",
            "field": "formation_limited_by",
            "kind": "value",
            "expected": "absent_induction"
          },
          {
            "name": "locus_competent_in_naive_gate",
            "role": "gate",
            "field": "locus_competent_in_naive",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "ready_silent_fraction_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 3,
            "expected": [
              {
                "key": "B16_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_KPC_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "activator_chromatin_footprint_gate",
            "role": "gate",
            "field": "activator_chromatin_footprint",
            "kind": "value",
            "expected": "inconsistent_across_formers"
          },
          {
            "name": "exhaustion_comparable_gate",
            "role": "gate",
            "field": "exhaustion_comparable_across_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_nondeterminant_gate",
            "role": "gate",
            "field": "exhaustion_explains_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_separable_program_gate",
            "role": "gate",
            "field": "exhaustion_separable_program",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "exhaustion_counterexample_gate",
            "role": "gate",
            "field": "exhaustion_counterexample_model",
            "kind": "value",
            "expected": "KPC_GP"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "exhaustion_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "cf_inconsistent_present_without_activator",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "program",
            "entity": "present",
            "valueField": "activator_state",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_no_block_but_absent",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "none",
            "valueField": "program",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_late_block_without_poised_locus",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "late_activation",
            "valueField": "locus_state",
            "expected": [
              "closing_committed",
              "active"
            ]
          },
          {
            "name": "cf_inconsistent_early_block_with_brake_released",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "early_release",
            "valueField": "brake_state",
            "expected": [
              "released"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-terra-pi",
      "modelName": "gpt-5.6-terra",
      "harness": "pi",
      "task": "green_klf2_hidden_repressor_of_trm_like_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 1897.430808,
      "cost": 2.1070667499999995,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "regulator_gate",
            "role": "gate",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "KLF2",
              "Klf2"
            ]
          },
          {
            "name": "controlling_node_character_gate",
            "role": "gate",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "high_differential_expression"
          },
          {
            "name": "expression_vs_activity_gate",
            "role": "gate",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "expression_exceeds_activity"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "target_program_partition",
            "kind": "list",
            "matchKey": "gene",
            "k": 7,
            "expected": [
              {
                "key": "S1pr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "S1pr4",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Klrg1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Cx3cr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Itgae",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Hic1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "P2rx7",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              }
            ]
          },
          {
            "name": "causal_status_gate",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "association_only_sufficiency_requires_perturbation"
          },
          {
            "name": "residency_control_logic_gate",
            "role": "gate",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_release_necessary_not_sufficient_multi_regulon"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "trm_like_program_by_model",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_absent"
                    ],
                    "canonical": "niche_absent"
                  }
                }
              }
            ]
          },
          {
            "name": "cf_regulator_activator_or_wrong_node",
            "role": "hard_fail",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "RUNX3",
              "Runx3",
              "PRDM1",
              "Prdm1",
              "BLIMP1",
              "Blimp1",
              "BHLHE40",
              "Bhlhe40",
              "ZNF683",
              "Zfp683",
              "HOBIT",
              "Hobit",
              "AHR",
              "Ahr",
              "BATF",
              "Batf",
              "TOX",
              "Tox",
              "NR4A1",
              "Nr4a1",
              "NR4A2",
              "Nr4a2",
              "NR4A3",
              "Nr4a3",
              "TCF7",
              "Tcf7",
              "TCF1",
              "Tcf1",
              "BACH2",
              "Bach2",
              "TBX21",
              "Tbx21",
              "TBET",
              "Tbet",
              "FLI1",
              "Fli1",
              "NR3C1",
              "Nr3c1",
              "EOMES",
              "Eomes",
              "ZEB2",
              "Zeb2",
              "ID2",
              "Id2",
              "ID3",
              "Id3",
              "IRF4",
              "Irf4"
            ]
          },
          {
            "name": "cf_node_character_hidden_driver_recall",
            "role": "hard_fail",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "hidden_driver_activity_not_expression"
          },
          {
            "name": "cf_effect_relation_hidden_pattern",
            "role": "hard_fail",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "activity_exceeds_expression"
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "causal_status",
            "kind": "value",
            "op": "in",
            "expected": [
              "sufficiency_established",
              "necessity_established",
              "causal_established_in_dataset",
              "sufficiency_via_perturbation_in_dataset"
            ]
          },
          {
            "name": "cf_residency_logic_activator_prior",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "residency_activator_induction_primary"
          },
          {
            "name": "cf_residency_logic_single_regulon",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_controls_entire_residency_program"
          },
          {
            "name": "cf_decoy_called_target",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_distractor_model_reported",
            "role": "hard_fail",
            "field": "$.trm_like_program_by_model[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_chromatin_only_decoy",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-terra-pi",
      "modelName": "gpt-5.6-terra",
      "harness": "pi",
      "task": "green_klf2_hidden_repressor_of_trm_like_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 582.955613,
      "cost": 1.154474125,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "regulator_gate",
            "role": "gate",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "KLF2",
              "Klf2"
            ]
          },
          {
            "name": "controlling_node_character_gate",
            "role": "gate",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "high_differential_expression"
          },
          {
            "name": "expression_vs_activity_gate",
            "role": "gate",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "expression_exceeds_activity"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "target_program_partition",
            "kind": "list",
            "matchKey": "gene",
            "k": 7,
            "expected": [
              {
                "key": "S1pr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "S1pr4",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Klrg1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Cx3cr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Itgae",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Hic1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "P2rx7",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              }
            ]
          },
          {
            "name": "causal_status_gate",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "association_only_sufficiency_requires_perturbation"
          },
          {
            "name": "residency_control_logic_gate",
            "role": "gate",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_release_necessary_not_sufficient_multi_regulon"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "trm_like_program_by_model",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_absent"
                    ],
                    "canonical": "niche_absent"
                  }
                }
              }
            ]
          },
          {
            "name": "cf_regulator_activator_or_wrong_node",
            "role": "hard_fail",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "RUNX3",
              "Runx3",
              "PRDM1",
              "Prdm1",
              "BLIMP1",
              "Blimp1",
              "BHLHE40",
              "Bhlhe40",
              "ZNF683",
              "Zfp683",
              "HOBIT",
              "Hobit",
              "AHR",
              "Ahr",
              "BATF",
              "Batf",
              "TOX",
              "Tox",
              "NR4A1",
              "Nr4a1",
              "NR4A2",
              "Nr4a2",
              "NR4A3",
              "Nr4a3",
              "TCF7",
              "Tcf7",
              "TCF1",
              "Tcf1",
              "BACH2",
              "Bach2",
              "TBX21",
              "Tbx21",
              "TBET",
              "Tbet",
              "FLI1",
              "Fli1",
              "NR3C1",
              "Nr3c1",
              "EOMES",
              "Eomes",
              "ZEB2",
              "Zeb2",
              "ID2",
              "Id2",
              "ID3",
              "Id3",
              "IRF4",
              "Irf4"
            ]
          },
          {
            "name": "cf_node_character_hidden_driver_recall",
            "role": "hard_fail",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "hidden_driver_activity_not_expression"
          },
          {
            "name": "cf_effect_relation_hidden_pattern",
            "role": "hard_fail",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "activity_exceeds_expression"
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "causal_status",
            "kind": "value",
            "op": "in",
            "expected": [
              "sufficiency_established",
              "necessity_established",
              "causal_established_in_dataset",
              "sufficiency_via_perturbation_in_dataset"
            ]
          },
          {
            "name": "cf_residency_logic_activator_prior",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "residency_activator_induction_primary"
          },
          {
            "name": "cf_residency_logic_single_regulon",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_controls_entire_residency_program"
          },
          {
            "name": "cf_decoy_called_target",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_distractor_model_reported",
            "role": "hard_fail",
            "field": "$.trm_like_program_by_model[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_chromatin_only_decoy",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-terra-pi",
      "modelName": "gpt-5.6-terra",
      "harness": "pi",
      "task": "green_klf2_hidden_repressor_of_trm_like_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 296.383474,
      "cost": 0.5884886249999999,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "regulator_gate",
            "role": "gate",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "KLF2",
              "Klf2"
            ]
          },
          {
            "name": "controlling_node_character_gate",
            "role": "gate",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "high_differential_expression"
          },
          {
            "name": "expression_vs_activity_gate",
            "role": "gate",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "expression_exceeds_activity"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "target_program_partition",
            "kind": "list",
            "matchKey": "gene",
            "k": 7,
            "expected": [
              {
                "key": "S1pr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "S1pr4",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Klrg1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Cx3cr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Itgae",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Hic1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "P2rx7",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              }
            ]
          },
          {
            "name": "causal_status_gate",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "association_only_sufficiency_requires_perturbation"
          },
          {
            "name": "residency_control_logic_gate",
            "role": "gate",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_release_necessary_not_sufficient_multi_regulon"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "trm_like_program_by_model",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_absent"
                    ],
                    "canonical": "niche_absent"
                  }
                }
              }
            ]
          },
          {
            "name": "cf_regulator_activator_or_wrong_node",
            "role": "hard_fail",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "RUNX3",
              "Runx3",
              "PRDM1",
              "Prdm1",
              "BLIMP1",
              "Blimp1",
              "BHLHE40",
              "Bhlhe40",
              "ZNF683",
              "Zfp683",
              "HOBIT",
              "Hobit",
              "AHR",
              "Ahr",
              "BATF",
              "Batf",
              "TOX",
              "Tox",
              "NR4A1",
              "Nr4a1",
              "NR4A2",
              "Nr4a2",
              "NR4A3",
              "Nr4a3",
              "TCF7",
              "Tcf7",
              "TCF1",
              "Tcf1",
              "BACH2",
              "Bach2",
              "TBX21",
              "Tbx21",
              "TBET",
              "Tbet",
              "FLI1",
              "Fli1",
              "NR3C1",
              "Nr3c1",
              "EOMES",
              "Eomes",
              "ZEB2",
              "Zeb2",
              "ID2",
              "Id2",
              "ID3",
              "Id3",
              "IRF4",
              "Irf4"
            ]
          },
          {
            "name": "cf_node_character_hidden_driver_recall",
            "role": "hard_fail",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "hidden_driver_activity_not_expression"
          },
          {
            "name": "cf_effect_relation_hidden_pattern",
            "role": "hard_fail",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "activity_exceeds_expression"
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "causal_status",
            "kind": "value",
            "op": "in",
            "expected": [
              "sufficiency_established",
              "necessity_established",
              "causal_established_in_dataset",
              "sufficiency_via_perturbation_in_dataset"
            ]
          },
          {
            "name": "cf_residency_logic_activator_prior",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "residency_activator_induction_primary"
          },
          {
            "name": "cf_residency_logic_single_regulon",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_controls_entire_residency_program"
          },
          {
            "name": "cf_decoy_called_target",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_distractor_model_reported",
            "role": "hard_fail",
            "field": "$.trm_like_program_by_model[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_chromatin_only_decoy",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-terra-pi",
      "modelName": "gpt-5.6-terra",
      "harness": "pi",
      "task": "green_shared_vs_distinct_regulators_trm_vs_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 1,
      "passed": true,
      "score": null,
      "durationS": 745.965313,
      "cost": 1.3229225,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_program_map",
            "kind": "list",
            "matchKey": "regulator",
            "k": 3,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "both"
                    ],
                    "canonical": "both"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "represses"
                    ],
                    "canonical": "represses"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Batf",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "trm_like_til_specific"
                    ],
                    "canonical": "trm_like_til_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "chromatin_over_rna"
                    ],
                    "canonical": "chromatin_over_rna"
                  }
                }
              },
              {
                "key": "Ahr",
                "fields": {
                  "program_specificity": {
                    "op": "in",
                    "values": [
                      "trm_specific",
                      "both"
                    ],
                    "canonical": "trm_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "rna_over_chromatin"
                    ],
                    "canonical": "rna_over_chromatin"
                  }
                }
              }
            ]
          },
          {
            "name": "regulator_set_no_extras_gate",
            "role": "gate",
            "field": "$.regulator_program_map[*]",
            "kind": "other",
            "expected": null
          },
          {
            "name": "program_identity_conclusion_gate",
            "role": "gate",
            "field": "program_identity_conclusion",
            "kind": "value",
            "expected": "distinct_state_shared_brake_divergent_activators"
          },
          {
            "name": "residency_install_logic_gate",
            "role": "gate",
            "field": "residency_install_logic",
            "kind": "value",
            "expected": "brake_release_permits_context_matched_activator"
          },
          {
            "name": "activator_context_dependence_gate",
            "role": "gate",
            "field": "activator_context_dependence",
            "kind": "value",
            "expected": "context_matched_crossover"
          },
          {
            "name": "evidence_ceiling_gate",
            "role": "gate",
            "field": "evidence_ceiling",
            "kind": "value",
            "expected": "specificity_dissociations_require_perturbation"
          },
          {
            "name": "cf_install_logic_flat_or_single_step",
            "role": "hard_fail",
            "field": "residency_install_logic",
            "kind": "value",
            "op": "in",
            "expected": [
              "flat_shared_residency_toolkit",
              "activator_induction_sufficient",
              "brake_release_alone_sufficient"
            ]
          },
          {
            "name": "cf_program_identity_lumped_or_wrong",
            "role": "hard_fail",
            "field": "program_identity_conclusion",
            "kind": "value",
            "op": "in",
            "expected": [
              "trm_like_til_is_tumor_trm",
              "trm_like_til_is_exhausted_subset",
              "trm_like_til_is_effector_state",
              "trm_like_til_is_transitional_intermediate",
              "trm_like_til_is_shared_program_distinct_niche"
            ]
          },
          {
            "name": "cf_ahr_assigned_to_til",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Ahr",
            "valueField": "program_specificity",
            "expected": [
              "trm_like_til_specific"
            ]
          },
          {
            "name": "cf_batf_assigned_to_trm_or_shared",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Batf",
            "valueField": "program_specificity",
            "expected": [
              "trm_specific",
              "both"
            ]
          },
          {
            "name": "cf_klf2_called_activator",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Klf2",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          },
          {
            "name": "cf_evidence_ceiling_overclaim",
            "role": "hard_fail",
            "field": "evidence_ceiling",
            "kind": "value",
            "op": "in",
            "expected": [
              "specificity_established_from_regulon_activity",
              "specificity_established_in_dataset",
              "causation_established_by_cross_sectional_concordance",
              "necessity_and_sufficiency_established"
            ]
          },
          {
            "name": "cf_activator_context_shared_or_single",
            "role": "hard_fail",
            "field": "activator_context_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "shared_activators_both_contexts",
              "single_global_activator",
              "redundant_activators"
            ]
          },
          {
            "name": "cf_single_modality_rule",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_hic1_motif_brake_mislabel",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_runx3_anticanonical_as_controller",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Runx3",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-terra-pi",
      "modelName": "gpt-5.6-terra",
      "harness": "pi",
      "task": "green_shared_vs_distinct_regulators_trm_vs_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 1972.390913,
      "cost": 0.9737152499999998,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_program_map",
            "kind": "list",
            "matchKey": "regulator",
            "k": 3,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "both"
                    ],
                    "canonical": "both"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "represses"
                    ],
                    "canonical": "represses"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Batf",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "trm_like_til_specific"
                    ],
                    "canonical": "trm_like_til_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "chromatin_over_rna"
                    ],
                    "canonical": "chromatin_over_rna"
                  }
                }
              },
              {
                "key": "Ahr",
                "fields": {
                  "program_specificity": {
                    "op": "in",
                    "values": [
                      "trm_specific",
                      "both"
                    ],
                    "canonical": "trm_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "rna_over_chromatin"
                    ],
                    "canonical": "rna_over_chromatin"
                  }
                }
              }
            ]
          },
          {
            "name": "regulator_set_no_extras_gate",
            "role": "gate",
            "field": "$.regulator_program_map[*]",
            "kind": "other",
            "expected": null
          },
          {
            "name": "program_identity_conclusion_gate",
            "role": "gate",
            "field": "program_identity_conclusion",
            "kind": "value",
            "expected": "distinct_state_shared_brake_divergent_activators"
          },
          {
            "name": "residency_install_logic_gate",
            "role": "gate",
            "field": "residency_install_logic",
            "kind": "value",
            "expected": "brake_release_permits_context_matched_activator"
          },
          {
            "name": "activator_context_dependence_gate",
            "role": "gate",
            "field": "activator_context_dependence",
            "kind": "value",
            "expected": "context_matched_crossover"
          },
          {
            "name": "evidence_ceiling_gate",
            "role": "gate",
            "field": "evidence_ceiling",
            "kind": "value",
            "expected": "specificity_dissociations_require_perturbation"
          },
          {
            "name": "cf_install_logic_flat_or_single_step",
            "role": "hard_fail",
            "field": "residency_install_logic",
            "kind": "value",
            "op": "in",
            "expected": [
              "flat_shared_residency_toolkit",
              "activator_induction_sufficient",
              "brake_release_alone_sufficient"
            ]
          },
          {
            "name": "cf_program_identity_lumped_or_wrong",
            "role": "hard_fail",
            "field": "program_identity_conclusion",
            "kind": "value",
            "op": "in",
            "expected": [
              "trm_like_til_is_tumor_trm",
              "trm_like_til_is_exhausted_subset",
              "trm_like_til_is_effector_state",
              "trm_like_til_is_transitional_intermediate",
              "trm_like_til_is_shared_program_distinct_niche"
            ]
          },
          {
            "name": "cf_ahr_assigned_to_til",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Ahr",
            "valueField": "program_specificity",
            "expected": [
              "trm_like_til_specific"
            ]
          },
          {
            "name": "cf_batf_assigned_to_trm_or_shared",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Batf",
            "valueField": "program_specificity",
            "expected": [
              "trm_specific",
              "both"
            ]
          },
          {
            "name": "cf_klf2_called_activator",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Klf2",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          },
          {
            "name": "cf_evidence_ceiling_overclaim",
            "role": "hard_fail",
            "field": "evidence_ceiling",
            "kind": "value",
            "op": "in",
            "expected": [
              "specificity_established_from_regulon_activity",
              "specificity_established_in_dataset",
              "causation_established_by_cross_sectional_concordance",
              "necessity_and_sufficiency_established"
            ]
          },
          {
            "name": "cf_activator_context_shared_or_single",
            "role": "hard_fail",
            "field": "activator_context_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "shared_activators_both_contexts",
              "single_global_activator",
              "redundant_activators"
            ]
          },
          {
            "name": "cf_single_modality_rule",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_hic1_motif_brake_mislabel",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_runx3_anticanonical_as_controller",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Runx3",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-terra-pi",
      "modelName": "gpt-5.6-terra",
      "harness": "pi",
      "task": "green_shared_vs_distinct_regulators_trm_vs_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1632.735761,
      "cost": 1.7790515000000002,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_program_map",
            "kind": "list",
            "matchKey": "regulator",
            "k": 3,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "both"
                    ],
                    "canonical": "both"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "represses"
                    ],
                    "canonical": "represses"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Batf",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "trm_like_til_specific"
                    ],
                    "canonical": "trm_like_til_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "chromatin_over_rna"
                    ],
                    "canonical": "chromatin_over_rna"
                  }
                }
              },
              {
                "key": "Ahr",
                "fields": {
                  "program_specificity": {
                    "op": "in",
                    "values": [
                      "trm_specific",
                      "both"
                    ],
                    "canonical": "trm_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "rna_over_chromatin"
                    ],
                    "canonical": "rna_over_chromatin"
                  }
                }
              }
            ]
          },
          {
            "name": "regulator_set_no_extras_gate",
            "role": "gate",
            "field": "$.regulator_program_map[*]",
            "kind": "other",
            "expected": null
          },
          {
            "name": "program_identity_conclusion_gate",
            "role": "gate",
            "field": "program_identity_conclusion",
            "kind": "value",
            "expected": "distinct_state_shared_brake_divergent_activators"
          },
          {
            "name": "residency_install_logic_gate",
            "role": "gate",
            "field": "residency_install_logic",
            "kind": "value",
            "expected": "brake_release_permits_context_matched_activator"
          },
          {
            "name": "activator_context_dependence_gate",
            "role": "gate",
            "field": "activator_context_dependence",
            "kind": "value",
            "expected": "context_matched_crossover"
          },
          {
            "name": "evidence_ceiling_gate",
            "role": "gate",
            "field": "evidence_ceiling",
            "kind": "value",
            "expected": "specificity_dissociations_require_perturbation"
          },
          {
            "name": "cf_install_logic_flat_or_single_step",
            "role": "hard_fail",
            "field": "residency_install_logic",
            "kind": "value",
            "op": "in",
            "expected": [
              "flat_shared_residency_toolkit",
              "activator_induction_sufficient",
              "brake_release_alone_sufficient"
            ]
          },
          {
            "name": "cf_program_identity_lumped_or_wrong",
            "role": "hard_fail",
            "field": "program_identity_conclusion",
            "kind": "value",
            "op": "in",
            "expected": [
              "trm_like_til_is_tumor_trm",
              "trm_like_til_is_exhausted_subset",
              "trm_like_til_is_effector_state",
              "trm_like_til_is_transitional_intermediate",
              "trm_like_til_is_shared_program_distinct_niche"
            ]
          },
          {
            "name": "cf_ahr_assigned_to_til",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Ahr",
            "valueField": "program_specificity",
            "expected": [
              "trm_like_til_specific"
            ]
          },
          {
            "name": "cf_batf_assigned_to_trm_or_shared",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Batf",
            "valueField": "program_specificity",
            "expected": [
              "trm_specific",
              "both"
            ]
          },
          {
            "name": "cf_klf2_called_activator",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Klf2",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          },
          {
            "name": "cf_evidence_ceiling_overclaim",
            "role": "hard_fail",
            "field": "evidence_ceiling",
            "kind": "value",
            "op": "in",
            "expected": [
              "specificity_established_from_regulon_activity",
              "specificity_established_in_dataset",
              "causation_established_by_cross_sectional_concordance",
              "necessity_and_sufficiency_established"
            ]
          },
          {
            "name": "cf_activator_context_shared_or_single",
            "role": "hard_fail",
            "field": "activator_context_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "shared_activators_both_contexts",
              "single_global_activator",
              "redundant_activators"
            ]
          },
          {
            "name": "cf_single_modality_rule",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_hic1_motif_brake_mislabel",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_runx3_anticanonical_as_controller",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Runx3",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-terra-pi",
      "modelName": "gpt-5.6-terra",
      "harness": "pi",
      "task": "gse297023_q2_senescence_not_mechanism",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 1,
      "passed": true,
      "score": null,
      "durationS": 1033.959777,
      "cost": 1.2990285,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "candidate_is_senescence",
            "role": "gate",
            "field": "$.candidate_mechanism",
            "kind": "value",
            "op": "in",
            "expected": [
              "cellular_senescence",
              "oncogene_induced_senescence",
              "replicative_senescence"
            ]
          },
          {
            "name": "arrest_core_not_engaged",
            "role": "gate",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_not_engaged"
            ]
          },
          {
            "name": "secretory_arm_is_inflammaging",
            "role": "gate",
            "field": "$.secretory_arm_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "acute_phase_inflammaging_response"
            ]
          },
          {
            "name": "secretory_arm_broadly_elevated",
            "role": "gate",
            "field": "$.secretory_arm_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "broadly_elevated"
            ]
          },
          {
            "name": "carries_aging_program",
            "role": "gate",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "carries_bona_fide_aging_program_distinct_from_nominated"
            ]
          },
          {
            "name": "conclusion_senescence_unlikely",
            "role": "gate",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "unlikely_major_contributor"
            ]
          },
          {
            "name": "cf_arrest_core_engaged_recall",
            "role": "hard_fail",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_engaged"
            ]
          },
          {
            "name": "cf_aging_program_conflation_or_explain_away",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "aging_program_indistinguishable_from_nominated_program",
              "aging_program_is_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_no_aging_program_sensitivity_failure",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_detectable_aging_program"
            ]
          },
          {
            "name": "cf_senescence_driver_conclusion",
            "role": "hard_fail",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "likely_primary_driver",
              "plausible_supported_contributor"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-terra-pi",
      "modelName": "gpt-5.6-terra",
      "harness": "pi",
      "task": "gse297023_q2_senescence_not_mechanism",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 1626.127917,
      "cost": 1.4015197499999998,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "candidate_is_senescence",
            "role": "gate",
            "field": "$.candidate_mechanism",
            "kind": "value",
            "op": "in",
            "expected": [
              "cellular_senescence",
              "oncogene_induced_senescence",
              "replicative_senescence"
            ]
          },
          {
            "name": "arrest_core_not_engaged",
            "role": "gate",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_not_engaged"
            ]
          },
          {
            "name": "secretory_arm_is_inflammaging",
            "role": "gate",
            "field": "$.secretory_arm_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "acute_phase_inflammaging_response"
            ]
          },
          {
            "name": "secretory_arm_broadly_elevated",
            "role": "gate",
            "field": "$.secretory_arm_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "broadly_elevated"
            ]
          },
          {
            "name": "carries_aging_program",
            "role": "gate",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "carries_bona_fide_aging_program_distinct_from_nominated"
            ]
          },
          {
            "name": "conclusion_senescence_unlikely",
            "role": "gate",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "unlikely_major_contributor"
            ]
          },
          {
            "name": "cf_arrest_core_engaged_recall",
            "role": "hard_fail",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_engaged"
            ]
          },
          {
            "name": "cf_aging_program_conflation_or_explain_away",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "aging_program_indistinguishable_from_nominated_program",
              "aging_program_is_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_no_aging_program_sensitivity_failure",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_detectable_aging_program"
            ]
          },
          {
            "name": "cf_senescence_driver_conclusion",
            "role": "hard_fail",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "likely_primary_driver",
              "plausible_supported_contributor"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-terra-pi",
      "modelName": "gpt-5.6-terra",
      "harness": "pi",
      "task": "gse297023_q2_senescence_not_mechanism",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 2634.586065,
      "cost": 1.060651125,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "candidate_is_senescence",
            "role": "gate",
            "field": "$.candidate_mechanism",
            "kind": "value",
            "op": "in",
            "expected": [
              "cellular_senescence",
              "oncogene_induced_senescence",
              "replicative_senescence"
            ]
          },
          {
            "name": "arrest_core_not_engaged",
            "role": "gate",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_not_engaged"
            ]
          },
          {
            "name": "secretory_arm_is_inflammaging",
            "role": "gate",
            "field": "$.secretory_arm_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "acute_phase_inflammaging_response"
            ]
          },
          {
            "name": "secretory_arm_broadly_elevated",
            "role": "gate",
            "field": "$.secretory_arm_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "broadly_elevated"
            ]
          },
          {
            "name": "carries_aging_program",
            "role": "gate",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "carries_bona_fide_aging_program_distinct_from_nominated"
            ]
          },
          {
            "name": "conclusion_senescence_unlikely",
            "role": "gate",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "unlikely_major_contributor"
            ]
          },
          {
            "name": "cf_arrest_core_engaged_recall",
            "role": "hard_fail",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_engaged"
            ]
          },
          {
            "name": "cf_aging_program_conflation_or_explain_away",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "aging_program_indistinguishable_from_nominated_program",
              "aging_program_is_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_no_aging_program_sensitivity_failure",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_detectable_aging_program"
            ]
          },
          {
            "name": "cf_senescence_driver_conclusion",
            "role": "hard_fail",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "likely_primary_driver",
              "plausible_supported_contributor"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-terra-pi",
      "modelName": "gpt-5.6-terra",
      "harness": "pi",
      "task": "gse297023_q3_age_retained_mapk_reduced",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 3550.497811,
      "cost": 2.2108291250000005,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "age_retained",
            "role": "gate",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_retained"
            ]
          },
          {
            "name": "identity_is_aging",
            "role": "gate",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_aging_signatures"
            ]
          },
          {
            "name": "nature_state_shift",
            "role": "gate",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_state_expression_shift"
            ]
          },
          {
            "name": "pathway_is_mapk",
            "role": "gate",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "mapk"
            ]
          },
          {
            "name": "pathway_reduced_in_aged",
            "role": "gate",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "reduced_in_aged"
            ]
          },
          {
            "name": "cf_mapk_increased_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_in_aged"
            ]
          },
          {
            "name": "cf_age_erased",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_erased"
            ]
          },
          {
            "name": "cf_pervasive_overcall",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_pervasive_artifactual"
            ]
          },
          {
            "name": "cf_explain_away_aging",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_compositional_or_identity",
            "role": "hard_fail",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_proportion_shift",
              "cell_identity_shift",
              "composition_dominant_state_secondary",
              "both_state_and_composition_equally"
            ]
          },
          {
            "name": "cf_biggest_inflammatory_signal_not_mapk",
            "role": "hard_fail",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "jak_stat",
              "nfkb",
              "tnfa"
            ]
          },
          {
            "name": "cf_senescence_misattribution",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_senescence_signature_specifically"
            ]
          },
          {
            "name": "cf_dismiss_as_batch_artifact",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_batch_or_technical_artifact"
            ]
          },
          {
            "name": "cf_mapk_comparable_constitutive_kras_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "comparable_across_ages"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-terra-pi",
      "modelName": "gpt-5.6-terra",
      "harness": "pi",
      "task": "gse297023_q3_age_retained_mapk_reduced",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 1854.678992,
      "cost": 1.6397368749999992,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "age_retained",
            "role": "gate",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_retained"
            ]
          },
          {
            "name": "identity_is_aging",
            "role": "gate",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_aging_signatures"
            ]
          },
          {
            "name": "nature_state_shift",
            "role": "gate",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_state_expression_shift"
            ]
          },
          {
            "name": "pathway_is_mapk",
            "role": "gate",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "mapk"
            ]
          },
          {
            "name": "pathway_reduced_in_aged",
            "role": "gate",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "reduced_in_aged"
            ]
          },
          {
            "name": "cf_mapk_increased_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_in_aged"
            ]
          },
          {
            "name": "cf_age_erased",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_erased"
            ]
          },
          {
            "name": "cf_pervasive_overcall",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_pervasive_artifactual"
            ]
          },
          {
            "name": "cf_explain_away_aging",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_compositional_or_identity",
            "role": "hard_fail",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_proportion_shift",
              "cell_identity_shift",
              "composition_dominant_state_secondary",
              "both_state_and_composition_equally"
            ]
          },
          {
            "name": "cf_biggest_inflammatory_signal_not_mapk",
            "role": "hard_fail",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "jak_stat",
              "nfkb",
              "tnfa"
            ]
          },
          {
            "name": "cf_senescence_misattribution",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_senescence_signature_specifically"
            ]
          },
          {
            "name": "cf_dismiss_as_batch_artifact",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_batch_or_technical_artifact"
            ]
          },
          {
            "name": "cf_mapk_comparable_constitutive_kras_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "comparable_across_ages"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-terra-pi",
      "modelName": "gpt-5.6-terra",
      "harness": "pi",
      "task": "gse297023_q3_age_retained_mapk_reduced",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1650.24956,
      "cost": 1.069840875,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "age_retained",
            "role": "gate",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_retained"
            ]
          },
          {
            "name": "identity_is_aging",
            "role": "gate",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_aging_signatures"
            ]
          },
          {
            "name": "nature_state_shift",
            "role": "gate",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_state_expression_shift"
            ]
          },
          {
            "name": "pathway_is_mapk",
            "role": "gate",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "mapk"
            ]
          },
          {
            "name": "pathway_reduced_in_aged",
            "role": "gate",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "reduced_in_aged"
            ]
          },
          {
            "name": "cf_mapk_increased_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_in_aged"
            ]
          },
          {
            "name": "cf_age_erased",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_erased"
            ]
          },
          {
            "name": "cf_pervasive_overcall",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_pervasive_artifactual"
            ]
          },
          {
            "name": "cf_explain_away_aging",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_compositional_or_identity",
            "role": "hard_fail",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_proportion_shift",
              "cell_identity_shift",
              "composition_dominant_state_secondary",
              "both_state_and_composition_equally"
            ]
          },
          {
            "name": "cf_biggest_inflammatory_signal_not_mapk",
            "role": "hard_fail",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "jak_stat",
              "nfkb",
              "tnfa"
            ]
          },
          {
            "name": "cf_senescence_misattribution",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_senescence_signature_specifically"
            ]
          },
          {
            "name": "cf_dismiss_as_batch_artifact",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_batch_or_technical_artifact"
            ]
          },
          {
            "name": "cf_mapk_comparable_constitutive_kras_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "comparable_across_ages"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-terra-pi",
      "modelName": "gpt-5.6-terra",
      "harness": "pi",
      "task": "gse297023_q5_tme_age_dependent_remodeling",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 1,
      "passed": true,
      "score": null,
      "durationS": 2907.529782,
      "cost": 0.940966,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "perturbation_is_pten",
            "role": "gate",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "pten"
            ]
          },
          {
            "name": "tme_non_cell_autonomous",
            "role": "gate",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "substantial_non_cell_autonomous_remodeling"
            ]
          },
          {
            "name": "tme_state_not_composition",
            "role": "gate",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "tme_age_dependent",
            "role": "gate",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_dependent"
            ]
          },
          {
            "name": "tme_response_broader_in_aged",
            "role": "gate",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_aged"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "most_responsive_noncancer_cell_types",
            "kind": "list",
            "matchKey": "cell_type",
            "k": 2,
            "expected": [
              {
                "key": "alveolar_macrophage",
                "fields": {}
              },
              {
                "key": "endothelial",
                "fields": {}
              }
            ]
          },
          {
            "name": "cf_cell_intrinsic_only",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "negligible_cell_intrinsic_only_effect",
              "remodeling_unresolved",
              "modest_remodeling_few_cell_types"
            ]
          },
          {
            "name": "cf_remodeling_is_contamination",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "apparent_remodeling_reflects_cancer_contamination"
            ]
          },
          {
            "name": "cf_perturbation_design_misread",
            "role": "hard_fail",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "multiple_tumor_suppressors_pooled",
              "no_additional_gene_difference_kras_only",
              "rb1_cdkn2a_double",
              "pcna_essential_gene_control",
              "difference_is_oncogene_not_suppressor"
            ]
          },
          {
            "name": "cf_age_independent_tme",
            "role": "hard_fail",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_independent"
            ]
          },
          {
            "name": "cf_compositional_shift",
            "role": "hard_fail",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_frequency_shift",
              "compositional_immune_infiltration",
              "composition_dominant_state_secondary",
              "both_state_and_composition"
            ]
          },
          {
            "name": "cf_epithelial_or_cancer_substitution",
            "role": "hard_fail",
            "field": "$.most_responsive_noncancer_cell_types[*].cell_type",
            "kind": "avoid",
            "expected": [
              "AT2_cell",
              "AT1_cell",
              "cancer_cell",
              "ciliated",
              "secretory",
              "club_cell",
              "basal_cell",
              "goblet_cell"
            ]
          },
          {
            "name": "cf_tme_breadth_broader_in_young",
            "role": "hard_fail",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_young"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-terra-pi",
      "modelName": "gpt-5.6-terra",
      "harness": "pi",
      "task": "gse297023_q5_tme_age_dependent_remodeling",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 1850.687058,
      "cost": 1.9268490000000003,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "perturbation_is_pten",
            "role": "gate",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "pten"
            ]
          },
          {
            "name": "tme_non_cell_autonomous",
            "role": "gate",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "substantial_non_cell_autonomous_remodeling"
            ]
          },
          {
            "name": "tme_state_not_composition",
            "role": "gate",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "tme_age_dependent",
            "role": "gate",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_dependent"
            ]
          },
          {
            "name": "tme_response_broader_in_aged",
            "role": "gate",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_aged"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "most_responsive_noncancer_cell_types",
            "kind": "list",
            "matchKey": "cell_type",
            "k": 2,
            "expected": [
              {
                "key": "alveolar_macrophage",
                "fields": {}
              },
              {
                "key": "endothelial",
                "fields": {}
              }
            ]
          },
          {
            "name": "cf_cell_intrinsic_only",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "negligible_cell_intrinsic_only_effect",
              "remodeling_unresolved",
              "modest_remodeling_few_cell_types"
            ]
          },
          {
            "name": "cf_remodeling_is_contamination",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "apparent_remodeling_reflects_cancer_contamination"
            ]
          },
          {
            "name": "cf_perturbation_design_misread",
            "role": "hard_fail",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "multiple_tumor_suppressors_pooled",
              "no_additional_gene_difference_kras_only",
              "rb1_cdkn2a_double",
              "pcna_essential_gene_control",
              "difference_is_oncogene_not_suppressor"
            ]
          },
          {
            "name": "cf_age_independent_tme",
            "role": "hard_fail",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_independent"
            ]
          },
          {
            "name": "cf_compositional_shift",
            "role": "hard_fail",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_frequency_shift",
              "compositional_immune_infiltration",
              "composition_dominant_state_secondary",
              "both_state_and_composition"
            ]
          },
          {
            "name": "cf_epithelial_or_cancer_substitution",
            "role": "hard_fail",
            "field": "$.most_responsive_noncancer_cell_types[*].cell_type",
            "kind": "avoid",
            "expected": [
              "AT2_cell",
              "AT1_cell",
              "cancer_cell",
              "ciliated",
              "secretory",
              "club_cell",
              "basal_cell",
              "goblet_cell"
            ]
          },
          {
            "name": "cf_tme_breadth_broader_in_young",
            "role": "hard_fail",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_young"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-terra-pi",
      "modelName": "gpt-5.6-terra",
      "harness": "pi",
      "task": "gse297023_q5_tme_age_dependent_remodeling",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1779.509432,
      "cost": 1.15075725,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "perturbation_is_pten",
            "role": "gate",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "pten"
            ]
          },
          {
            "name": "tme_non_cell_autonomous",
            "role": "gate",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "substantial_non_cell_autonomous_remodeling"
            ]
          },
          {
            "name": "tme_state_not_composition",
            "role": "gate",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "tme_age_dependent",
            "role": "gate",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_dependent"
            ]
          },
          {
            "name": "tme_response_broader_in_aged",
            "role": "gate",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_aged"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "most_responsive_noncancer_cell_types",
            "kind": "list",
            "matchKey": "cell_type",
            "k": 2,
            "expected": [
              {
                "key": "alveolar_macrophage",
                "fields": {}
              },
              {
                "key": "endothelial",
                "fields": {}
              }
            ]
          },
          {
            "name": "cf_cell_intrinsic_only",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "negligible_cell_intrinsic_only_effect",
              "remodeling_unresolved",
              "modest_remodeling_few_cell_types"
            ]
          },
          {
            "name": "cf_remodeling_is_contamination",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "apparent_remodeling_reflects_cancer_contamination"
            ]
          },
          {
            "name": "cf_perturbation_design_misread",
            "role": "hard_fail",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "multiple_tumor_suppressors_pooled",
              "no_additional_gene_difference_kras_only",
              "rb1_cdkn2a_double",
              "pcna_essential_gene_control",
              "difference_is_oncogene_not_suppressor"
            ]
          },
          {
            "name": "cf_age_independent_tme",
            "role": "hard_fail",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_independent"
            ]
          },
          {
            "name": "cf_compositional_shift",
            "role": "hard_fail",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_frequency_shift",
              "compositional_immune_infiltration",
              "composition_dominant_state_secondary",
              "both_state_and_composition"
            ]
          },
          {
            "name": "cf_epithelial_or_cancer_substitution",
            "role": "hard_fail",
            "field": "$.most_responsive_noncancer_cell_types[*].cell_type",
            "kind": "avoid",
            "expected": [
              "AT2_cell",
              "AT1_cell",
              "cancer_cell",
              "ciliated",
              "secretory",
              "club_cell",
              "basal_cell",
              "goblet_cell"
            ]
          },
          {
            "name": "cf_tme_breadth_broader_in_young",
            "role": "hard_fail",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_young"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-terra-pi",
      "modelName": "gpt-5.6-terra",
      "harness": "pi",
      "task": "gse297023_q6_age_immune_infiltration",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 2351.030076,
      "cost": 1.1154363750000003,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "composition_stable_with_age",
            "role": "gate",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "frequencies_largely_stable"
            ]
          },
          {
            "name": "age_signal_is_state_not_composition",
            "role": "gate",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "frequencies_uninformative_for_infiltration",
            "role": "gate",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_cannot_establish_infiltration"
            ]
          },
          {
            "name": "infiltration_increases_with_age",
            "role": "gate",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_with_age"
            ]
          },
          {
            "name": "infiltration_strength_calibrated",
            "role": "gate",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "significant_increase_strongest_markers_only",
              "consistent_increase_not_statistically_robust"
            ]
          },
          {
            "name": "genotype_marker_fraction_lower_in_pten",
            "role": "gate",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "lower_in_pten_deficient"
            ]
          },
          {
            "name": "cf_composition_shift_reading",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "major_compositional_shift_with_age",
              "increased_immune_fraction_with_age"
            ]
          },
          {
            "name": "cf_lineage_specific_composition_claim",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "myeloid_expansion_with_age",
              "lymphoid_expansion_with_age",
              "minor_shifts_in_some_cell_types"
            ]
          },
          {
            "name": "cf_immunosenescence_depletion_prior",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "immune_depletion_with_age"
            ]
          },
          {
            "name": "cf_age_modality_compositional",
            "role": "hard_fail",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_type_composition_shift",
              "both_equally",
              "composition_dominant_state_secondary"
            ]
          },
          {
            "name": "cf_overread_frequencies_as_infiltration",
            "role": "hard_fail",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_shows_no_infiltration_change",
              "single_cell_quantifies_infiltration",
              "frequencies_show_increased_infiltration",
              "frequencies_show_decreased_infiltration"
            ]
          },
          {
            "name": "cf_infiltration_decreases_immunosenescence",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "decreased_with_age"
            ]
          },
          {
            "name": "cf_infiltration_unchanged_scrna_overread",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "unchanged_with_age"
            ]
          },
          {
            "name": "cf_infiltration_indeterminate_unused_node",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "direction_indeterminate_from_data"
            ]
          },
          {
            "name": "cf_infiltration_mixed_direction",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "mixed_by_immune_cell_type"
            ]
          },
          {
            "name": "cf_infiltration_genotype_confusion",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_only_in_pten_deficient",
              "decreased_only_in_pten_deficient"
            ]
          },
          {
            "name": "cf_pten_infiltration_paper_text_recall",
            "role": "hard_fail",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "higher_in_pten_deficient",
              "higher_in_pten_deficient_only_in_young",
              "higher_in_pten_deficient_only_in_aged"
            ]
          },
          {
            "name": "cf_pseudoreplication_overclaim",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_increase_across_markers"
            ]
          },
          {
            "name": "cf_infiltration_strength_decrease_direction",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_decrease_across_markers",
              "significant_decrease_strongest_markers_only",
              "consistent_decrease_not_statistically_robust"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-terra-pi",
      "modelName": "gpt-5.6-terra",
      "harness": "pi",
      "task": "gse297023_q6_age_immune_infiltration",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 2158.517013,
      "cost": 0.8656494999999997,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "composition_stable_with_age",
            "role": "gate",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "frequencies_largely_stable"
            ]
          },
          {
            "name": "age_signal_is_state_not_composition",
            "role": "gate",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "frequencies_uninformative_for_infiltration",
            "role": "gate",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_cannot_establish_infiltration"
            ]
          },
          {
            "name": "infiltration_increases_with_age",
            "role": "gate",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_with_age"
            ]
          },
          {
            "name": "infiltration_strength_calibrated",
            "role": "gate",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "significant_increase_strongest_markers_only",
              "consistent_increase_not_statistically_robust"
            ]
          },
          {
            "name": "genotype_marker_fraction_lower_in_pten",
            "role": "gate",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "lower_in_pten_deficient"
            ]
          },
          {
            "name": "cf_composition_shift_reading",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "major_compositional_shift_with_age",
              "increased_immune_fraction_with_age"
            ]
          },
          {
            "name": "cf_lineage_specific_composition_claim",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "myeloid_expansion_with_age",
              "lymphoid_expansion_with_age",
              "minor_shifts_in_some_cell_types"
            ]
          },
          {
            "name": "cf_immunosenescence_depletion_prior",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "immune_depletion_with_age"
            ]
          },
          {
            "name": "cf_age_modality_compositional",
            "role": "hard_fail",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_type_composition_shift",
              "both_equally",
              "composition_dominant_state_secondary"
            ]
          },
          {
            "name": "cf_overread_frequencies_as_infiltration",
            "role": "hard_fail",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_shows_no_infiltration_change",
              "single_cell_quantifies_infiltration",
              "frequencies_show_increased_infiltration",
              "frequencies_show_decreased_infiltration"
            ]
          },
          {
            "name": "cf_infiltration_decreases_immunosenescence",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "decreased_with_age"
            ]
          },
          {
            "name": "cf_infiltration_unchanged_scrna_overread",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "unchanged_with_age"
            ]
          },
          {
            "name": "cf_infiltration_indeterminate_unused_node",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "direction_indeterminate_from_data"
            ]
          },
          {
            "name": "cf_infiltration_mixed_direction",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "mixed_by_immune_cell_type"
            ]
          },
          {
            "name": "cf_infiltration_genotype_confusion",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_only_in_pten_deficient",
              "decreased_only_in_pten_deficient"
            ]
          },
          {
            "name": "cf_pten_infiltration_paper_text_recall",
            "role": "hard_fail",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "higher_in_pten_deficient",
              "higher_in_pten_deficient_only_in_young",
              "higher_in_pten_deficient_only_in_aged"
            ]
          },
          {
            "name": "cf_pseudoreplication_overclaim",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_increase_across_markers"
            ]
          },
          {
            "name": "cf_infiltration_strength_decrease_direction",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_decrease_across_markers",
              "significant_decrease_strongest_markers_only",
              "consistent_decrease_not_statistically_robust"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-gpt-5-6-terra-pi",
      "modelName": "gpt-5.6-terra",
      "harness": "pi",
      "task": "gse297023_q6_age_immune_infiltration",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 3438.488982,
      "cost": 1.149433625,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "composition_stable_with_age",
            "role": "gate",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "frequencies_largely_stable"
            ]
          },
          {
            "name": "age_signal_is_state_not_composition",
            "role": "gate",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "frequencies_uninformative_for_infiltration",
            "role": "gate",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_cannot_establish_infiltration"
            ]
          },
          {
            "name": "infiltration_increases_with_age",
            "role": "gate",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_with_age"
            ]
          },
          {
            "name": "infiltration_strength_calibrated",
            "role": "gate",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "significant_increase_strongest_markers_only",
              "consistent_increase_not_statistically_robust"
            ]
          },
          {
            "name": "genotype_marker_fraction_lower_in_pten",
            "role": "gate",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "lower_in_pten_deficient"
            ]
          },
          {
            "name": "cf_composition_shift_reading",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "major_compositional_shift_with_age",
              "increased_immune_fraction_with_age"
            ]
          },
          {
            "name": "cf_lineage_specific_composition_claim",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "myeloid_expansion_with_age",
              "lymphoid_expansion_with_age",
              "minor_shifts_in_some_cell_types"
            ]
          },
          {
            "name": "cf_immunosenescence_depletion_prior",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "immune_depletion_with_age"
            ]
          },
          {
            "name": "cf_age_modality_compositional",
            "role": "hard_fail",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_type_composition_shift",
              "both_equally",
              "composition_dominant_state_secondary"
            ]
          },
          {
            "name": "cf_overread_frequencies_as_infiltration",
            "role": "hard_fail",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_shows_no_infiltration_change",
              "single_cell_quantifies_infiltration",
              "frequencies_show_increased_infiltration",
              "frequencies_show_decreased_infiltration"
            ]
          },
          {
            "name": "cf_infiltration_decreases_immunosenescence",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "decreased_with_age"
            ]
          },
          {
            "name": "cf_infiltration_unchanged_scrna_overread",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "unchanged_with_age"
            ]
          },
          {
            "name": "cf_infiltration_indeterminate_unused_node",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "direction_indeterminate_from_data"
            ]
          },
          {
            "name": "cf_infiltration_mixed_direction",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "mixed_by_immune_cell_type"
            ]
          },
          {
            "name": "cf_infiltration_genotype_confusion",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_only_in_pten_deficient",
              "decreased_only_in_pten_deficient"
            ]
          },
          {
            "name": "cf_pten_infiltration_paper_text_recall",
            "role": "hard_fail",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "higher_in_pten_deficient",
              "higher_in_pten_deficient_only_in_young",
              "higher_in_pten_deficient_only_in_aged"
            ]
          },
          {
            "name": "cf_pseudoreplication_overclaim",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_increase_across_markers"
            ]
          },
          {
            "name": "cf_infiltration_strength_decrease_direction",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_decrease_across_markers",
              "significant_decrease_strongest_markers_only",
              "consistent_decrease_not_statistically_robust"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-5-pi",
      "modelName": "grok-4.5",
      "harness": "pi",
      "task": "gse155381_q4_pseudotime_delay",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 1,
      "passed": true,
      "score": null,
      "durationS": 473.095036,
      "cost": 0.9529099999999999,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "pseudotime_direction_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "E17_chronological_match_weighted",
            "role": "additive",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pluripotency_exit_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pseudotime_direction_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_synchronous_with_monkey_EPI",
              "chimera_human_EPI_developmentally_advanced",
              "chimera_human_EPI_delayed_at_pre_implantation_only_resolved_post_implantation"
            ]
          },
          {
            "name": "E17_chronological_match_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "resembles_monkey_E17_Gast_EPI",
              "resembles_chimera_monkey_E17_EPI_specifically",
              "chimera_human_E17_EPI_more_advanced_than_monkey_E17_Gast"
            ]
          },
          {
            "name": "pluripotency_exit_paper_inverted_or_prior_driven",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_advances_pluripotency_exit_prematurely",
              "naive_pluripotency_program_specifically_engaged",
              "pluripotency_core_retained_via_KLF17_naive_marker_specifically"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-5-pi",
      "modelName": "grok-4.5",
      "harness": "pi",
      "task": "gse155381_q4_pseudotime_delay",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 1073.748459,
      "cost": 1.30034,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "pseudotime_direction_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "E17_chronological_match_weighted",
            "role": "additive",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pluripotency_exit_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pseudotime_direction_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_synchronous_with_monkey_EPI",
              "chimera_human_EPI_developmentally_advanced",
              "chimera_human_EPI_delayed_at_pre_implantation_only_resolved_post_implantation"
            ]
          },
          {
            "name": "E17_chronological_match_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "resembles_monkey_E17_Gast_EPI",
              "resembles_chimera_monkey_E17_EPI_specifically",
              "chimera_human_E17_EPI_more_advanced_than_monkey_E17_Gast"
            ]
          },
          {
            "name": "pluripotency_exit_paper_inverted_or_prior_driven",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_advances_pluripotency_exit_prematurely",
              "naive_pluripotency_program_specifically_engaged",
              "pluripotency_core_retained_via_KLF17_naive_marker_specifically"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-5-pi",
      "modelName": "grok-4.5",
      "harness": "pi",
      "task": "gse155381_q4_pseudotime_delay",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 974.728807,
      "cost": 1.8871819999999997,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "pseudotime_direction_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "E17_chronological_match_weighted",
            "role": "additive",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pluripotency_exit_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pseudotime_direction_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_synchronous_with_monkey_EPI",
              "chimera_human_EPI_developmentally_advanced",
              "chimera_human_EPI_delayed_at_pre_implantation_only_resolved_post_implantation"
            ]
          },
          {
            "name": "E17_chronological_match_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "resembles_monkey_E17_Gast_EPI",
              "resembles_chimera_monkey_E17_EPI_specifically",
              "chimera_human_E17_EPI_more_advanced_than_monkey_E17_Gast"
            ]
          },
          {
            "name": "pluripotency_exit_paper_inverted_or_prior_driven",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_advances_pluripotency_exit_prematurely",
              "naive_pluripotency_program_specifically_engaged",
              "pluripotency_core_retained_via_KLF17_naive_marker_specifically"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-5-pi",
      "modelName": "grok-4.5",
      "harness": "pi",
      "task": "gse155381_q5_chimera_monkey_perturbation",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 421.664112,
      "cost": 0.24521400000000002,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "deg_direction_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "comparison_subset_choice_weighted",
            "role": "additive",
            "field": "$.comparison_subset_choice",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "panel_resolved_strong_target_ccn1_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN1",
              "CYR61"
            ]
          },
          {
            "name": "panel_resolved_strong_target_ccn2_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN2",
              "CTGF"
            ]
          },
          {
            "name": "hippo_pathway_weighted",
            "role": "additive",
            "field": "$.hippo_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "tgf_beta_pathway_weighted",
            "role": "additive",
            "field": "$.tgf_beta_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "deg_direction_batch_effect_uncorrected",
            "role": "hard_fail",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_dominated_global",
              "up_dominated_in_chosen_subset"
            ]
          },
          {
            "name": "silent_panel_dropout_paired_with_directional_pathway_claim",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "hippo_pathway_direction": {
                "op": "in",
                "values": [
                  "hippo_pathway_upregulated",
                  "hippo_pathway_unchanged",
                  "wnt_pathway_specifically_perturbed",
                  "stress_response_pathway_dominantly_engaged",
                  "no_pathway_signal_above_confound",
                  "yap_tead_direct_target_panel_downregulated",
                  "yap_tead_panel_dependent_direction",
                  "hippo_pathway_downregulated"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-5-pi",
      "modelName": "grok-4.5",
      "harness": "pi",
      "task": "gse155381_q5_chimera_monkey_perturbation",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 2720.28296,
      "cost": 0.551434,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "deg_direction_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "comparison_subset_choice_weighted",
            "role": "additive",
            "field": "$.comparison_subset_choice",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "panel_resolved_strong_target_ccn1_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN1",
              "CYR61"
            ]
          },
          {
            "name": "panel_resolved_strong_target_ccn2_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN2",
              "CTGF"
            ]
          },
          {
            "name": "hippo_pathway_weighted",
            "role": "additive",
            "field": "$.hippo_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "tgf_beta_pathway_weighted",
            "role": "additive",
            "field": "$.tgf_beta_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "deg_direction_batch_effect_uncorrected",
            "role": "hard_fail",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_dominated_global",
              "up_dominated_in_chosen_subset"
            ]
          },
          {
            "name": "silent_panel_dropout_paired_with_directional_pathway_claim",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "hippo_pathway_direction": {
                "op": "in",
                "values": [
                  "hippo_pathway_upregulated",
                  "hippo_pathway_unchanged",
                  "wnt_pathway_specifically_perturbed",
                  "stress_response_pathway_dominantly_engaged",
                  "no_pathway_signal_above_confound",
                  "yap_tead_direct_target_panel_downregulated",
                  "yap_tead_panel_dependent_direction",
                  "hippo_pathway_downregulated"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-5-pi",
      "modelName": "grok-4.5",
      "harness": "pi",
      "task": "gse155381_q5_chimera_monkey_perturbation",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 389.500912,
      "cost": 0.57816,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "deg_direction_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "comparison_subset_choice_weighted",
            "role": "additive",
            "field": "$.comparison_subset_choice",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "panel_resolved_strong_target_ccn1_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN1",
              "CYR61"
            ]
          },
          {
            "name": "panel_resolved_strong_target_ccn2_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN2",
              "CTGF"
            ]
          },
          {
            "name": "hippo_pathway_weighted",
            "role": "additive",
            "field": "$.hippo_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "tgf_beta_pathway_weighted",
            "role": "additive",
            "field": "$.tgf_beta_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "deg_direction_batch_effect_uncorrected",
            "role": "hard_fail",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_dominated_global",
              "up_dominated_in_chosen_subset"
            ]
          },
          {
            "name": "silent_panel_dropout_paired_with_directional_pathway_claim",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "hippo_pathway_direction": {
                "op": "in",
                "values": [
                  "hippo_pathway_upregulated",
                  "hippo_pathway_unchanged",
                  "wnt_pathway_specifically_perturbed",
                  "stress_response_pathway_dominantly_engaged",
                  "no_pathway_signal_above_confound",
                  "yap_tead_direct_target_panel_downregulated",
                  "yap_tead_panel_dependent_direction",
                  "hippo_pathway_downregulated"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-5-pi",
      "modelName": "grok-4.5",
      "harness": "pi",
      "task": "gse155381_q7_xspecies_epi_communication",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 582.898892,
      "cost": 0.8003380000000001,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "richness_weighted",
            "role": "additive",
            "field": "$.xspecies_signaling_richness",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_weighted",
            "role": "additive",
            "field": "$.dominant_signaling_axis",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_wnt_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "WNT_FZD_LRP_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_fgf_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "FGF_FGFR_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_notch_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "NOTCH_DLL_JAG_family"
                ]
              }
            }
          },
          {
            "name": "signaling_directionality_weighted",
            "role": "additive",
            "field": "$.signaling_directionality",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "top_signal_interaction_class_weighted",
            "role": "additive",
            "field": "$.top_signal_interaction_class",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-5-pi",
      "modelName": "grok-4.5",
      "harness": "pi",
      "task": "gse155381_q7_xspecies_epi_communication",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 1252.594052,
      "cost": 0.606544,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "richness_weighted",
            "role": "additive",
            "field": "$.xspecies_signaling_richness",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_weighted",
            "role": "additive",
            "field": "$.dominant_signaling_axis",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_wnt_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "WNT_FZD_LRP_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_fgf_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "FGF_FGFR_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_notch_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "NOTCH_DLL_JAG_family"
                ]
              }
            }
          },
          {
            "name": "signaling_directionality_weighted",
            "role": "additive",
            "field": "$.signaling_directionality",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "top_signal_interaction_class_weighted",
            "role": "additive",
            "field": "$.top_signal_interaction_class",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-5-pi",
      "modelName": "grok-4.5",
      "harness": "pi",
      "task": "gse155381_q7_xspecies_epi_communication",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 631.097675,
      "cost": 0.530598,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "richness_weighted",
            "role": "additive",
            "field": "$.xspecies_signaling_richness",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_weighted",
            "role": "additive",
            "field": "$.dominant_signaling_axis",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_wnt_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "WNT_FZD_LRP_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_fgf_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "FGF_FGFR_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_notch_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "NOTCH_DLL_JAG_family"
                ]
              }
            }
          },
          {
            "name": "signaling_directionality_weighted",
            "role": "additive",
            "field": "$.signaling_directionality",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "top_signal_interaction_class_weighted",
            "role": "additive",
            "field": "$.top_signal_interaction_class",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-5-pi",
      "modelName": "grok-4.5",
      "harness": "pi",
      "task": "gse155381_q8_bidirectional_convergence",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 1,
      "passed": true,
      "score": null,
      "durationS": 809.161694,
      "cost": 1.5840739999999998,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "chimera_human_identity_shift_target_weighted",
            "role": "additive",
            "field": "$.chimera_human_identity_shift_target",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_in_vivo_state_assignment_weighted",
            "role": "additive",
            "field": "$.chimera_human_in_vivo_state_assignment",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_shift_magnitude_weighted",
            "role": "additive",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_monkey_identity_shift_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "both_arms_null_contradicted_by_calibration",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "chimera_human_identity_shift_target": {
                "op": "equals",
                "values": [
                  "chimera_human_maintains_origin_identity_no_measurable_shift"
                ]
              },
              "chimera_monkey_identity_shift": {
                "op": "equals",
                "values": [
                  "chimera_monkey_unchanged_relative_to_control"
                ]
              }
            }
          },
          {
            "name": "chimera_monkey_paper_claim_on_confounded_input",
            "role": "hard_fail",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "value",
            "expected": "chimera_monkey_shifts_toward_chimera_human_identity"
          },
          {
            "name": "chimera_human_magnitude_wrong_attribution_signature",
            "role": "hard_fail",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "value",
            "op": "in",
            "expected": [
              "dramatic_complete_replacement_of_origin_identity",
              "magnitude_dominated_by_culture_or_metabolic_axis",
              "magnitude_dominated_by_developmental_stage_axis",
              "magnitude_dominated_by_cell_cycle_axis",
              "magnitude_dominated_by_batch_effect_axis"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-5-pi",
      "modelName": "grok-4.5",
      "harness": "pi",
      "task": "gse155381_q8_bidirectional_convergence",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 1281.389093,
      "cost": 1.4792400000000003,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "chimera_human_identity_shift_target_weighted",
            "role": "additive",
            "field": "$.chimera_human_identity_shift_target",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_in_vivo_state_assignment_weighted",
            "role": "additive",
            "field": "$.chimera_human_in_vivo_state_assignment",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_shift_magnitude_weighted",
            "role": "additive",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_monkey_identity_shift_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "both_arms_null_contradicted_by_calibration",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "chimera_human_identity_shift_target": {
                "op": "equals",
                "values": [
                  "chimera_human_maintains_origin_identity_no_measurable_shift"
                ]
              },
              "chimera_monkey_identity_shift": {
                "op": "equals",
                "values": [
                  "chimera_monkey_unchanged_relative_to_control"
                ]
              }
            }
          },
          {
            "name": "chimera_monkey_paper_claim_on_confounded_input",
            "role": "hard_fail",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "value",
            "expected": "chimera_monkey_shifts_toward_chimera_human_identity"
          },
          {
            "name": "chimera_human_magnitude_wrong_attribution_signature",
            "role": "hard_fail",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "value",
            "op": "in",
            "expected": [
              "dramatic_complete_replacement_of_origin_identity",
              "magnitude_dominated_by_culture_or_metabolic_axis",
              "magnitude_dominated_by_developmental_stage_axis",
              "magnitude_dominated_by_cell_cycle_axis",
              "magnitude_dominated_by_batch_effect_axis"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-5-pi",
      "modelName": "grok-4.5",
      "harness": "pi",
      "task": "gse155381_q8_bidirectional_convergence",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1666.598623,
      "cost": 1.6446719999999997,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "chimera_human_identity_shift_target_weighted",
            "role": "additive",
            "field": "$.chimera_human_identity_shift_target",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_in_vivo_state_assignment_weighted",
            "role": "additive",
            "field": "$.chimera_human_in_vivo_state_assignment",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_shift_magnitude_weighted",
            "role": "additive",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_monkey_identity_shift_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "both_arms_null_contradicted_by_calibration",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "chimera_human_identity_shift_target": {
                "op": "equals",
                "values": [
                  "chimera_human_maintains_origin_identity_no_measurable_shift"
                ]
              },
              "chimera_monkey_identity_shift": {
                "op": "equals",
                "values": [
                  "chimera_monkey_unchanged_relative_to_control"
                ]
              }
            }
          },
          {
            "name": "chimera_monkey_paper_claim_on_confounded_input",
            "role": "hard_fail",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "value",
            "expected": "chimera_monkey_shifts_toward_chimera_human_identity"
          },
          {
            "name": "chimera_human_magnitude_wrong_attribution_signature",
            "role": "hard_fail",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "value",
            "op": "in",
            "expected": [
              "dramatic_complete_replacement_of_origin_identity",
              "magnitude_dominated_by_culture_or_metabolic_axis",
              "magnitude_dominated_by_developmental_stage_axis",
              "magnitude_dominated_by_cell_cycle_axis",
              "magnitude_dominated_by_batch_effect_axis"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-5-pi",
      "modelName": "grok-4.5",
      "harness": "pi",
      "task": "gse171524_ectopic_airway_rare_cell",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 2522.256644,
      "cost": 0.38915800000000006,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "focus_population_identity_gate",
            "role": "gate",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "chemosensory_marker_expressing_population_alpha"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "focus_population_marker_calls",
            "kind": "list",
            "matchKey": "marker",
            "k": 3,
            "expected": [
              {
                "key": "POU2F3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "CHAT",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "TRPM5",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "LRMP",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNAT3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "AVIL",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNG13",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              }
            ]
          },
          {
            "name": "focus_population_condition_distribution_gate",
            "role": "gate",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_enriched_with_substantial_control_presence",
              "covid_predominant",
              "covid_only",
              "covid_enriched_trend_not_significant"
            ]
          },
          {
            "name": "focus_population_location_gate",
            "role": "gate",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "ectopic_parenchyma_in_covid",
              "parenchyma_only",
              "bronchiolized_or_peribronchial_parenchyma",
              "airway_and_parenchyma"
            ]
          },
          {
            "name": "focus_population_recovery_mode_gate",
            "role": "gate",
            "field": "$.focus_population_recovery_mode",
            "kind": "value",
            "op": "in",
            "expected": [
              "continuum_not_discrete_cluster"
            ]
          },
          {
            "name": "focus_population_identity_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "ionocyte_like_beta",
              "neuroendocrine_like_gamma",
              "basal_stem_progenitor_like_delta",
              "aberrant_basaloid_transitional_like_epsilon",
              "neuronal_or_neural_gene_minor_population_zeta",
              "deuterosomal_ciliated_progenitor_like_eta",
              "hillock_transitional_like_theta",
              "low_quality_doublet_or_debris",
              "no_rare_cell_recovered",
              "not_assessed"
            ]
          },
          {
            "name": "cf_wrong_lineage_marker_called_positive",
            "role": "hard_fail",
            "field": "$.focus_population_marker_calls[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "focus_population_condition_distribution_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "equally_distributed",
              "control_predominant",
              "not_assessed"
            ]
          },
          {
            "name": "focus_population_location_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "airway_only",
              "not_assessed"
            ]
          },
          {
            "name": "cross_axis_canonical_identity_with_airway_only_location_incoherence",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "focus_population_identity": {
                "op": "in",
                "values": [
                  "chemosensory_marker_expressing_population_alpha"
                ]
              },
              "focus_population_location": {
                "op": "in",
                "values": [
                  "airway_only"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-5-pi",
      "modelName": "grok-4.5",
      "harness": "pi",
      "task": "gse171524_ectopic_airway_rare_cell",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 3664.399125,
      "cost": 0.604572,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "focus_population_identity_gate",
            "role": "gate",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "chemosensory_marker_expressing_population_alpha"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "focus_population_marker_calls",
            "kind": "list",
            "matchKey": "marker",
            "k": 3,
            "expected": [
              {
                "key": "POU2F3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "CHAT",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "TRPM5",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "LRMP",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNAT3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "AVIL",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNG13",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              }
            ]
          },
          {
            "name": "focus_population_condition_distribution_gate",
            "role": "gate",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_enriched_with_substantial_control_presence",
              "covid_predominant",
              "covid_only",
              "covid_enriched_trend_not_significant"
            ]
          },
          {
            "name": "focus_population_location_gate",
            "role": "gate",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "ectopic_parenchyma_in_covid",
              "parenchyma_only",
              "bronchiolized_or_peribronchial_parenchyma",
              "airway_and_parenchyma"
            ]
          },
          {
            "name": "focus_population_recovery_mode_gate",
            "role": "gate",
            "field": "$.focus_population_recovery_mode",
            "kind": "value",
            "op": "in",
            "expected": [
              "continuum_not_discrete_cluster"
            ]
          },
          {
            "name": "focus_population_identity_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "ionocyte_like_beta",
              "neuroendocrine_like_gamma",
              "basal_stem_progenitor_like_delta",
              "aberrant_basaloid_transitional_like_epsilon",
              "neuronal_or_neural_gene_minor_population_zeta",
              "deuterosomal_ciliated_progenitor_like_eta",
              "hillock_transitional_like_theta",
              "low_quality_doublet_or_debris",
              "no_rare_cell_recovered",
              "not_assessed"
            ]
          },
          {
            "name": "cf_wrong_lineage_marker_called_positive",
            "role": "hard_fail",
            "field": "$.focus_population_marker_calls[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "focus_population_condition_distribution_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "equally_distributed",
              "control_predominant",
              "not_assessed"
            ]
          },
          {
            "name": "focus_population_location_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "airway_only",
              "not_assessed"
            ]
          },
          {
            "name": "cross_axis_canonical_identity_with_airway_only_location_incoherence",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "focus_population_identity": {
                "op": "in",
                "values": [
                  "chemosensory_marker_expressing_population_alpha"
                ]
              },
              "focus_population_location": {
                "op": "in",
                "values": [
                  "airway_only"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-5-pi",
      "modelName": "grok-4.5",
      "harness": "pi",
      "task": "gse171524_ectopic_airway_rare_cell",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1262.95707,
      "cost": 0.32535,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "focus_population_identity_gate",
            "role": "gate",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "chemosensory_marker_expressing_population_alpha"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "focus_population_marker_calls",
            "kind": "list",
            "matchKey": "marker",
            "k": 3,
            "expected": [
              {
                "key": "POU2F3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "CHAT",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "TRPM5",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "LRMP",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNAT3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "AVIL",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNG13",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              }
            ]
          },
          {
            "name": "focus_population_condition_distribution_gate",
            "role": "gate",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_enriched_with_substantial_control_presence",
              "covid_predominant",
              "covid_only",
              "covid_enriched_trend_not_significant"
            ]
          },
          {
            "name": "focus_population_location_gate",
            "role": "gate",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "ectopic_parenchyma_in_covid",
              "parenchyma_only",
              "bronchiolized_or_peribronchial_parenchyma",
              "airway_and_parenchyma"
            ]
          },
          {
            "name": "focus_population_recovery_mode_gate",
            "role": "gate",
            "field": "$.focus_population_recovery_mode",
            "kind": "value",
            "op": "in",
            "expected": [
              "continuum_not_discrete_cluster"
            ]
          },
          {
            "name": "focus_population_identity_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "ionocyte_like_beta",
              "neuroendocrine_like_gamma",
              "basal_stem_progenitor_like_delta",
              "aberrant_basaloid_transitional_like_epsilon",
              "neuronal_or_neural_gene_minor_population_zeta",
              "deuterosomal_ciliated_progenitor_like_eta",
              "hillock_transitional_like_theta",
              "low_quality_doublet_or_debris",
              "no_rare_cell_recovered",
              "not_assessed"
            ]
          },
          {
            "name": "cf_wrong_lineage_marker_called_positive",
            "role": "hard_fail",
            "field": "$.focus_population_marker_calls[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "focus_population_condition_distribution_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "equally_distributed",
              "control_predominant",
              "not_assessed"
            ]
          },
          {
            "name": "focus_population_location_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "airway_only",
              "not_assessed"
            ]
          },
          {
            "name": "cross_axis_canonical_identity_with_airway_only_location_incoherence",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "focus_population_identity": {
                "op": "in",
                "values": [
                  "chemosensory_marker_expressing_population_alpha"
                ]
              },
              "focus_population_location": {
                "op": "in",
                "values": [
                  "airway_only"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-5-pi",
      "modelName": "grok-4.5",
      "harness": "pi",
      "task": "gse171524_pathological_fibroblast_master_regulator_druggable_targets",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 1,
      "passed": true,
      "score": null,
      "durationS": 1310.546801,
      "cost": 0.346962,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "disease_associated_fibroblast_subtype_weighted",
            "role": "additive",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "mr_tf_junb_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_alpha"
            ]
          },
          {
            "name": "mr_tf_jund_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_beta"
            ]
          },
          {
            "name": "drug_mmp14_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_alpha"
            ]
          },
          {
            "name": "drug_stat3_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_beta"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_weighted",
            "role": "additive",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "disease_associated_fibroblast_subtype_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "value",
            "op": "in",
            "expected": [
              "subtype_lambda",
              "subtype_epsilon",
              "subtype_iota",
              "subtype_beta",
              "subtype_eta",
              "subtype_zeta",
              "subtype_not_assessed"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "simple_de_ranking_by_log_fc",
              "marker_based_de_only",
              "not_computed"
            ]
          },
          {
            "name": "top_master_regulator_transcription_factors_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_not_computed"
            ]
          },
          {
            "name": "top_druggable_proteins_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_not_assessed"
            ]
          },
          {
            "name": "cross_axis_circular_marker_in_druggable_proteins",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_gamma",
              "drug_target_delta",
              "drug_target_eta"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-5-pi",
      "modelName": "grok-4.5",
      "harness": "pi",
      "task": "gse171524_pathological_fibroblast_master_regulator_druggable_targets",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 2,
      "passed": true,
      "score": null,
      "durationS": 3023.995877,
      "cost": 0.44233200000000006,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "disease_associated_fibroblast_subtype_weighted",
            "role": "additive",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "mr_tf_junb_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_alpha"
            ]
          },
          {
            "name": "mr_tf_jund_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_beta"
            ]
          },
          {
            "name": "drug_mmp14_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_alpha"
            ]
          },
          {
            "name": "drug_stat3_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_beta"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_weighted",
            "role": "additive",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "disease_associated_fibroblast_subtype_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "value",
            "op": "in",
            "expected": [
              "subtype_lambda",
              "subtype_epsilon",
              "subtype_iota",
              "subtype_beta",
              "subtype_eta",
              "subtype_zeta",
              "subtype_not_assessed"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "simple_de_ranking_by_log_fc",
              "marker_based_de_only",
              "not_computed"
            ]
          },
          {
            "name": "top_master_regulator_transcription_factors_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_not_computed"
            ]
          },
          {
            "name": "top_druggable_proteins_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_not_assessed"
            ]
          },
          {
            "name": "cross_axis_circular_marker_in_druggable_proteins",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_gamma",
              "drug_target_delta",
              "drug_target_eta"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-5-pi",
      "modelName": "grok-4.5",
      "harness": "pi",
      "task": "gse171524_pathological_fibroblast_master_regulator_druggable_targets",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1770.608886,
      "cost": 0.5842759999999999,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "disease_associated_fibroblast_subtype_weighted",
            "role": "additive",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "mr_tf_junb_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_alpha"
            ]
          },
          {
            "name": "mr_tf_jund_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_beta"
            ]
          },
          {
            "name": "drug_mmp14_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_alpha"
            ]
          },
          {
            "name": "drug_stat3_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_beta"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_weighted",
            "role": "additive",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "disease_associated_fibroblast_subtype_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "value",
            "op": "in",
            "expected": [
              "subtype_lambda",
              "subtype_epsilon",
              "subtype_iota",
              "subtype_beta",
              "subtype_eta",
              "subtype_zeta",
              "subtype_not_assessed"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "simple_de_ranking_by_log_fc",
              "marker_based_de_only",
              "not_computed"
            ]
          },
          {
            "name": "top_master_regulator_transcription_factors_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_not_computed"
            ]
          },
          {
            "name": "top_druggable_proteins_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_not_assessed"
            ]
          },
          {
            "name": "cross_axis_circular_marker_in_druggable_proteins",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_gamma",
              "drug_target_delta",
              "drug_target_eta"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-5-pi",
      "modelName": "grok-4.5",
      "harness": "pi",
      "task": "gse171524_plasma_bcr_neutralizing_clonotype",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 423.806757,
      "cost": 0.5495079999999999,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dominant_pairing_heavy_gate",
            "role": "gate",
            "field": "$.dominant_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV4-34"
            ]
          },
          {
            "name": "dominant_pairing_light_gate",
            "role": "gate",
            "field": "$.dominant_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGLV3-19"
            ]
          },
          {
            "name": "dominant_neutralizer_match_gate",
            "role": "gate",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_known_neutralizer_match"
            ]
          },
          {
            "name": "neutralizer_pairing_heavy_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV1-18"
            ]
          },
          {
            "name": "neutralizer_pairing_light_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGKV3-20"
            ]
          },
          {
            "name": "neutralizer_recurrence_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "observed_in_2_to_5_donors"
            ]
          },
          {
            "name": "neutralizer_rank_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "top_5_pairing",
              "top_6_to_20_pairing"
            ]
          },
          {
            "name": "dominant_clone_isotype_gate",
            "role": "gate",
            "field": "$.dominant_clone_isotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "IgG"
            ]
          },
          {
            "name": "blineage_class_switch_contrast_gate",
            "role": "gate",
            "field": "$.blineage_class_switch_contrast",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_class_switched_control_higher_unswitched",
              "class_switched_in_both_cohorts"
            ]
          },
          {
            "name": "dominant_match_narrative_trap_or_null",
            "role": "hard_fail",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_rbd_neutralizer",
              "matches_known_autoreactive_clone",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_heavy",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_light",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_recurrence_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_rank_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "cross_axis_found_dominant_but_failed_to_scan_for_embedded_neutralizer",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_pairing_neutralizer_match": {
                "op": "in",
                "values": [
                  "no_known_neutralizer_match"
                ]
              },
              "neutralizer_pairing_heavy": {
                "op": "in",
                "values": [
                  "none_detected"
                ]
              }
            }
          },
          {
            "name": "neutralizer_pairing_not_exactly_the_s309_pair",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "op": "not",
            "expected": {
              "op": "and",
              "args": [
                {
                  "op": "field",
                  "name": "neutralizer_pairing_heavy",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGHV1-18"
                    ]
                  }
                },
                {
                  "op": "field",
                  "name": "neutralizer_pairing_light",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGKV3-20"
                    ]
                  }
                }
              ]
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-5-pi",
      "modelName": "grok-4.5",
      "harness": "pi",
      "task": "gse171524_plasma_bcr_neutralizing_clonotype",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 365.910796,
      "cost": 0.37399999999999994,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dominant_pairing_heavy_gate",
            "role": "gate",
            "field": "$.dominant_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV4-34"
            ]
          },
          {
            "name": "dominant_pairing_light_gate",
            "role": "gate",
            "field": "$.dominant_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGLV3-19"
            ]
          },
          {
            "name": "dominant_neutralizer_match_gate",
            "role": "gate",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_known_neutralizer_match"
            ]
          },
          {
            "name": "neutralizer_pairing_heavy_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV1-18"
            ]
          },
          {
            "name": "neutralizer_pairing_light_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGKV3-20"
            ]
          },
          {
            "name": "neutralizer_recurrence_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "observed_in_2_to_5_donors"
            ]
          },
          {
            "name": "neutralizer_rank_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "top_5_pairing",
              "top_6_to_20_pairing"
            ]
          },
          {
            "name": "dominant_clone_isotype_gate",
            "role": "gate",
            "field": "$.dominant_clone_isotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "IgG"
            ]
          },
          {
            "name": "blineage_class_switch_contrast_gate",
            "role": "gate",
            "field": "$.blineage_class_switch_contrast",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_class_switched_control_higher_unswitched",
              "class_switched_in_both_cohorts"
            ]
          },
          {
            "name": "dominant_match_narrative_trap_or_null",
            "role": "hard_fail",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_rbd_neutralizer",
              "matches_known_autoreactive_clone",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_heavy",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_light",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_recurrence_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_rank_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "cross_axis_found_dominant_but_failed_to_scan_for_embedded_neutralizer",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_pairing_neutralizer_match": {
                "op": "in",
                "values": [
                  "no_known_neutralizer_match"
                ]
              },
              "neutralizer_pairing_heavy": {
                "op": "in",
                "values": [
                  "none_detected"
                ]
              }
            }
          },
          {
            "name": "neutralizer_pairing_not_exactly_the_s309_pair",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "op": "not",
            "expected": {
              "op": "and",
              "args": [
                {
                  "op": "field",
                  "name": "neutralizer_pairing_heavy",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGHV1-18"
                    ]
                  }
                },
                {
                  "op": "field",
                  "name": "neutralizer_pairing_light",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGKV3-20"
                    ]
                  }
                }
              ]
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-5-pi",
      "modelName": "grok-4.5",
      "harness": "pi",
      "task": "gse171524_plasma_bcr_neutralizing_clonotype",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 275.682283,
      "cost": 0.280294,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dominant_pairing_heavy_gate",
            "role": "gate",
            "field": "$.dominant_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV4-34"
            ]
          },
          {
            "name": "dominant_pairing_light_gate",
            "role": "gate",
            "field": "$.dominant_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGLV3-19"
            ]
          },
          {
            "name": "dominant_neutralizer_match_gate",
            "role": "gate",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_known_neutralizer_match"
            ]
          },
          {
            "name": "neutralizer_pairing_heavy_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV1-18"
            ]
          },
          {
            "name": "neutralizer_pairing_light_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGKV3-20"
            ]
          },
          {
            "name": "neutralizer_recurrence_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "observed_in_2_to_5_donors"
            ]
          },
          {
            "name": "neutralizer_rank_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "top_5_pairing",
              "top_6_to_20_pairing"
            ]
          },
          {
            "name": "dominant_clone_isotype_gate",
            "role": "gate",
            "field": "$.dominant_clone_isotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "IgG"
            ]
          },
          {
            "name": "blineage_class_switch_contrast_gate",
            "role": "gate",
            "field": "$.blineage_class_switch_contrast",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_class_switched_control_higher_unswitched",
              "class_switched_in_both_cohorts"
            ]
          },
          {
            "name": "dominant_match_narrative_trap_or_null",
            "role": "hard_fail",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_rbd_neutralizer",
              "matches_known_autoreactive_clone",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_heavy",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_light",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_recurrence_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_rank_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "cross_axis_found_dominant_but_failed_to_scan_for_embedded_neutralizer",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_pairing_neutralizer_match": {
                "op": "in",
                "values": [
                  "no_known_neutralizer_match"
                ]
              },
              "neutralizer_pairing_heavy": {
                "op": "in",
                "values": [
                  "none_detected"
                ]
              }
            }
          },
          {
            "name": "neutralizer_pairing_not_exactly_the_s309_pair",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "op": "not",
            "expected": {
              "op": "and",
              "args": [
                {
                  "op": "field",
                  "name": "neutralizer_pairing_heavy",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGHV1-18"
                    ]
                  }
                },
                {
                  "op": "field",
                  "name": "neutralizer_pairing_light",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGKV3-20"
                    ]
                  }
                }
              ]
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-5-pi",
      "modelName": "grok-4.5",
      "harness": "pi",
      "task": "q6_myeloid_subset_restricted_de",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 1686.069612,
      "cost": 0.28492399999999996,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "myeloid_subset_with_strongest_change_gate",
            "role": "gate",
            "field": "$.myeloid_subset_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "subset_gamma"
            ]
          },
          {
            "name": "gene_with_strongest_change_gate",
            "role": "gate",
            "field": "$.gene_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "AXL",
              "PROS1"
            ]
          },
          {
            "name": "direction_of_change_gate",
            "role": "gate",
            "field": "$.direction_of_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "down_in_covid"
            ]
          },
          {
            "name": "biological_function_interpretation_gate",
            "role": "gate",
            "field": "$.biological_function_interpretation",
            "kind": "value",
            "op": "in",
            "expected": [
              "efferocytosis",
              "phagocytosis"
            ]
          },
          {
            "name": "myeloid_compartment_abundance_gate",
            "role": "gate",
            "field": "$.myeloid_compartment_abundance",
            "kind": "value",
            "op": "in",
            "expected": [
              "expands_in_covid"
            ]
          },
          {
            "name": "myeloid_expansion_localization_gate",
            "role": "gate",
            "field": "$.myeloid_expansion_localization",
            "kind": "value",
            "op": "in",
            "expected": [
              "recruited_driven_resident_dysfunction"
            ]
          },
          {
            "name": "dominant_signature_gate",
            "role": "gate",
            "field": "$.dominant_signature",
            "kind": "value",
            "op": "in",
            "expected": [
              "technical_or_agonal_artifact"
            ]
          },
          {
            "name": "dominant_signature_direction_gate",
            "role": "gate",
            "field": "$.dominant_signature_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_in_covid"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-5-pi",
      "modelName": "grok-4.5",
      "harness": "pi",
      "task": "q6_myeloid_subset_restricted_de",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 1866.004831,
      "cost": 0.339458,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "myeloid_subset_with_strongest_change_gate",
            "role": "gate",
            "field": "$.myeloid_subset_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "subset_gamma"
            ]
          },
          {
            "name": "gene_with_strongest_change_gate",
            "role": "gate",
            "field": "$.gene_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "AXL",
              "PROS1"
            ]
          },
          {
            "name": "direction_of_change_gate",
            "role": "gate",
            "field": "$.direction_of_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "down_in_covid"
            ]
          },
          {
            "name": "biological_function_interpretation_gate",
            "role": "gate",
            "field": "$.biological_function_interpretation",
            "kind": "value",
            "op": "in",
            "expected": [
              "efferocytosis",
              "phagocytosis"
            ]
          },
          {
            "name": "myeloid_compartment_abundance_gate",
            "role": "gate",
            "field": "$.myeloid_compartment_abundance",
            "kind": "value",
            "op": "in",
            "expected": [
              "expands_in_covid"
            ]
          },
          {
            "name": "myeloid_expansion_localization_gate",
            "role": "gate",
            "field": "$.myeloid_expansion_localization",
            "kind": "value",
            "op": "in",
            "expected": [
              "recruited_driven_resident_dysfunction"
            ]
          },
          {
            "name": "dominant_signature_gate",
            "role": "gate",
            "field": "$.dominant_signature",
            "kind": "value",
            "op": "in",
            "expected": [
              "technical_or_agonal_artifact"
            ]
          },
          {
            "name": "dominant_signature_direction_gate",
            "role": "gate",
            "field": "$.dominant_signature_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_in_covid"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-5-pi",
      "modelName": "grok-4.5",
      "harness": "pi",
      "task": "q6_myeloid_subset_restricted_de",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1418.440208,
      "cost": 0.3426,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "myeloid_subset_with_strongest_change_gate",
            "role": "gate",
            "field": "$.myeloid_subset_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "subset_gamma"
            ]
          },
          {
            "name": "gene_with_strongest_change_gate",
            "role": "gate",
            "field": "$.gene_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "AXL",
              "PROS1"
            ]
          },
          {
            "name": "direction_of_change_gate",
            "role": "gate",
            "field": "$.direction_of_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "down_in_covid"
            ]
          },
          {
            "name": "biological_function_interpretation_gate",
            "role": "gate",
            "field": "$.biological_function_interpretation",
            "kind": "value",
            "op": "in",
            "expected": [
              "efferocytosis",
              "phagocytosis"
            ]
          },
          {
            "name": "myeloid_compartment_abundance_gate",
            "role": "gate",
            "field": "$.myeloid_compartment_abundance",
            "kind": "value",
            "op": "in",
            "expected": [
              "expands_in_covid"
            ]
          },
          {
            "name": "myeloid_expansion_localization_gate",
            "role": "gate",
            "field": "$.myeloid_expansion_localization",
            "kind": "value",
            "op": "in",
            "expected": [
              "recruited_driven_resident_dysfunction"
            ]
          },
          {
            "name": "dominant_signature_gate",
            "role": "gate",
            "field": "$.dominant_signature",
            "kind": "value",
            "op": "in",
            "expected": [
              "technical_or_agonal_artifact"
            ]
          },
          {
            "name": "dominant_signature_direction_gate",
            "role": "gate",
            "field": "$.dominant_signature_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_in_covid"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-5-pi",
      "modelName": "grok-4.5",
      "harness": "pi",
      "task": "snrnaseq_per_donor_composition_shifts_lethal_covid_lung",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 1567.143321,
      "cost": 0.149202,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dict_match",
            "role": "gate",
            "field": "cell_type_direction_map",
            "kind": "dict",
            "expected": {
              "Myeloid": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "Epithelia": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Fibroblasts": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "T_NK": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "B_plasma": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Endothelial": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Mast": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "APC": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Neuronal": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              }
            }
          },
          {
            "name": "over_merge_immune_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Immune",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_leukocyte_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Leukocyte",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_lymphoid_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Lymphoid",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_stromal_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Stromal",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-5-pi",
      "modelName": "grok-4.5",
      "harness": "pi",
      "task": "snrnaseq_per_donor_composition_shifts_lethal_covid_lung",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 1818.682882,
      "cost": 0.14574199999999998,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dict_match",
            "role": "gate",
            "field": "cell_type_direction_map",
            "kind": "dict",
            "expected": {
              "Myeloid": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "Epithelia": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Fibroblasts": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "T_NK": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "B_plasma": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Endothelial": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Mast": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "APC": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Neuronal": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              }
            }
          },
          {
            "name": "over_merge_immune_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Immune",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_leukocyte_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Leukocyte",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_lymphoid_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Lymphoid",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_stromal_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Stromal",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-5-pi",
      "modelName": "grok-4.5",
      "harness": "pi",
      "task": "snrnaseq_per_donor_composition_shifts_lethal_covid_lung",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1463.788642,
      "cost": 0.207692,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dict_match",
            "role": "gate",
            "field": "cell_type_direction_map",
            "kind": "dict",
            "expected": {
              "Myeloid": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "Epithelia": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Fibroblasts": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "T_NK": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "B_plasma": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Endothelial": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Mast": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "APC": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Neuronal": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              }
            }
          },
          {
            "name": "over_merge_immune_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Immune",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_leukocyte_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Leukocyte",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_lymphoid_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Lymphoid",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_stromal_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Stromal",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-5-pi",
      "modelName": "grok-4.5",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_enriched_apc_states_v2",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 2061.202098,
      "cost": 0.8684240000000001,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "top_3_cluster_enriched_apc_states",
            "kind": "list",
            "matchKey": "apc_state",
            "k": 3,
            "expected": [
              {
                "key": "plasmacytoid_DC",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "T_cell_stimulatory_role_supported",
                      "role_direction_unresolved"
                    ],
                    "canonical": "T_cell_stimulatory_role_supported"
                  }
                }
              },
              {
                "key": "C1qhigh_lipid_associated_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "C1qhigh_inflammatory_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "plasma_cell",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "cohort_significant_per_patient_inconsistent"
                    ],
                    "canonical": "cohort_significant_per_patient_inconsistent"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CD80_CD86": {
                "op": "in",
                "value": [
                  "mreg_DC",
                  "shared_across_APC_states"
                ]
              },
              "PD_L1_in_APC_clusters": {
                "op": "in",
                "value": [
                  "C1qhigh_macrophage_either_subtype",
                  "shared_across_APC_states",
                  "mreg_DC"
                ]
              },
              "type_I_IFN": {
                "op": "in",
                "value": [
                  "plasmacytoid_DC",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "cf_mreg_dc_claimed_enriched",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "mreg_DC",
              "mature_DC_enriched_in_immunoregulatory_molecules"
            ]
          },
          {
            "name": "cf_framework_offcompartment_or_generic_substitution",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "cDC1",
              "Maier_mregDC",
              "Cheng_C1QC_TAM",
              "Cheng_SPP1_TAM",
              "Cheng_FOLR2_TAM",
              "Villani_pDC",
              "Zhang_inflammatory_monocyte",
              "LAMP3_pos_DC",
              "myeloid_derived_suppressor_cell",
              "MDSC",
              "tumor_associated_neutrophil",
              "mast_cell",
              "APC",
              "antigen_presenting_cell",
              "macrophage",
              "dendritic_cell",
              "tumor_associated_macrophage",
              "TAM"
            ]
          },
          {
            "name": "cf_direction_commit_on_unresolvable_slot",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_not_resolvable_hedge",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].functional_attribution_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_type_I_IFN_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.type_I_IFN",
            "kind": "value",
            "op": "in",
            "expected": [
              "mreg_DC",
              "C1qhigh_macrophage_either_subtype"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-5-pi",
      "modelName": "grok-4.5",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_enriched_apc_states_v2",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 2157.896117,
      "cost": 1.042804,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "top_3_cluster_enriched_apc_states",
            "kind": "list",
            "matchKey": "apc_state",
            "k": 3,
            "expected": [
              {
                "key": "plasmacytoid_DC",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "T_cell_stimulatory_role_supported",
                      "role_direction_unresolved"
                    ],
                    "canonical": "T_cell_stimulatory_role_supported"
                  }
                }
              },
              {
                "key": "C1qhigh_lipid_associated_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "C1qhigh_inflammatory_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "plasma_cell",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "cohort_significant_per_patient_inconsistent"
                    ],
                    "canonical": "cohort_significant_per_patient_inconsistent"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CD80_CD86": {
                "op": "in",
                "value": [
                  "mreg_DC",
                  "shared_across_APC_states"
                ]
              },
              "PD_L1_in_APC_clusters": {
                "op": "in",
                "value": [
                  "C1qhigh_macrophage_either_subtype",
                  "shared_across_APC_states",
                  "mreg_DC"
                ]
              },
              "type_I_IFN": {
                "op": "in",
                "value": [
                  "plasmacytoid_DC",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "cf_mreg_dc_claimed_enriched",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "mreg_DC",
              "mature_DC_enriched_in_immunoregulatory_molecules"
            ]
          },
          {
            "name": "cf_framework_offcompartment_or_generic_substitution",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "cDC1",
              "Maier_mregDC",
              "Cheng_C1QC_TAM",
              "Cheng_SPP1_TAM",
              "Cheng_FOLR2_TAM",
              "Villani_pDC",
              "Zhang_inflammatory_monocyte",
              "LAMP3_pos_DC",
              "myeloid_derived_suppressor_cell",
              "MDSC",
              "tumor_associated_neutrophil",
              "mast_cell",
              "APC",
              "antigen_presenting_cell",
              "macrophage",
              "dendritic_cell",
              "tumor_associated_macrophage",
              "TAM"
            ]
          },
          {
            "name": "cf_direction_commit_on_unresolvable_slot",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_not_resolvable_hedge",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].functional_attribution_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_type_I_IFN_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.type_I_IFN",
            "kind": "value",
            "op": "in",
            "expected": [
              "mreg_DC",
              "C1qhigh_macrophage_either_subtype"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-5-pi",
      "modelName": "grok-4.5",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_enriched_apc_states_v2",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1233.425428,
      "cost": 0.553118,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "top_3_cluster_enriched_apc_states",
            "kind": "list",
            "matchKey": "apc_state",
            "k": 3,
            "expected": [
              {
                "key": "plasmacytoid_DC",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "T_cell_stimulatory_role_supported",
                      "role_direction_unresolved"
                    ],
                    "canonical": "T_cell_stimulatory_role_supported"
                  }
                }
              },
              {
                "key": "C1qhigh_lipid_associated_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "C1qhigh_inflammatory_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "plasma_cell",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "cohort_significant_per_patient_inconsistent"
                    ],
                    "canonical": "cohort_significant_per_patient_inconsistent"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CD80_CD86": {
                "op": "in",
                "value": [
                  "mreg_DC",
                  "shared_across_APC_states"
                ]
              },
              "PD_L1_in_APC_clusters": {
                "op": "in",
                "value": [
                  "C1qhigh_macrophage_either_subtype",
                  "shared_across_APC_states",
                  "mreg_DC"
                ]
              },
              "type_I_IFN": {
                "op": "in",
                "value": [
                  "plasmacytoid_DC",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "cf_mreg_dc_claimed_enriched",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "mreg_DC",
              "mature_DC_enriched_in_immunoregulatory_molecules"
            ]
          },
          {
            "name": "cf_framework_offcompartment_or_generic_substitution",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "cDC1",
              "Maier_mregDC",
              "Cheng_C1QC_TAM",
              "Cheng_SPP1_TAM",
              "Cheng_FOLR2_TAM",
              "Villani_pDC",
              "Zhang_inflammatory_monocyte",
              "LAMP3_pos_DC",
              "myeloid_derived_suppressor_cell",
              "MDSC",
              "tumor_associated_neutrophil",
              "mast_cell",
              "APC",
              "antigen_presenting_cell",
              "macrophage",
              "dendritic_cell",
              "tumor_associated_macrophage",
              "TAM"
            ]
          },
          {
            "name": "cf_direction_commit_on_unresolvable_slot",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_not_resolvable_hedge",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].functional_attribution_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_type_I_IFN_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.type_I_IFN",
            "kind": "value",
            "op": "in",
            "expected": [
              "mreg_DC",
              "C1qhigh_macrophage_either_subtype"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-5-pi",
      "modelName": "grok-4.5",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_tumor_state_causation_v5",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 1623.760542,
      "cost": 0.78935,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "enriched_tumor_states",
            "kind": "list",
            "matchKey": "tumor_state",
            "k": 3,
            "expected": [
              {
                "key": "immune_response",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "direction_unresolved_cross_sectional"
                    ],
                    "canonical": "direction_unresolved_cross_sectional"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "IFN_response_axis",
                      "broad_antigen_presentation_MHC"
                    ],
                    "canonical": "IFN_response_axis"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "enrichment_diminished_by_decontamination",
                      "threshold_sensitive"
                    ],
                    "canonical": "enrichment_diminished_by_decontamination"
                  }
                }
              },
              {
                "key": "stress_hypoxia",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "data_supports_cell_intrinsic_cause"
                    ],
                    "canonical": "data_supports_cell_intrinsic_cause"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "HIF_hypoxia_program_specific"
                    ],
                    "canonical": "HIF_hypoxia_program_specific"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "robust_across_methods",
                      "robust_after_decontamination"
                    ],
                    "canonical": "robust_across_methods"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CXCL9_CXCL10": {
                "op": "in",
                "value": [
                  "immune_response_tumor_state"
                ]
              },
              "CCL5": {
                "op": "in",
                "value": [
                  "shared_across_tumor_states",
                  "partner_lymphocyte_derived",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "gate_conjugate_gate_confounder",
            "role": "gate",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "partner_lymphocyte_capture"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "antigen_presentation_decomposition",
            "kind": "dict",
            "expected": {
              "MHC_class_I": {
                "op": "in",
                "value": [
                  "co_regulated_with_dominant_program",
                  "tumor_intrinsic_enriched_independent"
                ]
              },
              "MHC_class_II": {
                "op": "in",
                "value": [
                  "partner_cell_attributable",
                  "flat_not_enriched",
                  "method_sensitive_no_robust_call",
                  "co_regulated_with_dominant_program"
                ]
              }
            }
          },
          {
            "name": "cf_non_robust_state_claimed_enriched",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "transitory_melanocytic",
              "intermediate_melanocytic",
              "melanocytic",
              "MITF_high",
              "neural_crest_like",
              "mitotic",
              "stress_p53",
              "undifferentiated",
              "dedifferentiated",
              "mesenchymal_tumor",
              "AXL_high_tumor",
              "invasive_tumor",
              "senescence_like",
              "EMT_transition",
              "cancer_stem_cell_like",
              "oxidative_stress_tumor"
            ]
          },
          {
            "name": "cf_framework_generic_or_offtissue_substitution",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "Tsoi_undifferentiated",
              "Tsoi_neural_crest_like",
              "Tsoi_transitory",
              "Tsoi_melanocytic",
              "Rambow_NCSC",
              "Wouters_mesenchymal",
              "Pozniak_TCF4_dependent",
              "tumor_cell",
              "melanoma_cell",
              "malignant_cell",
              "tumor_subpopulation",
              "keratinocyte",
              "hepatocyte",
              "lung_adenocarcinoma_cell",
              "colon_adenocarcinoma_cell",
              "Tirosh_cell_cycle_program",
              "Jerby_Arnon_program",
              "glioblastoma_cell",
              "breast_carcinoma_cell"
            ]
          },
          {
            "name": "cf_immune_response_direction_committed",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_immune_robustness_recall",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_ambiguous_hedge",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].causation_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_stress_generic_stress_response_evidence",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_conjugate_confounder_contamination_denied",
            "role": "hard_fail",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected"
            ]
          },
          {
            "name": "cf_ccl5_recall_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CCL5",
            "kind": "avoid",
            "expected": [
              "immune_response_tumor_state"
            ]
          },
          {
            "name": "cf_cxcl9_cxcl10_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CXCL9_CXCL10",
            "kind": "avoid",
            "expected": [
              "stress_hypoxia_tumor_state"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-5-pi",
      "modelName": "grok-4.5",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_tumor_state_causation_v5",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 2831.790598,
      "cost": 0.661134,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "enriched_tumor_states",
            "kind": "list",
            "matchKey": "tumor_state",
            "k": 3,
            "expected": [
              {
                "key": "immune_response",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "direction_unresolved_cross_sectional"
                    ],
                    "canonical": "direction_unresolved_cross_sectional"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "IFN_response_axis",
                      "broad_antigen_presentation_MHC"
                    ],
                    "canonical": "IFN_response_axis"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "enrichment_diminished_by_decontamination",
                      "threshold_sensitive"
                    ],
                    "canonical": "enrichment_diminished_by_decontamination"
                  }
                }
              },
              {
                "key": "stress_hypoxia",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "data_supports_cell_intrinsic_cause"
                    ],
                    "canonical": "data_supports_cell_intrinsic_cause"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "HIF_hypoxia_program_specific"
                    ],
                    "canonical": "HIF_hypoxia_program_specific"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "robust_across_methods",
                      "robust_after_decontamination"
                    ],
                    "canonical": "robust_across_methods"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CXCL9_CXCL10": {
                "op": "in",
                "value": [
                  "immune_response_tumor_state"
                ]
              },
              "CCL5": {
                "op": "in",
                "value": [
                  "shared_across_tumor_states",
                  "partner_lymphocyte_derived",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "gate_conjugate_gate_confounder",
            "role": "gate",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "partner_lymphocyte_capture"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "antigen_presentation_decomposition",
            "kind": "dict",
            "expected": {
              "MHC_class_I": {
                "op": "in",
                "value": [
                  "co_regulated_with_dominant_program",
                  "tumor_intrinsic_enriched_independent"
                ]
              },
              "MHC_class_II": {
                "op": "in",
                "value": [
                  "partner_cell_attributable",
                  "flat_not_enriched",
                  "method_sensitive_no_robust_call",
                  "co_regulated_with_dominant_program"
                ]
              }
            }
          },
          {
            "name": "cf_non_robust_state_claimed_enriched",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "transitory_melanocytic",
              "intermediate_melanocytic",
              "melanocytic",
              "MITF_high",
              "neural_crest_like",
              "mitotic",
              "stress_p53",
              "undifferentiated",
              "dedifferentiated",
              "mesenchymal_tumor",
              "AXL_high_tumor",
              "invasive_tumor",
              "senescence_like",
              "EMT_transition",
              "cancer_stem_cell_like",
              "oxidative_stress_tumor"
            ]
          },
          {
            "name": "cf_framework_generic_or_offtissue_substitution",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "Tsoi_undifferentiated",
              "Tsoi_neural_crest_like",
              "Tsoi_transitory",
              "Tsoi_melanocytic",
              "Rambow_NCSC",
              "Wouters_mesenchymal",
              "Pozniak_TCF4_dependent",
              "tumor_cell",
              "melanoma_cell",
              "malignant_cell",
              "tumor_subpopulation",
              "keratinocyte",
              "hepatocyte",
              "lung_adenocarcinoma_cell",
              "colon_adenocarcinoma_cell",
              "Tirosh_cell_cycle_program",
              "Jerby_Arnon_program",
              "glioblastoma_cell",
              "breast_carcinoma_cell"
            ]
          },
          {
            "name": "cf_immune_response_direction_committed",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_immune_robustness_recall",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_ambiguous_hedge",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].causation_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_stress_generic_stress_response_evidence",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_conjugate_confounder_contamination_denied",
            "role": "hard_fail",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected"
            ]
          },
          {
            "name": "cf_ccl5_recall_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CCL5",
            "kind": "avoid",
            "expected": [
              "immune_response_tumor_state"
            ]
          },
          {
            "name": "cf_cxcl9_cxcl10_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CXCL9_CXCL10",
            "kind": "avoid",
            "expected": [
              "stress_hypoxia_tumor_state"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-5-pi",
      "modelName": "grok-4.5",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_tumor_state_causation_v5",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 896.601656,
      "cost": 0.507856,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "enriched_tumor_states",
            "kind": "list",
            "matchKey": "tumor_state",
            "k": 3,
            "expected": [
              {
                "key": "immune_response",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "direction_unresolved_cross_sectional"
                    ],
                    "canonical": "direction_unresolved_cross_sectional"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "IFN_response_axis",
                      "broad_antigen_presentation_MHC"
                    ],
                    "canonical": "IFN_response_axis"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "enrichment_diminished_by_decontamination",
                      "threshold_sensitive"
                    ],
                    "canonical": "enrichment_diminished_by_decontamination"
                  }
                }
              },
              {
                "key": "stress_hypoxia",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "data_supports_cell_intrinsic_cause"
                    ],
                    "canonical": "data_supports_cell_intrinsic_cause"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "HIF_hypoxia_program_specific"
                    ],
                    "canonical": "HIF_hypoxia_program_specific"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "robust_across_methods",
                      "robust_after_decontamination"
                    ],
                    "canonical": "robust_across_methods"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CXCL9_CXCL10": {
                "op": "in",
                "value": [
                  "immune_response_tumor_state"
                ]
              },
              "CCL5": {
                "op": "in",
                "value": [
                  "shared_across_tumor_states",
                  "partner_lymphocyte_derived",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "gate_conjugate_gate_confounder",
            "role": "gate",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "partner_lymphocyte_capture"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "antigen_presentation_decomposition",
            "kind": "dict",
            "expected": {
              "MHC_class_I": {
                "op": "in",
                "value": [
                  "co_regulated_with_dominant_program",
                  "tumor_intrinsic_enriched_independent"
                ]
              },
              "MHC_class_II": {
                "op": "in",
                "value": [
                  "partner_cell_attributable",
                  "flat_not_enriched",
                  "method_sensitive_no_robust_call",
                  "co_regulated_with_dominant_program"
                ]
              }
            }
          },
          {
            "name": "cf_non_robust_state_claimed_enriched",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "transitory_melanocytic",
              "intermediate_melanocytic",
              "melanocytic",
              "MITF_high",
              "neural_crest_like",
              "mitotic",
              "stress_p53",
              "undifferentiated",
              "dedifferentiated",
              "mesenchymal_tumor",
              "AXL_high_tumor",
              "invasive_tumor",
              "senescence_like",
              "EMT_transition",
              "cancer_stem_cell_like",
              "oxidative_stress_tumor"
            ]
          },
          {
            "name": "cf_framework_generic_or_offtissue_substitution",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "Tsoi_undifferentiated",
              "Tsoi_neural_crest_like",
              "Tsoi_transitory",
              "Tsoi_melanocytic",
              "Rambow_NCSC",
              "Wouters_mesenchymal",
              "Pozniak_TCF4_dependent",
              "tumor_cell",
              "melanoma_cell",
              "malignant_cell",
              "tumor_subpopulation",
              "keratinocyte",
              "hepatocyte",
              "lung_adenocarcinoma_cell",
              "colon_adenocarcinoma_cell",
              "Tirosh_cell_cycle_program",
              "Jerby_Arnon_program",
              "glioblastoma_cell",
              "breast_carcinoma_cell"
            ]
          },
          {
            "name": "cf_immune_response_direction_committed",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_immune_robustness_recall",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_ambiguous_hedge",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].causation_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_stress_generic_stress_response_evidence",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_conjugate_confounder_contamination_denied",
            "role": "hard_fail",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected"
            ]
          },
          {
            "name": "cf_ccl5_recall_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CCL5",
            "kind": "avoid",
            "expected": [
              "immune_response_tumor_state"
            ]
          },
          {
            "name": "cf_cxcl9_cxcl10_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CXCL9_CXCL10",
            "kind": "avoid",
            "expected": [
              "stress_hypoxia_tumor_state"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-5-pi",
      "modelName": "grok-4.5",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_vs_cd39_tumor_reactivity_dominance",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 1489.335235,
      "cost": 0.29386999999999996,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "population_findings",
            "kind": "list",
            "matchKey": "capture_gate",
            "k": 4,
            "expected": [
              {
                "key": "tumor_cell_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "apc_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "combined_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "unselected_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "naive",
                      "central_memory",
                      "effector_memory"
                    ],
                    "canonical": "naive"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "low"
                    ],
                    "canonical": "low"
                  },
                  "purity_caveat": {
                    "op": "equals",
                    "values": [
                      "pure_T_cell"
                    ],
                    "canonical": "pure_T_cell"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "no_reactivity_evidence"
                    ],
                    "canonical": "no_reactivity_evidence"
                  }
                }
              },
              {
                "key": "marker_sorted_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "equals",
                    "values": [
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "terminal_exhausted_LAG3_high"
                  },
                  "clonal_expansion_level": {
                    "op": "in",
                    "values": [
                      "high",
                      "intermediate"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "marker_on_non_T_lineages",
                      "single_cell_gating_loses_target"
                    ],
                    "canonical": "marker_on_non_T_lineages"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "marker_or_exhaustion_defined_only"
                    ],
                    "canonical": "marker_or_exhaustion_defined_only"
                  }
                }
              }
            ]
          },
          {
            "name": "best_capture_approach_gate",
            "role": "gate",
            "field": "best_capture_approach",
            "kind": "value",
            "expected": "heterotypic_cell_conjugate"
          },
          {
            "name": "best_selection_marker_gate",
            "role": "gate",
            "field": "best_selection_marker",
            "kind": "value",
            "expected": "none"
          },
          {
            "name": "reactivity_criterion_gate",
            "role": "gate",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "clonal_expansion_and_physical_engagement"
          },
          {
            "name": "cross_patient_robustness_gate",
            "role": "gate",
            "field": "cross_patient_robustness",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_across_patients",
              "marginal_in_some_patients"
            ]
          },
          {
            "name": "marker_selection_consequence_gate",
            "role": "gate",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "discards_clonally_expanded_reactive_cells"
          },
          {
            "name": "superiority_claim_basis_gate",
            "role": "gate",
            "field": "superiority_claim_basis",
            "kind": "value",
            "expected": "requires_functional_reactivity_assay"
          },
          {
            "name": "conjugate_enriched_state_gate",
            "role": "gate",
            "field": "conjugate_enriched_state",
            "kind": "value",
            "expected": "progenitor_exhausted_TCF7_pos_stem_like"
          },
          {
            "name": "cf_circular_exhaustion_criterion",
            "role": "hard_fail",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "exhaustion_signature_intensity"
          },
          {
            "name": "cf_marker_sort_prior",
            "role": "hard_fail",
            "field": "best_selection_marker",
            "kind": "value",
            "op": "in",
            "expected": [
              "CD39",
              "PD1",
              "PD1_high",
              "TIM3",
              "PD1_CD39",
              "CD39_CD103"
            ]
          },
          {
            "name": "cf_single_cell_capture_prior",
            "role": "hard_fail",
            "field": "best_capture_approach",
            "kind": "value",
            "op": "in",
            "expected": [
              "dissociated_single_cell",
              "dissociated_bulk_population"
            ]
          },
          {
            "name": "cf_off_data_capture_gate",
            "role": "hard_fail",
            "field": "$.population_findings[*].capture_gate",
            "kind": "avoid",
            "expected": [
              "dissociated_bulk",
              "matched_peripheral_blood",
              "matched_lymph_node",
              "post_hoc_expanded_clonotype_group"
            ]
          },
          {
            "name": "cf_conjugate_mischaracterized",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_marker_singlet_called_engaged",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "contains",
            "matchKey": "capture_gate",
            "entity": "marker_sorted_singlet",
            "valueField": "reactivity_evidence",
            "expected": [
              "engagement_and_clonal_expansion"
            ]
          },
          {
            "name": "cf_no_engaged_reactive_population",
            "role": "hard_fail",
            "field": "$.population_findings[*].reactivity_evidence",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_marker_captures_all_reactive",
            "role": "hard_fail",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "captures_full_reactive_repertoire"
          },
          {
            "name": "cf_superiority_overclaimed_from_phenotype",
            "role": "hard_fail",
            "field": "superiority_claim_basis",
            "kind": "value",
            "op": "in",
            "expected": [
              "established_by_scrna_phenotype",
              "established_by_reactivity_proxy_scores"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-5-pi",
      "modelName": "grok-4.5",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_vs_cd39_tumor_reactivity_dominance",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 1076.578559,
      "cost": 0.322324,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "population_findings",
            "kind": "list",
            "matchKey": "capture_gate",
            "k": 4,
            "expected": [
              {
                "key": "tumor_cell_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "apc_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "combined_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "unselected_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "naive",
                      "central_memory",
                      "effector_memory"
                    ],
                    "canonical": "naive"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "low"
                    ],
                    "canonical": "low"
                  },
                  "purity_caveat": {
                    "op": "equals",
                    "values": [
                      "pure_T_cell"
                    ],
                    "canonical": "pure_T_cell"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "no_reactivity_evidence"
                    ],
                    "canonical": "no_reactivity_evidence"
                  }
                }
              },
              {
                "key": "marker_sorted_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "equals",
                    "values": [
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "terminal_exhausted_LAG3_high"
                  },
                  "clonal_expansion_level": {
                    "op": "in",
                    "values": [
                      "high",
                      "intermediate"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "marker_on_non_T_lineages",
                      "single_cell_gating_loses_target"
                    ],
                    "canonical": "marker_on_non_T_lineages"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "marker_or_exhaustion_defined_only"
                    ],
                    "canonical": "marker_or_exhaustion_defined_only"
                  }
                }
              }
            ]
          },
          {
            "name": "best_capture_approach_gate",
            "role": "gate",
            "field": "best_capture_approach",
            "kind": "value",
            "expected": "heterotypic_cell_conjugate"
          },
          {
            "name": "best_selection_marker_gate",
            "role": "gate",
            "field": "best_selection_marker",
            "kind": "value",
            "expected": "none"
          },
          {
            "name": "reactivity_criterion_gate",
            "role": "gate",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "clonal_expansion_and_physical_engagement"
          },
          {
            "name": "cross_patient_robustness_gate",
            "role": "gate",
            "field": "cross_patient_robustness",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_across_patients",
              "marginal_in_some_patients"
            ]
          },
          {
            "name": "marker_selection_consequence_gate",
            "role": "gate",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "discards_clonally_expanded_reactive_cells"
          },
          {
            "name": "superiority_claim_basis_gate",
            "role": "gate",
            "field": "superiority_claim_basis",
            "kind": "value",
            "expected": "requires_functional_reactivity_assay"
          },
          {
            "name": "conjugate_enriched_state_gate",
            "role": "gate",
            "field": "conjugate_enriched_state",
            "kind": "value",
            "expected": "progenitor_exhausted_TCF7_pos_stem_like"
          },
          {
            "name": "cf_circular_exhaustion_criterion",
            "role": "hard_fail",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "exhaustion_signature_intensity"
          },
          {
            "name": "cf_marker_sort_prior",
            "role": "hard_fail",
            "field": "best_selection_marker",
            "kind": "value",
            "op": "in",
            "expected": [
              "CD39",
              "PD1",
              "PD1_high",
              "TIM3",
              "PD1_CD39",
              "CD39_CD103"
            ]
          },
          {
            "name": "cf_single_cell_capture_prior",
            "role": "hard_fail",
            "field": "best_capture_approach",
            "kind": "value",
            "op": "in",
            "expected": [
              "dissociated_single_cell",
              "dissociated_bulk_population"
            ]
          },
          {
            "name": "cf_off_data_capture_gate",
            "role": "hard_fail",
            "field": "$.population_findings[*].capture_gate",
            "kind": "avoid",
            "expected": [
              "dissociated_bulk",
              "matched_peripheral_blood",
              "matched_lymph_node",
              "post_hoc_expanded_clonotype_group"
            ]
          },
          {
            "name": "cf_conjugate_mischaracterized",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_marker_singlet_called_engaged",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "contains",
            "matchKey": "capture_gate",
            "entity": "marker_sorted_singlet",
            "valueField": "reactivity_evidence",
            "expected": [
              "engagement_and_clonal_expansion"
            ]
          },
          {
            "name": "cf_no_engaged_reactive_population",
            "role": "hard_fail",
            "field": "$.population_findings[*].reactivity_evidence",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_marker_captures_all_reactive",
            "role": "hard_fail",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "captures_full_reactive_repertoire"
          },
          {
            "name": "cf_superiority_overclaimed_from_phenotype",
            "role": "hard_fail",
            "field": "superiority_claim_basis",
            "kind": "value",
            "op": "in",
            "expected": [
              "established_by_scrna_phenotype",
              "established_by_reactivity_proxy_scores"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-5-pi",
      "modelName": "grok-4.5",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_vs_cd39_tumor_reactivity_dominance",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 498.738363,
      "cost": 0.308732,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "population_findings",
            "kind": "list",
            "matchKey": "capture_gate",
            "k": 4,
            "expected": [
              {
                "key": "tumor_cell_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "apc_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "combined_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "unselected_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "naive",
                      "central_memory",
                      "effector_memory"
                    ],
                    "canonical": "naive"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "low"
                    ],
                    "canonical": "low"
                  },
                  "purity_caveat": {
                    "op": "equals",
                    "values": [
                      "pure_T_cell"
                    ],
                    "canonical": "pure_T_cell"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "no_reactivity_evidence"
                    ],
                    "canonical": "no_reactivity_evidence"
                  }
                }
              },
              {
                "key": "marker_sorted_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "equals",
                    "values": [
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "terminal_exhausted_LAG3_high"
                  },
                  "clonal_expansion_level": {
                    "op": "in",
                    "values": [
                      "high",
                      "intermediate"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "marker_on_non_T_lineages",
                      "single_cell_gating_loses_target"
                    ],
                    "canonical": "marker_on_non_T_lineages"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "marker_or_exhaustion_defined_only"
                    ],
                    "canonical": "marker_or_exhaustion_defined_only"
                  }
                }
              }
            ]
          },
          {
            "name": "best_capture_approach_gate",
            "role": "gate",
            "field": "best_capture_approach",
            "kind": "value",
            "expected": "heterotypic_cell_conjugate"
          },
          {
            "name": "best_selection_marker_gate",
            "role": "gate",
            "field": "best_selection_marker",
            "kind": "value",
            "expected": "none"
          },
          {
            "name": "reactivity_criterion_gate",
            "role": "gate",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "clonal_expansion_and_physical_engagement"
          },
          {
            "name": "cross_patient_robustness_gate",
            "role": "gate",
            "field": "cross_patient_robustness",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_across_patients",
              "marginal_in_some_patients"
            ]
          },
          {
            "name": "marker_selection_consequence_gate",
            "role": "gate",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "discards_clonally_expanded_reactive_cells"
          },
          {
            "name": "superiority_claim_basis_gate",
            "role": "gate",
            "field": "superiority_claim_basis",
            "kind": "value",
            "expected": "requires_functional_reactivity_assay"
          },
          {
            "name": "conjugate_enriched_state_gate",
            "role": "gate",
            "field": "conjugate_enriched_state",
            "kind": "value",
            "expected": "progenitor_exhausted_TCF7_pos_stem_like"
          },
          {
            "name": "cf_circular_exhaustion_criterion",
            "role": "hard_fail",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "exhaustion_signature_intensity"
          },
          {
            "name": "cf_marker_sort_prior",
            "role": "hard_fail",
            "field": "best_selection_marker",
            "kind": "value",
            "op": "in",
            "expected": [
              "CD39",
              "PD1",
              "PD1_high",
              "TIM3",
              "PD1_CD39",
              "CD39_CD103"
            ]
          },
          {
            "name": "cf_single_cell_capture_prior",
            "role": "hard_fail",
            "field": "best_capture_approach",
            "kind": "value",
            "op": "in",
            "expected": [
              "dissociated_single_cell",
              "dissociated_bulk_population"
            ]
          },
          {
            "name": "cf_off_data_capture_gate",
            "role": "hard_fail",
            "field": "$.population_findings[*].capture_gate",
            "kind": "avoid",
            "expected": [
              "dissociated_bulk",
              "matched_peripheral_blood",
              "matched_lymph_node",
              "post_hoc_expanded_clonotype_group"
            ]
          },
          {
            "name": "cf_conjugate_mischaracterized",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_marker_singlet_called_engaged",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "contains",
            "matchKey": "capture_gate",
            "entity": "marker_sorted_singlet",
            "valueField": "reactivity_evidence",
            "expected": [
              "engagement_and_clonal_expansion"
            ]
          },
          {
            "name": "cf_no_engaged_reactive_population",
            "role": "hard_fail",
            "field": "$.population_findings[*].reactivity_evidence",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_marker_captures_all_reactive",
            "role": "hard_fail",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "captures_full_reactive_repertoire"
          },
          {
            "name": "cf_superiority_overclaimed_from_phenotype",
            "role": "hard_fail",
            "field": "superiority_claim_basis",
            "kind": "value",
            "op": "in",
            "expected": [
              "established_by_scrna_phenotype",
              "established_by_reactivity_proxy_scores"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-5-pi",
      "modelName": "grok-4.5",
      "harness": "pi",
      "task": "10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 1082.180132,
      "cost": 0.8415360000000002,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_singlet_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "TOXhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "LAG3high_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "GZMKhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "intermediate_exhausted_gzmk"
                    ],
                    "canonical": "intermediate_exhausted_gzmk"
                  }
                }
              },
              {
                "key": "TCF7_stem_like_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "stem_like_exhausted"
                    ],
                    "canonical": "stem_like_exhausted"
                  }
                }
              },
              {
                "key": "MKI67_pos_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              },
              {
                "key": "MKI67high_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_tumor_cluster_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "Tn",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "Tn_Tmem",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "early_Tem_GZMK",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              },
              {
                "key": "Tem_GZMH",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              }
            ]
          },
          {
            "name": "clonotype_expansion_direction_data_true",
            "role": "gate",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_substantially_more_expanded_than_cluster",
              "singlet_modestly_more_expanded_than_cluster"
            ]
          },
          {
            "name": "biological_inference_data_true",
            "role": "gate",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_antigen_experienced_cluster_bystander"
            ]
          },
          {
            "name": "singlet_top3_contains_naive_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Tn",
              "Tn_Tmem",
              "early_Tem_GZMK",
              "Tem_GZMH"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "TOXhigh_Tex",
              "LAG3high_Tex",
              "GZMKhigh_Tex",
              "TCF7_stem_like_Tex",
              "MKI67_pos_Tex_Tprol",
              "MKI67high_Tex_Tprol"
            ]
          },
          {
            "name": "singlet_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "TOXhigh_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "TCF7_stem_like_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "MKI67_state_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "Tn_or_Tn_Tmem_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_stemlike_precursor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_reported_canonical_expansion_direction",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_substantially_more_expanded_than_singlet",
              "cluster_modestly_more_expanded_than_singlet"
            ]
          },
          {
            "name": "cf_expansion_lazy_defer",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "expansion_pattern_not_detectable"
            ]
          },
          {
            "name": "cf_reported_canonical_expected_biology",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_antigen_experienced_singlet_bystander"
            ]
          },
          {
            "name": "cf_biological_inference_null_artifact_or_lineage",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "differences_reflect_dissociation_or_sorting_artifact",
              "cluster_and_singlet_capture_same_biology",
              "cluster_and_singlet_capture_distinct_cd8_lineages"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "biological_relations",
            "kind": "dict",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_greater_than",
                  "greater_than"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "less_than",
                  "much_less_than"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than",
                  "approximately_equal"
                ]
              },
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal"
                ]
              },
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal",
                  "greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_exhaustion_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_naive_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_proliferation_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_expansion_richness_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_multichain_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_intermediate_gzmk_overgeneralized",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_melanocytic_conjugate_overcall",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "values": [
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_relations_mass_hedge",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-5-pi",
      "modelName": "grok-4.5",
      "harness": "pi",
      "task": "10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 818.667985,
      "cost": 0.625628,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_singlet_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "TOXhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "LAG3high_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "GZMKhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "intermediate_exhausted_gzmk"
                    ],
                    "canonical": "intermediate_exhausted_gzmk"
                  }
                }
              },
              {
                "key": "TCF7_stem_like_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "stem_like_exhausted"
                    ],
                    "canonical": "stem_like_exhausted"
                  }
                }
              },
              {
                "key": "MKI67_pos_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              },
              {
                "key": "MKI67high_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_tumor_cluster_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "Tn",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "Tn_Tmem",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "early_Tem_GZMK",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              },
              {
                "key": "Tem_GZMH",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              }
            ]
          },
          {
            "name": "clonotype_expansion_direction_data_true",
            "role": "gate",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_substantially_more_expanded_than_cluster",
              "singlet_modestly_more_expanded_than_cluster"
            ]
          },
          {
            "name": "biological_inference_data_true",
            "role": "gate",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_antigen_experienced_cluster_bystander"
            ]
          },
          {
            "name": "singlet_top3_contains_naive_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Tn",
              "Tn_Tmem",
              "early_Tem_GZMK",
              "Tem_GZMH"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "TOXhigh_Tex",
              "LAG3high_Tex",
              "GZMKhigh_Tex",
              "TCF7_stem_like_Tex",
              "MKI67_pos_Tex_Tprol",
              "MKI67high_Tex_Tprol"
            ]
          },
          {
            "name": "singlet_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "TOXhigh_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "TCF7_stem_like_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "MKI67_state_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "Tn_or_Tn_Tmem_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_stemlike_precursor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_reported_canonical_expansion_direction",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_substantially_more_expanded_than_singlet",
              "cluster_modestly_more_expanded_than_singlet"
            ]
          },
          {
            "name": "cf_expansion_lazy_defer",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "expansion_pattern_not_detectable"
            ]
          },
          {
            "name": "cf_reported_canonical_expected_biology",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_antigen_experienced_singlet_bystander"
            ]
          },
          {
            "name": "cf_biological_inference_null_artifact_or_lineage",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "differences_reflect_dissociation_or_sorting_artifact",
              "cluster_and_singlet_capture_same_biology",
              "cluster_and_singlet_capture_distinct_cd8_lineages"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "biological_relations",
            "kind": "dict",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_greater_than",
                  "greater_than"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "less_than",
                  "much_less_than"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than",
                  "approximately_equal"
                ]
              },
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal"
                ]
              },
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal",
                  "greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_exhaustion_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_naive_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_proliferation_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_expansion_richness_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_multichain_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_intermediate_gzmk_overgeneralized",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_melanocytic_conjugate_overcall",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "values": [
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_relations_mass_hedge",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-5-pi",
      "modelName": "grok-4.5",
      "harness": "pi",
      "task": "10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1387.877251,
      "cost": 0.62766,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_singlet_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "TOXhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "LAG3high_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "GZMKhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "intermediate_exhausted_gzmk"
                    ],
                    "canonical": "intermediate_exhausted_gzmk"
                  }
                }
              },
              {
                "key": "TCF7_stem_like_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "stem_like_exhausted"
                    ],
                    "canonical": "stem_like_exhausted"
                  }
                }
              },
              {
                "key": "MKI67_pos_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              },
              {
                "key": "MKI67high_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_tumor_cluster_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "Tn",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "Tn_Tmem",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "early_Tem_GZMK",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              },
              {
                "key": "Tem_GZMH",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              }
            ]
          },
          {
            "name": "clonotype_expansion_direction_data_true",
            "role": "gate",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_substantially_more_expanded_than_cluster",
              "singlet_modestly_more_expanded_than_cluster"
            ]
          },
          {
            "name": "biological_inference_data_true",
            "role": "gate",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_antigen_experienced_cluster_bystander"
            ]
          },
          {
            "name": "singlet_top3_contains_naive_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Tn",
              "Tn_Tmem",
              "early_Tem_GZMK",
              "Tem_GZMH"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "TOXhigh_Tex",
              "LAG3high_Tex",
              "GZMKhigh_Tex",
              "TCF7_stem_like_Tex",
              "MKI67_pos_Tex_Tprol",
              "MKI67high_Tex_Tprol"
            ]
          },
          {
            "name": "singlet_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "TOXhigh_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "TCF7_stem_like_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "MKI67_state_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "Tn_or_Tn_Tmem_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_stemlike_precursor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_reported_canonical_expansion_direction",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_substantially_more_expanded_than_singlet",
              "cluster_modestly_more_expanded_than_singlet"
            ]
          },
          {
            "name": "cf_expansion_lazy_defer",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "expansion_pattern_not_detectable"
            ]
          },
          {
            "name": "cf_reported_canonical_expected_biology",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_antigen_experienced_singlet_bystander"
            ]
          },
          {
            "name": "cf_biological_inference_null_artifact_or_lineage",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "differences_reflect_dissociation_or_sorting_artifact",
              "cluster_and_singlet_capture_same_biology",
              "cluster_and_singlet_capture_distinct_cd8_lineages"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "biological_relations",
            "kind": "dict",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_greater_than",
                  "greater_than"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "less_than",
                  "much_less_than"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than",
                  "approximately_equal"
                ]
              },
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal"
                ]
              },
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal",
                  "greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_exhaustion_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_naive_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_proliferation_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_expansion_richness_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_multichain_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_intermediate_gzmk_overgeneralized",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_melanocytic_conjugate_overcall",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "values": [
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_relations_mass_hedge",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-5-pi",
      "modelName": "grok-4.5",
      "harness": "pi",
      "task": "10x_5p_melanoma_tcr_matched_apc_vs_tumor_exhaustion",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 1,
      "passed": true,
      "score": null,
      "durationS": 511.495185,
      "cost": 0.346606,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "state_discriminating_genes",
            "kind": "list",
            "matchKey": "gene",
            "k": 3,
            "expected": [
              {
                "key": "CTLA4",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TOX",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "ENTPD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "HAVCR2",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TIGIT",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "PDCD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "difference_attributable_to",
            "role": "gate",
            "field": "difference_attributable_to",
            "kind": "value",
            "expected": "covaries_with_apc_conjugate_state"
          },
          {
            "name": "causal_status",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "direction_unresolved_selection_confound"
          },
          {
            "name": "functional_interpretation",
            "role": "gate",
            "field": "functional_interpretation",
            "kind": "value",
            "expected": "costimulation_signature_function_unresolved"
          },
          {
            "name": "cf_contamination_gene_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "MLANA",
              "PMEL",
              "DCT",
              "TYR",
              "TYRP1",
              "SOX10",
              "MITF",
              "MAGEA4"
            ]
          },
          {
            "name": "cf_apc_partner_contamination_gene",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "HLA-DRA",
              "CD68",
              "LYZ",
              "CD14",
              "C1QA",
              "ITGAX",
              "CD1C"
            ]
          },
          {
            "name": "cf_ligand_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "CD274",
              "PDCD1LG2",
              "CD80",
              "CD86"
            ]
          },
          {
            "name": "cf_attribution_causal_or_baseline",
            "role": "hard_fail",
            "field": "$.difference_attributable_to",
            "kind": "avoid",
            "expected": [
              "apc_engagement",
              "tumor_cell_engagement",
              "clonotype_intrinsic_baseline"
            ]
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "engagement_induces_exhausted_state",
              "pre_exhausted_cells_select_into_apc_conjugates"
            ]
          },
          {
            "name": "cf_causal_fabrication",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "resolved_by_pseudotemporal_ordering",
              "resolved_by_perturbation_experiment"
            ]
          },
          {
            "name": "cf_functional_overclaim",
            "role": "hard_fail",
            "field": "$.functional_interpretation",
            "kind": "avoid",
            "expected": [
              "costimulation_licensed_reinvigoration_competent",
              "costimulation_licensed_checkpoint_responsive",
              "tumor_driven_terminal_exhaustion"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-5-pi",
      "modelName": "grok-4.5",
      "harness": "pi",
      "task": "10x_5p_melanoma_tcr_matched_apc_vs_tumor_exhaustion",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 505.823731,
      "cost": 0.297578,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "state_discriminating_genes",
            "kind": "list",
            "matchKey": "gene",
            "k": 3,
            "expected": [
              {
                "key": "CTLA4",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TOX",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "ENTPD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "HAVCR2",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TIGIT",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "PDCD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "difference_attributable_to",
            "role": "gate",
            "field": "difference_attributable_to",
            "kind": "value",
            "expected": "covaries_with_apc_conjugate_state"
          },
          {
            "name": "causal_status",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "direction_unresolved_selection_confound"
          },
          {
            "name": "functional_interpretation",
            "role": "gate",
            "field": "functional_interpretation",
            "kind": "value",
            "expected": "costimulation_signature_function_unresolved"
          },
          {
            "name": "cf_contamination_gene_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "MLANA",
              "PMEL",
              "DCT",
              "TYR",
              "TYRP1",
              "SOX10",
              "MITF",
              "MAGEA4"
            ]
          },
          {
            "name": "cf_apc_partner_contamination_gene",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "HLA-DRA",
              "CD68",
              "LYZ",
              "CD14",
              "C1QA",
              "ITGAX",
              "CD1C"
            ]
          },
          {
            "name": "cf_ligand_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "CD274",
              "PDCD1LG2",
              "CD80",
              "CD86"
            ]
          },
          {
            "name": "cf_attribution_causal_or_baseline",
            "role": "hard_fail",
            "field": "$.difference_attributable_to",
            "kind": "avoid",
            "expected": [
              "apc_engagement",
              "tumor_cell_engagement",
              "clonotype_intrinsic_baseline"
            ]
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "engagement_induces_exhausted_state",
              "pre_exhausted_cells_select_into_apc_conjugates"
            ]
          },
          {
            "name": "cf_causal_fabrication",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "resolved_by_pseudotemporal_ordering",
              "resolved_by_perturbation_experiment"
            ]
          },
          {
            "name": "cf_functional_overclaim",
            "role": "hard_fail",
            "field": "$.functional_interpretation",
            "kind": "avoid",
            "expected": [
              "costimulation_licensed_reinvigoration_competent",
              "costimulation_licensed_checkpoint_responsive",
              "tumor_driven_terminal_exhaustion"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-5-pi",
      "modelName": "grok-4.5",
      "harness": "pi",
      "task": "10x_5p_melanoma_tcr_matched_apc_vs_tumor_exhaustion",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 575.150711,
      "cost": 0.35646999999999995,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "state_discriminating_genes",
            "kind": "list",
            "matchKey": "gene",
            "k": 3,
            "expected": [
              {
                "key": "CTLA4",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TOX",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "ENTPD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "HAVCR2",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TIGIT",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "PDCD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "difference_attributable_to",
            "role": "gate",
            "field": "difference_attributable_to",
            "kind": "value",
            "expected": "covaries_with_apc_conjugate_state"
          },
          {
            "name": "causal_status",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "direction_unresolved_selection_confound"
          },
          {
            "name": "functional_interpretation",
            "role": "gate",
            "field": "functional_interpretation",
            "kind": "value",
            "expected": "costimulation_signature_function_unresolved"
          },
          {
            "name": "cf_contamination_gene_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "MLANA",
              "PMEL",
              "DCT",
              "TYR",
              "TYRP1",
              "SOX10",
              "MITF",
              "MAGEA4"
            ]
          },
          {
            "name": "cf_apc_partner_contamination_gene",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "HLA-DRA",
              "CD68",
              "LYZ",
              "CD14",
              "C1QA",
              "ITGAX",
              "CD1C"
            ]
          },
          {
            "name": "cf_ligand_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "CD274",
              "PDCD1LG2",
              "CD80",
              "CD86"
            ]
          },
          {
            "name": "cf_attribution_causal_or_baseline",
            "role": "hard_fail",
            "field": "$.difference_attributable_to",
            "kind": "avoid",
            "expected": [
              "apc_engagement",
              "tumor_cell_engagement",
              "clonotype_intrinsic_baseline"
            ]
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "engagement_induces_exhausted_state",
              "pre_exhausted_cells_select_into_apc_conjugates"
            ]
          },
          {
            "name": "cf_causal_fabrication",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "resolved_by_pseudotemporal_ordering",
              "resolved_by_perturbation_experiment"
            ]
          },
          {
            "name": "cf_functional_overclaim",
            "role": "hard_fail",
            "field": "$.functional_interpretation",
            "kind": "avoid",
            "expected": [
              "costimulation_licensed_reinvigoration_competent",
              "costimulation_licensed_checkpoint_responsive",
              "tumor_driven_terminal_exhaustion"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-5-pi",
      "modelName": "grok-4.5",
      "harness": "pi",
      "task": "green_2_tme_context_dependence",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 4970.06936,
      "cost": 0.969368,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "program_map",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "sig_includes_itgae",
            "role": "gate",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Itgae"
            ]
          },
          {
            "name": "declared_a_known_confounder",
            "role": "gate",
            "field": "$.excluded_confounders[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "cf_signature_gut_contaminant",
            "role": "hard_fail",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "generic_signature_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd103_program_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "<"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "per_model_regulatory_state",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "poised_unfired"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "late_activation"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "closing_committed"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_tuples",
            "kind": "list",
            "matchKey": "regulator",
            "k": 1,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "role": {
                    "op": "equals",
                    "values": [
                      "repressor"
                    ]
                  },
                  "gating_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  },
                  "evidence_basis": {
                    "op": "equals",
                    "values": [
                      "target_program_behavior"
                    ]
                  },
                  "expr_vs_activity": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "locus_differential_gate",
            "role": "gate",
            "field": "locus_differential",
            "kind": "value",
            "expected": "uniform_across_models"
          },
          {
            "name": "formation_limited_by_gate",
            "role": "gate",
            "field": "formation_limited_by",
            "kind": "value",
            "expected": "absent_induction"
          },
          {
            "name": "locus_competent_in_naive_gate",
            "role": "gate",
            "field": "locus_competent_in_naive",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "ready_silent_fraction_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 3,
            "expected": [
              {
                "key": "B16_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_KPC_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "activator_chromatin_footprint_gate",
            "role": "gate",
            "field": "activator_chromatin_footprint",
            "kind": "value",
            "expected": "inconsistent_across_formers"
          },
          {
            "name": "exhaustion_comparable_gate",
            "role": "gate",
            "field": "exhaustion_comparable_across_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_nondeterminant_gate",
            "role": "gate",
            "field": "exhaustion_explains_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_separable_program_gate",
            "role": "gate",
            "field": "exhaustion_separable_program",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "exhaustion_counterexample_gate",
            "role": "gate",
            "field": "exhaustion_counterexample_model",
            "kind": "value",
            "expected": "KPC_GP"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "exhaustion_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "cf_inconsistent_present_without_activator",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "program",
            "entity": "present",
            "valueField": "activator_state",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_no_block_but_absent",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "none",
            "valueField": "program",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_late_block_without_poised_locus",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "late_activation",
            "valueField": "locus_state",
            "expected": [
              "closing_committed",
              "active"
            ]
          },
          {
            "name": "cf_inconsistent_early_block_with_brake_released",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "early_release",
            "valueField": "brake_state",
            "expected": [
              "released"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-5-pi",
      "modelName": "grok-4.5",
      "harness": "pi",
      "task": "green_2_tme_context_dependence",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 4433.074058,
      "cost": 0.6276820000000001,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "program_map",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "sig_includes_itgae",
            "role": "gate",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Itgae"
            ]
          },
          {
            "name": "declared_a_known_confounder",
            "role": "gate",
            "field": "$.excluded_confounders[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "cf_signature_gut_contaminant",
            "role": "hard_fail",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "generic_signature_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd103_program_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "<"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "per_model_regulatory_state",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "poised_unfired"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "late_activation"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "closing_committed"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_tuples",
            "kind": "list",
            "matchKey": "regulator",
            "k": 1,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "role": {
                    "op": "equals",
                    "values": [
                      "repressor"
                    ]
                  },
                  "gating_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  },
                  "evidence_basis": {
                    "op": "equals",
                    "values": [
                      "target_program_behavior"
                    ]
                  },
                  "expr_vs_activity": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "locus_differential_gate",
            "role": "gate",
            "field": "locus_differential",
            "kind": "value",
            "expected": "uniform_across_models"
          },
          {
            "name": "formation_limited_by_gate",
            "role": "gate",
            "field": "formation_limited_by",
            "kind": "value",
            "expected": "absent_induction"
          },
          {
            "name": "locus_competent_in_naive_gate",
            "role": "gate",
            "field": "locus_competent_in_naive",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "ready_silent_fraction_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 3,
            "expected": [
              {
                "key": "B16_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_KPC_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "activator_chromatin_footprint_gate",
            "role": "gate",
            "field": "activator_chromatin_footprint",
            "kind": "value",
            "expected": "inconsistent_across_formers"
          },
          {
            "name": "exhaustion_comparable_gate",
            "role": "gate",
            "field": "exhaustion_comparable_across_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_nondeterminant_gate",
            "role": "gate",
            "field": "exhaustion_explains_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_separable_program_gate",
            "role": "gate",
            "field": "exhaustion_separable_program",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "exhaustion_counterexample_gate",
            "role": "gate",
            "field": "exhaustion_counterexample_model",
            "kind": "value",
            "expected": "KPC_GP"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "exhaustion_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "cf_inconsistent_present_without_activator",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "program",
            "entity": "present",
            "valueField": "activator_state",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_no_block_but_absent",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "none",
            "valueField": "program",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_late_block_without_poised_locus",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "late_activation",
            "valueField": "locus_state",
            "expected": [
              "closing_committed",
              "active"
            ]
          },
          {
            "name": "cf_inconsistent_early_block_with_brake_released",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "early_release",
            "valueField": "brake_state",
            "expected": [
              "released"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-5-pi",
      "modelName": "grok-4.5",
      "harness": "pi",
      "task": "green_2_tme_context_dependence",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 3913.063607,
      "cost": 0.8655980000000001,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "program_map",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "sig_includes_itgae",
            "role": "gate",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Itgae"
            ]
          },
          {
            "name": "declared_a_known_confounder",
            "role": "gate",
            "field": "$.excluded_confounders[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "cf_signature_gut_contaminant",
            "role": "hard_fail",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "generic_signature_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd103_program_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "<"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "per_model_regulatory_state",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "poised_unfired"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "late_activation"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "closing_committed"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_tuples",
            "kind": "list",
            "matchKey": "regulator",
            "k": 1,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "role": {
                    "op": "equals",
                    "values": [
                      "repressor"
                    ]
                  },
                  "gating_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  },
                  "evidence_basis": {
                    "op": "equals",
                    "values": [
                      "target_program_behavior"
                    ]
                  },
                  "expr_vs_activity": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "locus_differential_gate",
            "role": "gate",
            "field": "locus_differential",
            "kind": "value",
            "expected": "uniform_across_models"
          },
          {
            "name": "formation_limited_by_gate",
            "role": "gate",
            "field": "formation_limited_by",
            "kind": "value",
            "expected": "absent_induction"
          },
          {
            "name": "locus_competent_in_naive_gate",
            "role": "gate",
            "field": "locus_competent_in_naive",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "ready_silent_fraction_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 3,
            "expected": [
              {
                "key": "B16_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_KPC_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "activator_chromatin_footprint_gate",
            "role": "gate",
            "field": "activator_chromatin_footprint",
            "kind": "value",
            "expected": "inconsistent_across_formers"
          },
          {
            "name": "exhaustion_comparable_gate",
            "role": "gate",
            "field": "exhaustion_comparable_across_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_nondeterminant_gate",
            "role": "gate",
            "field": "exhaustion_explains_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_separable_program_gate",
            "role": "gate",
            "field": "exhaustion_separable_program",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "exhaustion_counterexample_gate",
            "role": "gate",
            "field": "exhaustion_counterexample_model",
            "kind": "value",
            "expected": "KPC_GP"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "exhaustion_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "cf_inconsistent_present_without_activator",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "program",
            "entity": "present",
            "valueField": "activator_state",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_no_block_but_absent",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "none",
            "valueField": "program",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_late_block_without_poised_locus",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "late_activation",
            "valueField": "locus_state",
            "expected": [
              "closing_committed",
              "active"
            ]
          },
          {
            "name": "cf_inconsistent_early_block_with_brake_released",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "early_release",
            "valueField": "brake_state",
            "expected": [
              "released"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-5-pi",
      "modelName": "grok-4.5",
      "harness": "pi",
      "task": "green_klf2_hidden_repressor_of_trm_like_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 4111.039414,
      "cost": 0.830644,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "regulator_gate",
            "role": "gate",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "KLF2",
              "Klf2"
            ]
          },
          {
            "name": "controlling_node_character_gate",
            "role": "gate",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "high_differential_expression"
          },
          {
            "name": "expression_vs_activity_gate",
            "role": "gate",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "expression_exceeds_activity"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "target_program_partition",
            "kind": "list",
            "matchKey": "gene",
            "k": 7,
            "expected": [
              {
                "key": "S1pr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "S1pr4",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Klrg1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Cx3cr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Itgae",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Hic1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "P2rx7",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              }
            ]
          },
          {
            "name": "causal_status_gate",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "association_only_sufficiency_requires_perturbation"
          },
          {
            "name": "residency_control_logic_gate",
            "role": "gate",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_release_necessary_not_sufficient_multi_regulon"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "trm_like_program_by_model",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_absent"
                    ],
                    "canonical": "niche_absent"
                  }
                }
              }
            ]
          },
          {
            "name": "cf_regulator_activator_or_wrong_node",
            "role": "hard_fail",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "RUNX3",
              "Runx3",
              "PRDM1",
              "Prdm1",
              "BLIMP1",
              "Blimp1",
              "BHLHE40",
              "Bhlhe40",
              "ZNF683",
              "Zfp683",
              "HOBIT",
              "Hobit",
              "AHR",
              "Ahr",
              "BATF",
              "Batf",
              "TOX",
              "Tox",
              "NR4A1",
              "Nr4a1",
              "NR4A2",
              "Nr4a2",
              "NR4A3",
              "Nr4a3",
              "TCF7",
              "Tcf7",
              "TCF1",
              "Tcf1",
              "BACH2",
              "Bach2",
              "TBX21",
              "Tbx21",
              "TBET",
              "Tbet",
              "FLI1",
              "Fli1",
              "NR3C1",
              "Nr3c1",
              "EOMES",
              "Eomes",
              "ZEB2",
              "Zeb2",
              "ID2",
              "Id2",
              "ID3",
              "Id3",
              "IRF4",
              "Irf4"
            ]
          },
          {
            "name": "cf_node_character_hidden_driver_recall",
            "role": "hard_fail",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "hidden_driver_activity_not_expression"
          },
          {
            "name": "cf_effect_relation_hidden_pattern",
            "role": "hard_fail",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "activity_exceeds_expression"
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "causal_status",
            "kind": "value",
            "op": "in",
            "expected": [
              "sufficiency_established",
              "necessity_established",
              "causal_established_in_dataset",
              "sufficiency_via_perturbation_in_dataset"
            ]
          },
          {
            "name": "cf_residency_logic_activator_prior",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "residency_activator_induction_primary"
          },
          {
            "name": "cf_residency_logic_single_regulon",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_controls_entire_residency_program"
          },
          {
            "name": "cf_decoy_called_target",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_distractor_model_reported",
            "role": "hard_fail",
            "field": "$.trm_like_program_by_model[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_chromatin_only_decoy",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-5-pi",
      "modelName": "grok-4.5",
      "harness": "pi",
      "task": "green_klf2_hidden_repressor_of_trm_like_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 7844.584875,
      "cost": 1.025456,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "regulator_gate",
            "role": "gate",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "KLF2",
              "Klf2"
            ]
          },
          {
            "name": "controlling_node_character_gate",
            "role": "gate",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "high_differential_expression"
          },
          {
            "name": "expression_vs_activity_gate",
            "role": "gate",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "expression_exceeds_activity"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "target_program_partition",
            "kind": "list",
            "matchKey": "gene",
            "k": 7,
            "expected": [
              {
                "key": "S1pr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "S1pr4",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Klrg1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Cx3cr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Itgae",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Hic1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "P2rx7",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              }
            ]
          },
          {
            "name": "causal_status_gate",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "association_only_sufficiency_requires_perturbation"
          },
          {
            "name": "residency_control_logic_gate",
            "role": "gate",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_release_necessary_not_sufficient_multi_regulon"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "trm_like_program_by_model",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_absent"
                    ],
                    "canonical": "niche_absent"
                  }
                }
              }
            ]
          },
          {
            "name": "cf_regulator_activator_or_wrong_node",
            "role": "hard_fail",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "RUNX3",
              "Runx3",
              "PRDM1",
              "Prdm1",
              "BLIMP1",
              "Blimp1",
              "BHLHE40",
              "Bhlhe40",
              "ZNF683",
              "Zfp683",
              "HOBIT",
              "Hobit",
              "AHR",
              "Ahr",
              "BATF",
              "Batf",
              "TOX",
              "Tox",
              "NR4A1",
              "Nr4a1",
              "NR4A2",
              "Nr4a2",
              "NR4A3",
              "Nr4a3",
              "TCF7",
              "Tcf7",
              "TCF1",
              "Tcf1",
              "BACH2",
              "Bach2",
              "TBX21",
              "Tbx21",
              "TBET",
              "Tbet",
              "FLI1",
              "Fli1",
              "NR3C1",
              "Nr3c1",
              "EOMES",
              "Eomes",
              "ZEB2",
              "Zeb2",
              "ID2",
              "Id2",
              "ID3",
              "Id3",
              "IRF4",
              "Irf4"
            ]
          },
          {
            "name": "cf_node_character_hidden_driver_recall",
            "role": "hard_fail",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "hidden_driver_activity_not_expression"
          },
          {
            "name": "cf_effect_relation_hidden_pattern",
            "role": "hard_fail",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "activity_exceeds_expression"
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "causal_status",
            "kind": "value",
            "op": "in",
            "expected": [
              "sufficiency_established",
              "necessity_established",
              "causal_established_in_dataset",
              "sufficiency_via_perturbation_in_dataset"
            ]
          },
          {
            "name": "cf_residency_logic_activator_prior",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "residency_activator_induction_primary"
          },
          {
            "name": "cf_residency_logic_single_regulon",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_controls_entire_residency_program"
          },
          {
            "name": "cf_decoy_called_target",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_distractor_model_reported",
            "role": "hard_fail",
            "field": "$.trm_like_program_by_model[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_chromatin_only_decoy",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-5-pi",
      "modelName": "grok-4.5",
      "harness": "pi",
      "task": "green_klf2_hidden_repressor_of_trm_like_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 3663.111467,
      "cost": 0.6714259999999999,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "regulator_gate",
            "role": "gate",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "KLF2",
              "Klf2"
            ]
          },
          {
            "name": "controlling_node_character_gate",
            "role": "gate",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "high_differential_expression"
          },
          {
            "name": "expression_vs_activity_gate",
            "role": "gate",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "expression_exceeds_activity"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "target_program_partition",
            "kind": "list",
            "matchKey": "gene",
            "k": 7,
            "expected": [
              {
                "key": "S1pr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "S1pr4",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Klrg1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Cx3cr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Itgae",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Hic1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "P2rx7",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              }
            ]
          },
          {
            "name": "causal_status_gate",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "association_only_sufficiency_requires_perturbation"
          },
          {
            "name": "residency_control_logic_gate",
            "role": "gate",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_release_necessary_not_sufficient_multi_regulon"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "trm_like_program_by_model",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_absent"
                    ],
                    "canonical": "niche_absent"
                  }
                }
              }
            ]
          },
          {
            "name": "cf_regulator_activator_or_wrong_node",
            "role": "hard_fail",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "RUNX3",
              "Runx3",
              "PRDM1",
              "Prdm1",
              "BLIMP1",
              "Blimp1",
              "BHLHE40",
              "Bhlhe40",
              "ZNF683",
              "Zfp683",
              "HOBIT",
              "Hobit",
              "AHR",
              "Ahr",
              "BATF",
              "Batf",
              "TOX",
              "Tox",
              "NR4A1",
              "Nr4a1",
              "NR4A2",
              "Nr4a2",
              "NR4A3",
              "Nr4a3",
              "TCF7",
              "Tcf7",
              "TCF1",
              "Tcf1",
              "BACH2",
              "Bach2",
              "TBX21",
              "Tbx21",
              "TBET",
              "Tbet",
              "FLI1",
              "Fli1",
              "NR3C1",
              "Nr3c1",
              "EOMES",
              "Eomes",
              "ZEB2",
              "Zeb2",
              "ID2",
              "Id2",
              "ID3",
              "Id3",
              "IRF4",
              "Irf4"
            ]
          },
          {
            "name": "cf_node_character_hidden_driver_recall",
            "role": "hard_fail",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "hidden_driver_activity_not_expression"
          },
          {
            "name": "cf_effect_relation_hidden_pattern",
            "role": "hard_fail",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "activity_exceeds_expression"
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "causal_status",
            "kind": "value",
            "op": "in",
            "expected": [
              "sufficiency_established",
              "necessity_established",
              "causal_established_in_dataset",
              "sufficiency_via_perturbation_in_dataset"
            ]
          },
          {
            "name": "cf_residency_logic_activator_prior",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "residency_activator_induction_primary"
          },
          {
            "name": "cf_residency_logic_single_regulon",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_controls_entire_residency_program"
          },
          {
            "name": "cf_decoy_called_target",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_distractor_model_reported",
            "role": "hard_fail",
            "field": "$.trm_like_program_by_model[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_chromatin_only_decoy",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-5-pi",
      "modelName": "grok-4.5",
      "harness": "pi",
      "task": "green_shared_vs_distinct_regulators_trm_vs_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 5255.233255,
      "cost": 1.641136,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_program_map",
            "kind": "list",
            "matchKey": "regulator",
            "k": 3,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "both"
                    ],
                    "canonical": "both"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "represses"
                    ],
                    "canonical": "represses"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Batf",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "trm_like_til_specific"
                    ],
                    "canonical": "trm_like_til_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "chromatin_over_rna"
                    ],
                    "canonical": "chromatin_over_rna"
                  }
                }
              },
              {
                "key": "Ahr",
                "fields": {
                  "program_specificity": {
                    "op": "in",
                    "values": [
                      "trm_specific",
                      "both"
                    ],
                    "canonical": "trm_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "rna_over_chromatin"
                    ],
                    "canonical": "rna_over_chromatin"
                  }
                }
              }
            ]
          },
          {
            "name": "regulator_set_no_extras_gate",
            "role": "gate",
            "field": "$.regulator_program_map[*]",
            "kind": "other",
            "expected": null
          },
          {
            "name": "program_identity_conclusion_gate",
            "role": "gate",
            "field": "program_identity_conclusion",
            "kind": "value",
            "expected": "distinct_state_shared_brake_divergent_activators"
          },
          {
            "name": "residency_install_logic_gate",
            "role": "gate",
            "field": "residency_install_logic",
            "kind": "value",
            "expected": "brake_release_permits_context_matched_activator"
          },
          {
            "name": "activator_context_dependence_gate",
            "role": "gate",
            "field": "activator_context_dependence",
            "kind": "value",
            "expected": "context_matched_crossover"
          },
          {
            "name": "evidence_ceiling_gate",
            "role": "gate",
            "field": "evidence_ceiling",
            "kind": "value",
            "expected": "specificity_dissociations_require_perturbation"
          },
          {
            "name": "cf_install_logic_flat_or_single_step",
            "role": "hard_fail",
            "field": "residency_install_logic",
            "kind": "value",
            "op": "in",
            "expected": [
              "flat_shared_residency_toolkit",
              "activator_induction_sufficient",
              "brake_release_alone_sufficient"
            ]
          },
          {
            "name": "cf_program_identity_lumped_or_wrong",
            "role": "hard_fail",
            "field": "program_identity_conclusion",
            "kind": "value",
            "op": "in",
            "expected": [
              "trm_like_til_is_tumor_trm",
              "trm_like_til_is_exhausted_subset",
              "trm_like_til_is_effector_state",
              "trm_like_til_is_transitional_intermediate",
              "trm_like_til_is_shared_program_distinct_niche"
            ]
          },
          {
            "name": "cf_ahr_assigned_to_til",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Ahr",
            "valueField": "program_specificity",
            "expected": [
              "trm_like_til_specific"
            ]
          },
          {
            "name": "cf_batf_assigned_to_trm_or_shared",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Batf",
            "valueField": "program_specificity",
            "expected": [
              "trm_specific",
              "both"
            ]
          },
          {
            "name": "cf_klf2_called_activator",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Klf2",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          },
          {
            "name": "cf_evidence_ceiling_overclaim",
            "role": "hard_fail",
            "field": "evidence_ceiling",
            "kind": "value",
            "op": "in",
            "expected": [
              "specificity_established_from_regulon_activity",
              "specificity_established_in_dataset",
              "causation_established_by_cross_sectional_concordance",
              "necessity_and_sufficiency_established"
            ]
          },
          {
            "name": "cf_activator_context_shared_or_single",
            "role": "hard_fail",
            "field": "activator_context_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "shared_activators_both_contexts",
              "single_global_activator",
              "redundant_activators"
            ]
          },
          {
            "name": "cf_single_modality_rule",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_hic1_motif_brake_mislabel",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_runx3_anticanonical_as_controller",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Runx3",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-5-pi",
      "modelName": "grok-4.5",
      "harness": "pi",
      "task": "green_shared_vs_distinct_regulators_trm_vs_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 3305.56299,
      "cost": 0.9734999999999999,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_program_map",
            "kind": "list",
            "matchKey": "regulator",
            "k": 3,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "both"
                    ],
                    "canonical": "both"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "represses"
                    ],
                    "canonical": "represses"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Batf",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "trm_like_til_specific"
                    ],
                    "canonical": "trm_like_til_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "chromatin_over_rna"
                    ],
                    "canonical": "chromatin_over_rna"
                  }
                }
              },
              {
                "key": "Ahr",
                "fields": {
                  "program_specificity": {
                    "op": "in",
                    "values": [
                      "trm_specific",
                      "both"
                    ],
                    "canonical": "trm_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "rna_over_chromatin"
                    ],
                    "canonical": "rna_over_chromatin"
                  }
                }
              }
            ]
          },
          {
            "name": "regulator_set_no_extras_gate",
            "role": "gate",
            "field": "$.regulator_program_map[*]",
            "kind": "other",
            "expected": null
          },
          {
            "name": "program_identity_conclusion_gate",
            "role": "gate",
            "field": "program_identity_conclusion",
            "kind": "value",
            "expected": "distinct_state_shared_brake_divergent_activators"
          },
          {
            "name": "residency_install_logic_gate",
            "role": "gate",
            "field": "residency_install_logic",
            "kind": "value",
            "expected": "brake_release_permits_context_matched_activator"
          },
          {
            "name": "activator_context_dependence_gate",
            "role": "gate",
            "field": "activator_context_dependence",
            "kind": "value",
            "expected": "context_matched_crossover"
          },
          {
            "name": "evidence_ceiling_gate",
            "role": "gate",
            "field": "evidence_ceiling",
            "kind": "value",
            "expected": "specificity_dissociations_require_perturbation"
          },
          {
            "name": "cf_install_logic_flat_or_single_step",
            "role": "hard_fail",
            "field": "residency_install_logic",
            "kind": "value",
            "op": "in",
            "expected": [
              "flat_shared_residency_toolkit",
              "activator_induction_sufficient",
              "brake_release_alone_sufficient"
            ]
          },
          {
            "name": "cf_program_identity_lumped_or_wrong",
            "role": "hard_fail",
            "field": "program_identity_conclusion",
            "kind": "value",
            "op": "in",
            "expected": [
              "trm_like_til_is_tumor_trm",
              "trm_like_til_is_exhausted_subset",
              "trm_like_til_is_effector_state",
              "trm_like_til_is_transitional_intermediate",
              "trm_like_til_is_shared_program_distinct_niche"
            ]
          },
          {
            "name": "cf_ahr_assigned_to_til",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Ahr",
            "valueField": "program_specificity",
            "expected": [
              "trm_like_til_specific"
            ]
          },
          {
            "name": "cf_batf_assigned_to_trm_or_shared",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Batf",
            "valueField": "program_specificity",
            "expected": [
              "trm_specific",
              "both"
            ]
          },
          {
            "name": "cf_klf2_called_activator",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Klf2",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          },
          {
            "name": "cf_evidence_ceiling_overclaim",
            "role": "hard_fail",
            "field": "evidence_ceiling",
            "kind": "value",
            "op": "in",
            "expected": [
              "specificity_established_from_regulon_activity",
              "specificity_established_in_dataset",
              "causation_established_by_cross_sectional_concordance",
              "necessity_and_sufficiency_established"
            ]
          },
          {
            "name": "cf_activator_context_shared_or_single",
            "role": "hard_fail",
            "field": "activator_context_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "shared_activators_both_contexts",
              "single_global_activator",
              "redundant_activators"
            ]
          },
          {
            "name": "cf_single_modality_rule",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_hic1_motif_brake_mislabel",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_runx3_anticanonical_as_controller",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Runx3",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-5-pi",
      "modelName": "grok-4.5",
      "harness": "pi",
      "task": "green_shared_vs_distinct_regulators_trm_vs_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 2229.710949,
      "cost": 1.0736640000000002,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_program_map",
            "kind": "list",
            "matchKey": "regulator",
            "k": 3,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "both"
                    ],
                    "canonical": "both"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "represses"
                    ],
                    "canonical": "represses"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Batf",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "trm_like_til_specific"
                    ],
                    "canonical": "trm_like_til_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "chromatin_over_rna"
                    ],
                    "canonical": "chromatin_over_rna"
                  }
                }
              },
              {
                "key": "Ahr",
                "fields": {
                  "program_specificity": {
                    "op": "in",
                    "values": [
                      "trm_specific",
                      "both"
                    ],
                    "canonical": "trm_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "rna_over_chromatin"
                    ],
                    "canonical": "rna_over_chromatin"
                  }
                }
              }
            ]
          },
          {
            "name": "regulator_set_no_extras_gate",
            "role": "gate",
            "field": "$.regulator_program_map[*]",
            "kind": "other",
            "expected": null
          },
          {
            "name": "program_identity_conclusion_gate",
            "role": "gate",
            "field": "program_identity_conclusion",
            "kind": "value",
            "expected": "distinct_state_shared_brake_divergent_activators"
          },
          {
            "name": "residency_install_logic_gate",
            "role": "gate",
            "field": "residency_install_logic",
            "kind": "value",
            "expected": "brake_release_permits_context_matched_activator"
          },
          {
            "name": "activator_context_dependence_gate",
            "role": "gate",
            "field": "activator_context_dependence",
            "kind": "value",
            "expected": "context_matched_crossover"
          },
          {
            "name": "evidence_ceiling_gate",
            "role": "gate",
            "field": "evidence_ceiling",
            "kind": "value",
            "expected": "specificity_dissociations_require_perturbation"
          },
          {
            "name": "cf_install_logic_flat_or_single_step",
            "role": "hard_fail",
            "field": "residency_install_logic",
            "kind": "value",
            "op": "in",
            "expected": [
              "flat_shared_residency_toolkit",
              "activator_induction_sufficient",
              "brake_release_alone_sufficient"
            ]
          },
          {
            "name": "cf_program_identity_lumped_or_wrong",
            "role": "hard_fail",
            "field": "program_identity_conclusion",
            "kind": "value",
            "op": "in",
            "expected": [
              "trm_like_til_is_tumor_trm",
              "trm_like_til_is_exhausted_subset",
              "trm_like_til_is_effector_state",
              "trm_like_til_is_transitional_intermediate",
              "trm_like_til_is_shared_program_distinct_niche"
            ]
          },
          {
            "name": "cf_ahr_assigned_to_til",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Ahr",
            "valueField": "program_specificity",
            "expected": [
              "trm_like_til_specific"
            ]
          },
          {
            "name": "cf_batf_assigned_to_trm_or_shared",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Batf",
            "valueField": "program_specificity",
            "expected": [
              "trm_specific",
              "both"
            ]
          },
          {
            "name": "cf_klf2_called_activator",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Klf2",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          },
          {
            "name": "cf_evidence_ceiling_overclaim",
            "role": "hard_fail",
            "field": "evidence_ceiling",
            "kind": "value",
            "op": "in",
            "expected": [
              "specificity_established_from_regulon_activity",
              "specificity_established_in_dataset",
              "causation_established_by_cross_sectional_concordance",
              "necessity_and_sufficiency_established"
            ]
          },
          {
            "name": "cf_activator_context_shared_or_single",
            "role": "hard_fail",
            "field": "activator_context_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "shared_activators_both_contexts",
              "single_global_activator",
              "redundant_activators"
            ]
          },
          {
            "name": "cf_single_modality_rule",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_hic1_motif_brake_mislabel",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_runx3_anticanonical_as_controller",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Runx3",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-5-pi",
      "modelName": "grok-4.5",
      "harness": "pi",
      "task": "gse297023_q2_senescence_not_mechanism",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 722.647974,
      "cost": 0.228872,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "candidate_is_senescence",
            "role": "gate",
            "field": "$.candidate_mechanism",
            "kind": "value",
            "op": "in",
            "expected": [
              "cellular_senescence",
              "oncogene_induced_senescence",
              "replicative_senescence"
            ]
          },
          {
            "name": "arrest_core_not_engaged",
            "role": "gate",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_not_engaged"
            ]
          },
          {
            "name": "secretory_arm_is_inflammaging",
            "role": "gate",
            "field": "$.secretory_arm_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "acute_phase_inflammaging_response"
            ]
          },
          {
            "name": "secretory_arm_broadly_elevated",
            "role": "gate",
            "field": "$.secretory_arm_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "broadly_elevated"
            ]
          },
          {
            "name": "carries_aging_program",
            "role": "gate",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "carries_bona_fide_aging_program_distinct_from_nominated"
            ]
          },
          {
            "name": "conclusion_senescence_unlikely",
            "role": "gate",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "unlikely_major_contributor"
            ]
          },
          {
            "name": "cf_arrest_core_engaged_recall",
            "role": "hard_fail",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_engaged"
            ]
          },
          {
            "name": "cf_aging_program_conflation_or_explain_away",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "aging_program_indistinguishable_from_nominated_program",
              "aging_program_is_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_no_aging_program_sensitivity_failure",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_detectable_aging_program"
            ]
          },
          {
            "name": "cf_senescence_driver_conclusion",
            "role": "hard_fail",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "likely_primary_driver",
              "plausible_supported_contributor"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-5-pi",
      "modelName": "grok-4.5",
      "harness": "pi",
      "task": "gse297023_q2_senescence_not_mechanism",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 1229.668716,
      "cost": 0.30716599999999994,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "candidate_is_senescence",
            "role": "gate",
            "field": "$.candidate_mechanism",
            "kind": "value",
            "op": "in",
            "expected": [
              "cellular_senescence",
              "oncogene_induced_senescence",
              "replicative_senescence"
            ]
          },
          {
            "name": "arrest_core_not_engaged",
            "role": "gate",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_not_engaged"
            ]
          },
          {
            "name": "secretory_arm_is_inflammaging",
            "role": "gate",
            "field": "$.secretory_arm_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "acute_phase_inflammaging_response"
            ]
          },
          {
            "name": "secretory_arm_broadly_elevated",
            "role": "gate",
            "field": "$.secretory_arm_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "broadly_elevated"
            ]
          },
          {
            "name": "carries_aging_program",
            "role": "gate",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "carries_bona_fide_aging_program_distinct_from_nominated"
            ]
          },
          {
            "name": "conclusion_senescence_unlikely",
            "role": "gate",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "unlikely_major_contributor"
            ]
          },
          {
            "name": "cf_arrest_core_engaged_recall",
            "role": "hard_fail",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_engaged"
            ]
          },
          {
            "name": "cf_aging_program_conflation_or_explain_away",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "aging_program_indistinguishable_from_nominated_program",
              "aging_program_is_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_no_aging_program_sensitivity_failure",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_detectable_aging_program"
            ]
          },
          {
            "name": "cf_senescence_driver_conclusion",
            "role": "hard_fail",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "likely_primary_driver",
              "plausible_supported_contributor"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-5-pi",
      "modelName": "grok-4.5",
      "harness": "pi",
      "task": "gse297023_q2_senescence_not_mechanism",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 641.529285,
      "cost": 0.349184,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "candidate_is_senescence",
            "role": "gate",
            "field": "$.candidate_mechanism",
            "kind": "value",
            "op": "in",
            "expected": [
              "cellular_senescence",
              "oncogene_induced_senescence",
              "replicative_senescence"
            ]
          },
          {
            "name": "arrest_core_not_engaged",
            "role": "gate",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_not_engaged"
            ]
          },
          {
            "name": "secretory_arm_is_inflammaging",
            "role": "gate",
            "field": "$.secretory_arm_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "acute_phase_inflammaging_response"
            ]
          },
          {
            "name": "secretory_arm_broadly_elevated",
            "role": "gate",
            "field": "$.secretory_arm_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "broadly_elevated"
            ]
          },
          {
            "name": "carries_aging_program",
            "role": "gate",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "carries_bona_fide_aging_program_distinct_from_nominated"
            ]
          },
          {
            "name": "conclusion_senescence_unlikely",
            "role": "gate",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "unlikely_major_contributor"
            ]
          },
          {
            "name": "cf_arrest_core_engaged_recall",
            "role": "hard_fail",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_engaged"
            ]
          },
          {
            "name": "cf_aging_program_conflation_or_explain_away",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "aging_program_indistinguishable_from_nominated_program",
              "aging_program_is_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_no_aging_program_sensitivity_failure",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_detectable_aging_program"
            ]
          },
          {
            "name": "cf_senescence_driver_conclusion",
            "role": "hard_fail",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "likely_primary_driver",
              "plausible_supported_contributor"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-5-pi",
      "modelName": "grok-4.5",
      "harness": "pi",
      "task": "gse297023_q3_age_retained_mapk_reduced",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 3408.761834,
      "cost": 0.382342,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "age_retained",
            "role": "gate",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_retained"
            ]
          },
          {
            "name": "identity_is_aging",
            "role": "gate",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_aging_signatures"
            ]
          },
          {
            "name": "nature_state_shift",
            "role": "gate",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_state_expression_shift"
            ]
          },
          {
            "name": "pathway_is_mapk",
            "role": "gate",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "mapk"
            ]
          },
          {
            "name": "pathway_reduced_in_aged",
            "role": "gate",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "reduced_in_aged"
            ]
          },
          {
            "name": "cf_mapk_increased_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_in_aged"
            ]
          },
          {
            "name": "cf_age_erased",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_erased"
            ]
          },
          {
            "name": "cf_pervasive_overcall",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_pervasive_artifactual"
            ]
          },
          {
            "name": "cf_explain_away_aging",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_compositional_or_identity",
            "role": "hard_fail",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_proportion_shift",
              "cell_identity_shift",
              "composition_dominant_state_secondary",
              "both_state_and_composition_equally"
            ]
          },
          {
            "name": "cf_biggest_inflammatory_signal_not_mapk",
            "role": "hard_fail",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "jak_stat",
              "nfkb",
              "tnfa"
            ]
          },
          {
            "name": "cf_senescence_misattribution",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_senescence_signature_specifically"
            ]
          },
          {
            "name": "cf_dismiss_as_batch_artifact",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_batch_or_technical_artifact"
            ]
          },
          {
            "name": "cf_mapk_comparable_constitutive_kras_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "comparable_across_ages"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-5-pi",
      "modelName": "grok-4.5",
      "harness": "pi",
      "task": "gse297023_q3_age_retained_mapk_reduced",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 2474.367427,
      "cost": 0.43126800000000004,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "age_retained",
            "role": "gate",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_retained"
            ]
          },
          {
            "name": "identity_is_aging",
            "role": "gate",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_aging_signatures"
            ]
          },
          {
            "name": "nature_state_shift",
            "role": "gate",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_state_expression_shift"
            ]
          },
          {
            "name": "pathway_is_mapk",
            "role": "gate",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "mapk"
            ]
          },
          {
            "name": "pathway_reduced_in_aged",
            "role": "gate",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "reduced_in_aged"
            ]
          },
          {
            "name": "cf_mapk_increased_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_in_aged"
            ]
          },
          {
            "name": "cf_age_erased",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_erased"
            ]
          },
          {
            "name": "cf_pervasive_overcall",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_pervasive_artifactual"
            ]
          },
          {
            "name": "cf_explain_away_aging",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_compositional_or_identity",
            "role": "hard_fail",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_proportion_shift",
              "cell_identity_shift",
              "composition_dominant_state_secondary",
              "both_state_and_composition_equally"
            ]
          },
          {
            "name": "cf_biggest_inflammatory_signal_not_mapk",
            "role": "hard_fail",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "jak_stat",
              "nfkb",
              "tnfa"
            ]
          },
          {
            "name": "cf_senescence_misattribution",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_senescence_signature_specifically"
            ]
          },
          {
            "name": "cf_dismiss_as_batch_artifact",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_batch_or_technical_artifact"
            ]
          },
          {
            "name": "cf_mapk_comparable_constitutive_kras_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "comparable_across_ages"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-5-pi",
      "modelName": "grok-4.5",
      "harness": "pi",
      "task": "gse297023_q3_age_retained_mapk_reduced",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 2756.573522,
      "cost": 0.60202,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "age_retained",
            "role": "gate",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_retained"
            ]
          },
          {
            "name": "identity_is_aging",
            "role": "gate",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_aging_signatures"
            ]
          },
          {
            "name": "nature_state_shift",
            "role": "gate",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_state_expression_shift"
            ]
          },
          {
            "name": "pathway_is_mapk",
            "role": "gate",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "mapk"
            ]
          },
          {
            "name": "pathway_reduced_in_aged",
            "role": "gate",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "reduced_in_aged"
            ]
          },
          {
            "name": "cf_mapk_increased_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_in_aged"
            ]
          },
          {
            "name": "cf_age_erased",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_erased"
            ]
          },
          {
            "name": "cf_pervasive_overcall",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_pervasive_artifactual"
            ]
          },
          {
            "name": "cf_explain_away_aging",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_compositional_or_identity",
            "role": "hard_fail",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_proportion_shift",
              "cell_identity_shift",
              "composition_dominant_state_secondary",
              "both_state_and_composition_equally"
            ]
          },
          {
            "name": "cf_biggest_inflammatory_signal_not_mapk",
            "role": "hard_fail",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "jak_stat",
              "nfkb",
              "tnfa"
            ]
          },
          {
            "name": "cf_senescence_misattribution",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_senescence_signature_specifically"
            ]
          },
          {
            "name": "cf_dismiss_as_batch_artifact",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_batch_or_technical_artifact"
            ]
          },
          {
            "name": "cf_mapk_comparable_constitutive_kras_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "comparable_across_ages"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-5-pi",
      "modelName": "grok-4.5",
      "harness": "pi",
      "task": "gse297023_q5_tme_age_dependent_remodeling",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 2681.845119,
      "cost": 0.35131599999999996,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "perturbation_is_pten",
            "role": "gate",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "pten"
            ]
          },
          {
            "name": "tme_non_cell_autonomous",
            "role": "gate",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "substantial_non_cell_autonomous_remodeling"
            ]
          },
          {
            "name": "tme_state_not_composition",
            "role": "gate",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "tme_age_dependent",
            "role": "gate",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_dependent"
            ]
          },
          {
            "name": "tme_response_broader_in_aged",
            "role": "gate",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_aged"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "most_responsive_noncancer_cell_types",
            "kind": "list",
            "matchKey": "cell_type",
            "k": 2,
            "expected": [
              {
                "key": "alveolar_macrophage",
                "fields": {}
              },
              {
                "key": "endothelial",
                "fields": {}
              }
            ]
          },
          {
            "name": "cf_cell_intrinsic_only",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "negligible_cell_intrinsic_only_effect",
              "remodeling_unresolved",
              "modest_remodeling_few_cell_types"
            ]
          },
          {
            "name": "cf_remodeling_is_contamination",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "apparent_remodeling_reflects_cancer_contamination"
            ]
          },
          {
            "name": "cf_perturbation_design_misread",
            "role": "hard_fail",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "multiple_tumor_suppressors_pooled",
              "no_additional_gene_difference_kras_only",
              "rb1_cdkn2a_double",
              "pcna_essential_gene_control",
              "difference_is_oncogene_not_suppressor"
            ]
          },
          {
            "name": "cf_age_independent_tme",
            "role": "hard_fail",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_independent"
            ]
          },
          {
            "name": "cf_compositional_shift",
            "role": "hard_fail",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_frequency_shift",
              "compositional_immune_infiltration",
              "composition_dominant_state_secondary",
              "both_state_and_composition"
            ]
          },
          {
            "name": "cf_epithelial_or_cancer_substitution",
            "role": "hard_fail",
            "field": "$.most_responsive_noncancer_cell_types[*].cell_type",
            "kind": "avoid",
            "expected": [
              "AT2_cell",
              "AT1_cell",
              "cancer_cell",
              "ciliated",
              "secretory",
              "club_cell",
              "basal_cell",
              "goblet_cell"
            ]
          },
          {
            "name": "cf_tme_breadth_broader_in_young",
            "role": "hard_fail",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_young"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-5-pi",
      "modelName": "grok-4.5",
      "harness": "pi",
      "task": "gse297023_q5_tme_age_dependent_remodeling",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 2945.192235,
      "cost": 0.44414800000000004,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "perturbation_is_pten",
            "role": "gate",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "pten"
            ]
          },
          {
            "name": "tme_non_cell_autonomous",
            "role": "gate",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "substantial_non_cell_autonomous_remodeling"
            ]
          },
          {
            "name": "tme_state_not_composition",
            "role": "gate",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "tme_age_dependent",
            "role": "gate",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_dependent"
            ]
          },
          {
            "name": "tme_response_broader_in_aged",
            "role": "gate",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_aged"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "most_responsive_noncancer_cell_types",
            "kind": "list",
            "matchKey": "cell_type",
            "k": 2,
            "expected": [
              {
                "key": "alveolar_macrophage",
                "fields": {}
              },
              {
                "key": "endothelial",
                "fields": {}
              }
            ]
          },
          {
            "name": "cf_cell_intrinsic_only",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "negligible_cell_intrinsic_only_effect",
              "remodeling_unresolved",
              "modest_remodeling_few_cell_types"
            ]
          },
          {
            "name": "cf_remodeling_is_contamination",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "apparent_remodeling_reflects_cancer_contamination"
            ]
          },
          {
            "name": "cf_perturbation_design_misread",
            "role": "hard_fail",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "multiple_tumor_suppressors_pooled",
              "no_additional_gene_difference_kras_only",
              "rb1_cdkn2a_double",
              "pcna_essential_gene_control",
              "difference_is_oncogene_not_suppressor"
            ]
          },
          {
            "name": "cf_age_independent_tme",
            "role": "hard_fail",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_independent"
            ]
          },
          {
            "name": "cf_compositional_shift",
            "role": "hard_fail",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_frequency_shift",
              "compositional_immune_infiltration",
              "composition_dominant_state_secondary",
              "both_state_and_composition"
            ]
          },
          {
            "name": "cf_epithelial_or_cancer_substitution",
            "role": "hard_fail",
            "field": "$.most_responsive_noncancer_cell_types[*].cell_type",
            "kind": "avoid",
            "expected": [
              "AT2_cell",
              "AT1_cell",
              "cancer_cell",
              "ciliated",
              "secretory",
              "club_cell",
              "basal_cell",
              "goblet_cell"
            ]
          },
          {
            "name": "cf_tme_breadth_broader_in_young",
            "role": "hard_fail",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_young"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-5-pi",
      "modelName": "grok-4.5",
      "harness": "pi",
      "task": "gse297023_q5_tme_age_dependent_remodeling",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 3990.433205,
      "cost": 0.525766,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "perturbation_is_pten",
            "role": "gate",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "pten"
            ]
          },
          {
            "name": "tme_non_cell_autonomous",
            "role": "gate",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "substantial_non_cell_autonomous_remodeling"
            ]
          },
          {
            "name": "tme_state_not_composition",
            "role": "gate",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "tme_age_dependent",
            "role": "gate",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_dependent"
            ]
          },
          {
            "name": "tme_response_broader_in_aged",
            "role": "gate",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_aged"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "most_responsive_noncancer_cell_types",
            "kind": "list",
            "matchKey": "cell_type",
            "k": 2,
            "expected": [
              {
                "key": "alveolar_macrophage",
                "fields": {}
              },
              {
                "key": "endothelial",
                "fields": {}
              }
            ]
          },
          {
            "name": "cf_cell_intrinsic_only",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "negligible_cell_intrinsic_only_effect",
              "remodeling_unresolved",
              "modest_remodeling_few_cell_types"
            ]
          },
          {
            "name": "cf_remodeling_is_contamination",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "apparent_remodeling_reflects_cancer_contamination"
            ]
          },
          {
            "name": "cf_perturbation_design_misread",
            "role": "hard_fail",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "multiple_tumor_suppressors_pooled",
              "no_additional_gene_difference_kras_only",
              "rb1_cdkn2a_double",
              "pcna_essential_gene_control",
              "difference_is_oncogene_not_suppressor"
            ]
          },
          {
            "name": "cf_age_independent_tme",
            "role": "hard_fail",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_independent"
            ]
          },
          {
            "name": "cf_compositional_shift",
            "role": "hard_fail",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_frequency_shift",
              "compositional_immune_infiltration",
              "composition_dominant_state_secondary",
              "both_state_and_composition"
            ]
          },
          {
            "name": "cf_epithelial_or_cancer_substitution",
            "role": "hard_fail",
            "field": "$.most_responsive_noncancer_cell_types[*].cell_type",
            "kind": "avoid",
            "expected": [
              "AT2_cell",
              "AT1_cell",
              "cancer_cell",
              "ciliated",
              "secretory",
              "club_cell",
              "basal_cell",
              "goblet_cell"
            ]
          },
          {
            "name": "cf_tme_breadth_broader_in_young",
            "role": "hard_fail",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_young"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-5-pi",
      "modelName": "grok-4.5",
      "harness": "pi",
      "task": "gse297023_q6_age_immune_infiltration",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 1,
      "passed": true,
      "score": null,
      "durationS": 2517.058782,
      "cost": 0.375156,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "composition_stable_with_age",
            "role": "gate",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "frequencies_largely_stable"
            ]
          },
          {
            "name": "age_signal_is_state_not_composition",
            "role": "gate",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "frequencies_uninformative_for_infiltration",
            "role": "gate",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_cannot_establish_infiltration"
            ]
          },
          {
            "name": "infiltration_increases_with_age",
            "role": "gate",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_with_age"
            ]
          },
          {
            "name": "infiltration_strength_calibrated",
            "role": "gate",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "significant_increase_strongest_markers_only",
              "consistent_increase_not_statistically_robust"
            ]
          },
          {
            "name": "genotype_marker_fraction_lower_in_pten",
            "role": "gate",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "lower_in_pten_deficient"
            ]
          },
          {
            "name": "cf_composition_shift_reading",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "major_compositional_shift_with_age",
              "increased_immune_fraction_with_age"
            ]
          },
          {
            "name": "cf_lineage_specific_composition_claim",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "myeloid_expansion_with_age",
              "lymphoid_expansion_with_age",
              "minor_shifts_in_some_cell_types"
            ]
          },
          {
            "name": "cf_immunosenescence_depletion_prior",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "immune_depletion_with_age"
            ]
          },
          {
            "name": "cf_age_modality_compositional",
            "role": "hard_fail",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_type_composition_shift",
              "both_equally",
              "composition_dominant_state_secondary"
            ]
          },
          {
            "name": "cf_overread_frequencies_as_infiltration",
            "role": "hard_fail",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_shows_no_infiltration_change",
              "single_cell_quantifies_infiltration",
              "frequencies_show_increased_infiltration",
              "frequencies_show_decreased_infiltration"
            ]
          },
          {
            "name": "cf_infiltration_decreases_immunosenescence",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "decreased_with_age"
            ]
          },
          {
            "name": "cf_infiltration_unchanged_scrna_overread",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "unchanged_with_age"
            ]
          },
          {
            "name": "cf_infiltration_indeterminate_unused_node",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "direction_indeterminate_from_data"
            ]
          },
          {
            "name": "cf_infiltration_mixed_direction",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "mixed_by_immune_cell_type"
            ]
          },
          {
            "name": "cf_infiltration_genotype_confusion",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_only_in_pten_deficient",
              "decreased_only_in_pten_deficient"
            ]
          },
          {
            "name": "cf_pten_infiltration_paper_text_recall",
            "role": "hard_fail",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "higher_in_pten_deficient",
              "higher_in_pten_deficient_only_in_young",
              "higher_in_pten_deficient_only_in_aged"
            ]
          },
          {
            "name": "cf_pseudoreplication_overclaim",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_increase_across_markers"
            ]
          },
          {
            "name": "cf_infiltration_strength_decrease_direction",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_decrease_across_markers",
              "significant_decrease_strongest_markers_only",
              "consistent_decrease_not_statistically_robust"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-5-pi",
      "modelName": "grok-4.5",
      "harness": "pi",
      "task": "gse297023_q6_age_immune_infiltration",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 2,
      "passed": true,
      "score": null,
      "durationS": 3453.584266,
      "cost": 0.3840259999999999,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "composition_stable_with_age",
            "role": "gate",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "frequencies_largely_stable"
            ]
          },
          {
            "name": "age_signal_is_state_not_composition",
            "role": "gate",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "frequencies_uninformative_for_infiltration",
            "role": "gate",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_cannot_establish_infiltration"
            ]
          },
          {
            "name": "infiltration_increases_with_age",
            "role": "gate",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_with_age"
            ]
          },
          {
            "name": "infiltration_strength_calibrated",
            "role": "gate",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "significant_increase_strongest_markers_only",
              "consistent_increase_not_statistically_robust"
            ]
          },
          {
            "name": "genotype_marker_fraction_lower_in_pten",
            "role": "gate",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "lower_in_pten_deficient"
            ]
          },
          {
            "name": "cf_composition_shift_reading",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "major_compositional_shift_with_age",
              "increased_immune_fraction_with_age"
            ]
          },
          {
            "name": "cf_lineage_specific_composition_claim",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "myeloid_expansion_with_age",
              "lymphoid_expansion_with_age",
              "minor_shifts_in_some_cell_types"
            ]
          },
          {
            "name": "cf_immunosenescence_depletion_prior",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "immune_depletion_with_age"
            ]
          },
          {
            "name": "cf_age_modality_compositional",
            "role": "hard_fail",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_type_composition_shift",
              "both_equally",
              "composition_dominant_state_secondary"
            ]
          },
          {
            "name": "cf_overread_frequencies_as_infiltration",
            "role": "hard_fail",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_shows_no_infiltration_change",
              "single_cell_quantifies_infiltration",
              "frequencies_show_increased_infiltration",
              "frequencies_show_decreased_infiltration"
            ]
          },
          {
            "name": "cf_infiltration_decreases_immunosenescence",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "decreased_with_age"
            ]
          },
          {
            "name": "cf_infiltration_unchanged_scrna_overread",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "unchanged_with_age"
            ]
          },
          {
            "name": "cf_infiltration_indeterminate_unused_node",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "direction_indeterminate_from_data"
            ]
          },
          {
            "name": "cf_infiltration_mixed_direction",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "mixed_by_immune_cell_type"
            ]
          },
          {
            "name": "cf_infiltration_genotype_confusion",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_only_in_pten_deficient",
              "decreased_only_in_pten_deficient"
            ]
          },
          {
            "name": "cf_pten_infiltration_paper_text_recall",
            "role": "hard_fail",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "higher_in_pten_deficient",
              "higher_in_pten_deficient_only_in_young",
              "higher_in_pten_deficient_only_in_aged"
            ]
          },
          {
            "name": "cf_pseudoreplication_overclaim",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_increase_across_markers"
            ]
          },
          {
            "name": "cf_infiltration_strength_decrease_direction",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_decrease_across_markers",
              "significant_decrease_strongest_markers_only",
              "consistent_decrease_not_statistically_robust"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-grok-4-5-pi",
      "modelName": "grok-4.5",
      "harness": "pi",
      "task": "gse297023_q6_age_immune_infiltration",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 3,
      "passed": true,
      "score": null,
      "durationS": 1544.032875,
      "cost": 0.446328,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "composition_stable_with_age",
            "role": "gate",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "frequencies_largely_stable"
            ]
          },
          {
            "name": "age_signal_is_state_not_composition",
            "role": "gate",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "frequencies_uninformative_for_infiltration",
            "role": "gate",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_cannot_establish_infiltration"
            ]
          },
          {
            "name": "infiltration_increases_with_age",
            "role": "gate",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_with_age"
            ]
          },
          {
            "name": "infiltration_strength_calibrated",
            "role": "gate",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "significant_increase_strongest_markers_only",
              "consistent_increase_not_statistically_robust"
            ]
          },
          {
            "name": "genotype_marker_fraction_lower_in_pten",
            "role": "gate",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "lower_in_pten_deficient"
            ]
          },
          {
            "name": "cf_composition_shift_reading",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "major_compositional_shift_with_age",
              "increased_immune_fraction_with_age"
            ]
          },
          {
            "name": "cf_lineage_specific_composition_claim",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "myeloid_expansion_with_age",
              "lymphoid_expansion_with_age",
              "minor_shifts_in_some_cell_types"
            ]
          },
          {
            "name": "cf_immunosenescence_depletion_prior",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "immune_depletion_with_age"
            ]
          },
          {
            "name": "cf_age_modality_compositional",
            "role": "hard_fail",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_type_composition_shift",
              "both_equally",
              "composition_dominant_state_secondary"
            ]
          },
          {
            "name": "cf_overread_frequencies_as_infiltration",
            "role": "hard_fail",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_shows_no_infiltration_change",
              "single_cell_quantifies_infiltration",
              "frequencies_show_increased_infiltration",
              "frequencies_show_decreased_infiltration"
            ]
          },
          {
            "name": "cf_infiltration_decreases_immunosenescence",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "decreased_with_age"
            ]
          },
          {
            "name": "cf_infiltration_unchanged_scrna_overread",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "unchanged_with_age"
            ]
          },
          {
            "name": "cf_infiltration_indeterminate_unused_node",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "direction_indeterminate_from_data"
            ]
          },
          {
            "name": "cf_infiltration_mixed_direction",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "mixed_by_immune_cell_type"
            ]
          },
          {
            "name": "cf_infiltration_genotype_confusion",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_only_in_pten_deficient",
              "decreased_only_in_pten_deficient"
            ]
          },
          {
            "name": "cf_pten_infiltration_paper_text_recall",
            "role": "hard_fail",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "higher_in_pten_deficient",
              "higher_in_pten_deficient_only_in_young",
              "higher_in_pten_deficient_only_in_aged"
            ]
          },
          {
            "name": "cf_pseudoreplication_overclaim",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_increase_across_markers"
            ]
          },
          {
            "name": "cf_infiltration_strength_decrease_direction",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_decrease_across_markers",
              "significant_decrease_strongest_markers_only",
              "consistent_decrease_not_statistically_robust"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-5-claude-code",
      "modelName": "claude-opus-5",
      "harness": "claude-code",
      "task": "gse155381_q4_pseudotime_delay",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 1,
      "passed": true,
      "score": null,
      "durationS": 1536.04,
      "cost": 11.692817250000004,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "pseudotime_direction_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "E17_chronological_match_weighted",
            "role": "additive",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pluripotency_exit_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pseudotime_direction_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_synchronous_with_monkey_EPI",
              "chimera_human_EPI_developmentally_advanced",
              "chimera_human_EPI_delayed_at_pre_implantation_only_resolved_post_implantation"
            ]
          },
          {
            "name": "E17_chronological_match_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "resembles_monkey_E17_Gast_EPI",
              "resembles_chimera_monkey_E17_EPI_specifically",
              "chimera_human_E17_EPI_more_advanced_than_monkey_E17_Gast"
            ]
          },
          {
            "name": "pluripotency_exit_paper_inverted_or_prior_driven",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_advances_pluripotency_exit_prematurely",
              "naive_pluripotency_program_specifically_engaged",
              "pluripotency_core_retained_via_KLF17_naive_marker_specifically"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-5-claude-code",
      "modelName": "claude-opus-5",
      "harness": "claude-code",
      "task": "gse155381_q4_pseudotime_delay",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 2,
      "passed": true,
      "score": null,
      "durationS": 1489.74,
      "cost": 5.209649750000001,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "pseudotime_direction_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "E17_chronological_match_weighted",
            "role": "additive",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pluripotency_exit_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pseudotime_direction_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_synchronous_with_monkey_EPI",
              "chimera_human_EPI_developmentally_advanced",
              "chimera_human_EPI_delayed_at_pre_implantation_only_resolved_post_implantation"
            ]
          },
          {
            "name": "E17_chronological_match_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "resembles_monkey_E17_Gast_EPI",
              "resembles_chimera_monkey_E17_EPI_specifically",
              "chimera_human_E17_EPI_more_advanced_than_monkey_E17_Gast"
            ]
          },
          {
            "name": "pluripotency_exit_paper_inverted_or_prior_driven",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_advances_pluripotency_exit_prematurely",
              "naive_pluripotency_program_specifically_engaged",
              "pluripotency_core_retained_via_KLF17_naive_marker_specifically"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-5-claude-code",
      "modelName": "claude-opus-5",
      "harness": "claude-code",
      "task": "gse155381_q4_pseudotime_delay",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 3,
      "passed": true,
      "score": null,
      "durationS": 2719.59,
      "cost": 16.96496349999999,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "pseudotime_direction_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "E17_chronological_match_weighted",
            "role": "additive",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pluripotency_exit_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pseudotime_direction_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_synchronous_with_monkey_EPI",
              "chimera_human_EPI_developmentally_advanced",
              "chimera_human_EPI_delayed_at_pre_implantation_only_resolved_post_implantation"
            ]
          },
          {
            "name": "E17_chronological_match_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "resembles_monkey_E17_Gast_EPI",
              "resembles_chimera_monkey_E17_EPI_specifically",
              "chimera_human_E17_EPI_more_advanced_than_monkey_E17_Gast"
            ]
          },
          {
            "name": "pluripotency_exit_paper_inverted_or_prior_driven",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_advances_pluripotency_exit_prematurely",
              "naive_pluripotency_program_specifically_engaged",
              "pluripotency_core_retained_via_KLF17_naive_marker_specifically"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-5-claude-code",
      "modelName": "claude-opus-5",
      "harness": "claude-code",
      "task": "gse155381_q5_chimera_monkey_perturbation",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 1,
      "passed": true,
      "score": null,
      "durationS": 1460.6,
      "cost": 3.75546875,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "deg_direction_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "comparison_subset_choice_weighted",
            "role": "additive",
            "field": "$.comparison_subset_choice",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "panel_resolved_strong_target_ccn1_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN1",
              "CYR61"
            ]
          },
          {
            "name": "panel_resolved_strong_target_ccn2_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN2",
              "CTGF"
            ]
          },
          {
            "name": "hippo_pathway_weighted",
            "role": "additive",
            "field": "$.hippo_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "tgf_beta_pathway_weighted",
            "role": "additive",
            "field": "$.tgf_beta_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "deg_direction_batch_effect_uncorrected",
            "role": "hard_fail",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_dominated_global",
              "up_dominated_in_chosen_subset"
            ]
          },
          {
            "name": "silent_panel_dropout_paired_with_directional_pathway_claim",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "hippo_pathway_direction": {
                "op": "in",
                "values": [
                  "hippo_pathway_upregulated",
                  "hippo_pathway_unchanged",
                  "wnt_pathway_specifically_perturbed",
                  "stress_response_pathway_dominantly_engaged",
                  "no_pathway_signal_above_confound",
                  "yap_tead_direct_target_panel_downregulated",
                  "yap_tead_panel_dependent_direction",
                  "hippo_pathway_downregulated"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-5-claude-code",
      "modelName": "claude-opus-5",
      "harness": "claude-code",
      "task": "gse155381_q5_chimera_monkey_perturbation",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 2,
      "passed": true,
      "score": null,
      "durationS": 1231.96,
      "cost": 4.184642750000002,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "deg_direction_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "comparison_subset_choice_weighted",
            "role": "additive",
            "field": "$.comparison_subset_choice",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "panel_resolved_strong_target_ccn1_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN1",
              "CYR61"
            ]
          },
          {
            "name": "panel_resolved_strong_target_ccn2_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN2",
              "CTGF"
            ]
          },
          {
            "name": "hippo_pathway_weighted",
            "role": "additive",
            "field": "$.hippo_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "tgf_beta_pathway_weighted",
            "role": "additive",
            "field": "$.tgf_beta_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "deg_direction_batch_effect_uncorrected",
            "role": "hard_fail",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_dominated_global",
              "up_dominated_in_chosen_subset"
            ]
          },
          {
            "name": "silent_panel_dropout_paired_with_directional_pathway_claim",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "hippo_pathway_direction": {
                "op": "in",
                "values": [
                  "hippo_pathway_upregulated",
                  "hippo_pathway_unchanged",
                  "wnt_pathway_specifically_perturbed",
                  "stress_response_pathway_dominantly_engaged",
                  "no_pathway_signal_above_confound",
                  "yap_tead_direct_target_panel_downregulated",
                  "yap_tead_panel_dependent_direction",
                  "hippo_pathway_downregulated"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-5-claude-code",
      "modelName": "claude-opus-5",
      "harness": "claude-code",
      "task": "gse155381_q5_chimera_monkey_perturbation",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1060.64,
      "cost": 4.6075040000000005,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "deg_direction_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "comparison_subset_choice_weighted",
            "role": "additive",
            "field": "$.comparison_subset_choice",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "panel_resolved_strong_target_ccn1_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN1",
              "CYR61"
            ]
          },
          {
            "name": "panel_resolved_strong_target_ccn2_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN2",
              "CTGF"
            ]
          },
          {
            "name": "hippo_pathway_weighted",
            "role": "additive",
            "field": "$.hippo_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "tgf_beta_pathway_weighted",
            "role": "additive",
            "field": "$.tgf_beta_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "deg_direction_batch_effect_uncorrected",
            "role": "hard_fail",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_dominated_global",
              "up_dominated_in_chosen_subset"
            ]
          },
          {
            "name": "silent_panel_dropout_paired_with_directional_pathway_claim",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "hippo_pathway_direction": {
                "op": "in",
                "values": [
                  "hippo_pathway_upregulated",
                  "hippo_pathway_unchanged",
                  "wnt_pathway_specifically_perturbed",
                  "stress_response_pathway_dominantly_engaged",
                  "no_pathway_signal_above_confound",
                  "yap_tead_direct_target_panel_downregulated",
                  "yap_tead_panel_dependent_direction",
                  "hippo_pathway_downregulated"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-5-claude-code",
      "modelName": "claude-opus-5",
      "harness": "claude-code",
      "task": "gse155381_q7_xspecies_epi_communication",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 1838,
      "cost": 15.495813249999998,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "richness_weighted",
            "role": "additive",
            "field": "$.xspecies_signaling_richness",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_weighted",
            "role": "additive",
            "field": "$.dominant_signaling_axis",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_wnt_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "WNT_FZD_LRP_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_fgf_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "FGF_FGFR_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_notch_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "NOTCH_DLL_JAG_family"
                ]
              }
            }
          },
          {
            "name": "signaling_directionality_weighted",
            "role": "additive",
            "field": "$.signaling_directionality",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "top_signal_interaction_class_weighted",
            "role": "additive",
            "field": "$.top_signal_interaction_class",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-5-claude-code",
      "modelName": "claude-opus-5",
      "harness": "claude-code",
      "task": "gse155381_q7_xspecies_epi_communication",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 1475.85,
      "cost": 5.84141225,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "richness_weighted",
            "role": "additive",
            "field": "$.xspecies_signaling_richness",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_weighted",
            "role": "additive",
            "field": "$.dominant_signaling_axis",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_wnt_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "WNT_FZD_LRP_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_fgf_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "FGF_FGFR_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_notch_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "NOTCH_DLL_JAG_family"
                ]
              }
            }
          },
          {
            "name": "signaling_directionality_weighted",
            "role": "additive",
            "field": "$.signaling_directionality",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "top_signal_interaction_class_weighted",
            "role": "additive",
            "field": "$.top_signal_interaction_class",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-5-claude-code",
      "modelName": "claude-opus-5",
      "harness": "claude-code",
      "task": "gse155381_q7_xspecies_epi_communication",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1496.33,
      "cost": 6.793883999999998,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "richness_weighted",
            "role": "additive",
            "field": "$.xspecies_signaling_richness",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_weighted",
            "role": "additive",
            "field": "$.dominant_signaling_axis",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_wnt_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "WNT_FZD_LRP_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_fgf_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "FGF_FGFR_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_notch_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "NOTCH_DLL_JAG_family"
                ]
              }
            }
          },
          {
            "name": "signaling_directionality_weighted",
            "role": "additive",
            "field": "$.signaling_directionality",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "top_signal_interaction_class_weighted",
            "role": "additive",
            "field": "$.top_signal_interaction_class",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-5-claude-code",
      "modelName": "claude-opus-5",
      "harness": "claude-code",
      "task": "gse155381_q8_bidirectional_convergence",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 1,
      "passed": true,
      "score": null,
      "durationS": 2689.5,
      "cost": 20.002233000000007,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "chimera_human_identity_shift_target_weighted",
            "role": "additive",
            "field": "$.chimera_human_identity_shift_target",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_in_vivo_state_assignment_weighted",
            "role": "additive",
            "field": "$.chimera_human_in_vivo_state_assignment",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_shift_magnitude_weighted",
            "role": "additive",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_monkey_identity_shift_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "both_arms_null_contradicted_by_calibration",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "chimera_human_identity_shift_target": {
                "op": "equals",
                "values": [
                  "chimera_human_maintains_origin_identity_no_measurable_shift"
                ]
              },
              "chimera_monkey_identity_shift": {
                "op": "equals",
                "values": [
                  "chimera_monkey_unchanged_relative_to_control"
                ]
              }
            }
          },
          {
            "name": "chimera_monkey_paper_claim_on_confounded_input",
            "role": "hard_fail",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "value",
            "expected": "chimera_monkey_shifts_toward_chimera_human_identity"
          },
          {
            "name": "chimera_human_magnitude_wrong_attribution_signature",
            "role": "hard_fail",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "value",
            "op": "in",
            "expected": [
              "dramatic_complete_replacement_of_origin_identity",
              "magnitude_dominated_by_culture_or_metabolic_axis",
              "magnitude_dominated_by_developmental_stage_axis",
              "magnitude_dominated_by_cell_cycle_axis",
              "magnitude_dominated_by_batch_effect_axis"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-5-claude-code",
      "modelName": "claude-opus-5",
      "harness": "claude-code",
      "task": "gse155381_q8_bidirectional_convergence",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": null,
      "cost": null,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "chimera_human_identity_shift_target_weighted",
            "role": "additive",
            "field": "$.chimera_human_identity_shift_target",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_in_vivo_state_assignment_weighted",
            "role": "additive",
            "field": "$.chimera_human_in_vivo_state_assignment",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_shift_magnitude_weighted",
            "role": "additive",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_monkey_identity_shift_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "both_arms_null_contradicted_by_calibration",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "chimera_human_identity_shift_target": {
                "op": "equals",
                "values": [
                  "chimera_human_maintains_origin_identity_no_measurable_shift"
                ]
              },
              "chimera_monkey_identity_shift": {
                "op": "equals",
                "values": [
                  "chimera_monkey_unchanged_relative_to_control"
                ]
              }
            }
          },
          {
            "name": "chimera_monkey_paper_claim_on_confounded_input",
            "role": "hard_fail",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "value",
            "expected": "chimera_monkey_shifts_toward_chimera_human_identity"
          },
          {
            "name": "chimera_human_magnitude_wrong_attribution_signature",
            "role": "hard_fail",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "value",
            "op": "in",
            "expected": [
              "dramatic_complete_replacement_of_origin_identity",
              "magnitude_dominated_by_culture_or_metabolic_axis",
              "magnitude_dominated_by_developmental_stage_axis",
              "magnitude_dominated_by_cell_cycle_axis",
              "magnitude_dominated_by_batch_effect_axis"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-5-claude-code",
      "modelName": "claude-opus-5",
      "harness": "claude-code",
      "task": "gse155381_q8_bidirectional_convergence",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 2151.69,
      "cost": 7.70546075,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "chimera_human_identity_shift_target_weighted",
            "role": "additive",
            "field": "$.chimera_human_identity_shift_target",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_in_vivo_state_assignment_weighted",
            "role": "additive",
            "field": "$.chimera_human_in_vivo_state_assignment",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_shift_magnitude_weighted",
            "role": "additive",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_monkey_identity_shift_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "both_arms_null_contradicted_by_calibration",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "chimera_human_identity_shift_target": {
                "op": "equals",
                "values": [
                  "chimera_human_maintains_origin_identity_no_measurable_shift"
                ]
              },
              "chimera_monkey_identity_shift": {
                "op": "equals",
                "values": [
                  "chimera_monkey_unchanged_relative_to_control"
                ]
              }
            }
          },
          {
            "name": "chimera_monkey_paper_claim_on_confounded_input",
            "role": "hard_fail",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "value",
            "expected": "chimera_monkey_shifts_toward_chimera_human_identity"
          },
          {
            "name": "chimera_human_magnitude_wrong_attribution_signature",
            "role": "hard_fail",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "value",
            "op": "in",
            "expected": [
              "dramatic_complete_replacement_of_origin_identity",
              "magnitude_dominated_by_culture_or_metabolic_axis",
              "magnitude_dominated_by_developmental_stage_axis",
              "magnitude_dominated_by_cell_cycle_axis",
              "magnitude_dominated_by_batch_effect_axis"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-5-claude-code",
      "modelName": "claude-opus-5",
      "harness": "claude-code",
      "task": "gse171524_ectopic_airway_rare_cell",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 1,
      "passed": true,
      "score": null,
      "durationS": 4571.3,
      "cost": 9.25611575,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "focus_population_identity_gate",
            "role": "gate",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "chemosensory_marker_expressing_population_alpha"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "focus_population_marker_calls",
            "kind": "list",
            "matchKey": "marker",
            "k": 3,
            "expected": [
              {
                "key": "POU2F3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "CHAT",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "TRPM5",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "LRMP",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNAT3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "AVIL",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNG13",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              }
            ]
          },
          {
            "name": "focus_population_condition_distribution_gate",
            "role": "gate",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_enriched_with_substantial_control_presence",
              "covid_predominant",
              "covid_only",
              "covid_enriched_trend_not_significant"
            ]
          },
          {
            "name": "focus_population_location_gate",
            "role": "gate",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "ectopic_parenchyma_in_covid",
              "parenchyma_only",
              "bronchiolized_or_peribronchial_parenchyma",
              "airway_and_parenchyma"
            ]
          },
          {
            "name": "focus_population_recovery_mode_gate",
            "role": "gate",
            "field": "$.focus_population_recovery_mode",
            "kind": "value",
            "op": "in",
            "expected": [
              "continuum_not_discrete_cluster"
            ]
          },
          {
            "name": "focus_population_identity_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "ionocyte_like_beta",
              "neuroendocrine_like_gamma",
              "basal_stem_progenitor_like_delta",
              "aberrant_basaloid_transitional_like_epsilon",
              "neuronal_or_neural_gene_minor_population_zeta",
              "deuterosomal_ciliated_progenitor_like_eta",
              "hillock_transitional_like_theta",
              "low_quality_doublet_or_debris",
              "no_rare_cell_recovered",
              "not_assessed"
            ]
          },
          {
            "name": "cf_wrong_lineage_marker_called_positive",
            "role": "hard_fail",
            "field": "$.focus_population_marker_calls[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "focus_population_condition_distribution_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "equally_distributed",
              "control_predominant",
              "not_assessed"
            ]
          },
          {
            "name": "focus_population_location_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "airway_only",
              "not_assessed"
            ]
          },
          {
            "name": "cross_axis_canonical_identity_with_airway_only_location_incoherence",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "focus_population_identity": {
                "op": "in",
                "values": [
                  "chemosensory_marker_expressing_population_alpha"
                ]
              },
              "focus_population_location": {
                "op": "in",
                "values": [
                  "airway_only"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-5-claude-code",
      "modelName": "claude-opus-5",
      "harness": "claude-code",
      "task": "gse171524_ectopic_airway_rare_cell",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 2442.49,
      "cost": 5.44163425,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "focus_population_identity_gate",
            "role": "gate",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "chemosensory_marker_expressing_population_alpha"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "focus_population_marker_calls",
            "kind": "list",
            "matchKey": "marker",
            "k": 3,
            "expected": [
              {
                "key": "POU2F3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "CHAT",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "TRPM5",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "LRMP",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNAT3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "AVIL",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNG13",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              }
            ]
          },
          {
            "name": "focus_population_condition_distribution_gate",
            "role": "gate",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_enriched_with_substantial_control_presence",
              "covid_predominant",
              "covid_only",
              "covid_enriched_trend_not_significant"
            ]
          },
          {
            "name": "focus_population_location_gate",
            "role": "gate",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "ectopic_parenchyma_in_covid",
              "parenchyma_only",
              "bronchiolized_or_peribronchial_parenchyma",
              "airway_and_parenchyma"
            ]
          },
          {
            "name": "focus_population_recovery_mode_gate",
            "role": "gate",
            "field": "$.focus_population_recovery_mode",
            "kind": "value",
            "op": "in",
            "expected": [
              "continuum_not_discrete_cluster"
            ]
          },
          {
            "name": "focus_population_identity_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "ionocyte_like_beta",
              "neuroendocrine_like_gamma",
              "basal_stem_progenitor_like_delta",
              "aberrant_basaloid_transitional_like_epsilon",
              "neuronal_or_neural_gene_minor_population_zeta",
              "deuterosomal_ciliated_progenitor_like_eta",
              "hillock_transitional_like_theta",
              "low_quality_doublet_or_debris",
              "no_rare_cell_recovered",
              "not_assessed"
            ]
          },
          {
            "name": "cf_wrong_lineage_marker_called_positive",
            "role": "hard_fail",
            "field": "$.focus_population_marker_calls[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "focus_population_condition_distribution_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "equally_distributed",
              "control_predominant",
              "not_assessed"
            ]
          },
          {
            "name": "focus_population_location_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "airway_only",
              "not_assessed"
            ]
          },
          {
            "name": "cross_axis_canonical_identity_with_airway_only_location_incoherence",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "focus_population_identity": {
                "op": "in",
                "values": [
                  "chemosensory_marker_expressing_population_alpha"
                ]
              },
              "focus_population_location": {
                "op": "in",
                "values": [
                  "airway_only"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-5-claude-code",
      "modelName": "claude-opus-5",
      "harness": "claude-code",
      "task": "gse171524_ectopic_airway_rare_cell",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 2648.3,
      "cost": 5.245062750000001,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "focus_population_identity_gate",
            "role": "gate",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "chemosensory_marker_expressing_population_alpha"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "focus_population_marker_calls",
            "kind": "list",
            "matchKey": "marker",
            "k": 3,
            "expected": [
              {
                "key": "POU2F3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "CHAT",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "TRPM5",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "LRMP",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNAT3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "AVIL",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNG13",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              }
            ]
          },
          {
            "name": "focus_population_condition_distribution_gate",
            "role": "gate",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_enriched_with_substantial_control_presence",
              "covid_predominant",
              "covid_only",
              "covid_enriched_trend_not_significant"
            ]
          },
          {
            "name": "focus_population_location_gate",
            "role": "gate",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "ectopic_parenchyma_in_covid",
              "parenchyma_only",
              "bronchiolized_or_peribronchial_parenchyma",
              "airway_and_parenchyma"
            ]
          },
          {
            "name": "focus_population_recovery_mode_gate",
            "role": "gate",
            "field": "$.focus_population_recovery_mode",
            "kind": "value",
            "op": "in",
            "expected": [
              "continuum_not_discrete_cluster"
            ]
          },
          {
            "name": "focus_population_identity_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "ionocyte_like_beta",
              "neuroendocrine_like_gamma",
              "basal_stem_progenitor_like_delta",
              "aberrant_basaloid_transitional_like_epsilon",
              "neuronal_or_neural_gene_minor_population_zeta",
              "deuterosomal_ciliated_progenitor_like_eta",
              "hillock_transitional_like_theta",
              "low_quality_doublet_or_debris",
              "no_rare_cell_recovered",
              "not_assessed"
            ]
          },
          {
            "name": "cf_wrong_lineage_marker_called_positive",
            "role": "hard_fail",
            "field": "$.focus_population_marker_calls[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "focus_population_condition_distribution_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "equally_distributed",
              "control_predominant",
              "not_assessed"
            ]
          },
          {
            "name": "focus_population_location_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "airway_only",
              "not_assessed"
            ]
          },
          {
            "name": "cross_axis_canonical_identity_with_airway_only_location_incoherence",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "focus_population_identity": {
                "op": "in",
                "values": [
                  "chemosensory_marker_expressing_population_alpha"
                ]
              },
              "focus_population_location": {
                "op": "in",
                "values": [
                  "airway_only"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-5-claude-code",
      "modelName": "claude-opus-5",
      "harness": "claude-code",
      "task": "gse171524_pathological_fibroblast_master_regulator_druggable_targets",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 1,
      "passed": true,
      "score": null,
      "durationS": null,
      "cost": null,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "disease_associated_fibroblast_subtype_weighted",
            "role": "additive",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "mr_tf_junb_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_alpha"
            ]
          },
          {
            "name": "mr_tf_jund_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_beta"
            ]
          },
          {
            "name": "drug_mmp14_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_alpha"
            ]
          },
          {
            "name": "drug_stat3_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_beta"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_weighted",
            "role": "additive",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "disease_associated_fibroblast_subtype_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "value",
            "op": "in",
            "expected": [
              "subtype_lambda",
              "subtype_epsilon",
              "subtype_iota",
              "subtype_beta",
              "subtype_eta",
              "subtype_zeta",
              "subtype_not_assessed"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "simple_de_ranking_by_log_fc",
              "marker_based_de_only",
              "not_computed"
            ]
          },
          {
            "name": "top_master_regulator_transcription_factors_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_not_computed"
            ]
          },
          {
            "name": "top_druggable_proteins_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_not_assessed"
            ]
          },
          {
            "name": "cross_axis_circular_marker_in_druggable_proteins",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_gamma",
              "drug_target_delta",
              "drug_target_eta"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-5-claude-code",
      "modelName": "claude-opus-5",
      "harness": "claude-code",
      "task": "gse171524_pathological_fibroblast_master_regulator_druggable_targets",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 2244,
      "cost": 8.65447725,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "disease_associated_fibroblast_subtype_weighted",
            "role": "additive",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "mr_tf_junb_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_alpha"
            ]
          },
          {
            "name": "mr_tf_jund_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_beta"
            ]
          },
          {
            "name": "drug_mmp14_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_alpha"
            ]
          },
          {
            "name": "drug_stat3_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_beta"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_weighted",
            "role": "additive",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "disease_associated_fibroblast_subtype_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "value",
            "op": "in",
            "expected": [
              "subtype_lambda",
              "subtype_epsilon",
              "subtype_iota",
              "subtype_beta",
              "subtype_eta",
              "subtype_zeta",
              "subtype_not_assessed"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "simple_de_ranking_by_log_fc",
              "marker_based_de_only",
              "not_computed"
            ]
          },
          {
            "name": "top_master_regulator_transcription_factors_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_not_computed"
            ]
          },
          {
            "name": "top_druggable_proteins_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_not_assessed"
            ]
          },
          {
            "name": "cross_axis_circular_marker_in_druggable_proteins",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_gamma",
              "drug_target_delta",
              "drug_target_eta"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-5-claude-code",
      "modelName": "claude-opus-5",
      "harness": "claude-code",
      "task": "gse171524_pathological_fibroblast_master_regulator_druggable_targets",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 6937.96,
      "cost": 26.833112250000003,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "disease_associated_fibroblast_subtype_weighted",
            "role": "additive",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "mr_tf_junb_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_alpha"
            ]
          },
          {
            "name": "mr_tf_jund_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_beta"
            ]
          },
          {
            "name": "drug_mmp14_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_alpha"
            ]
          },
          {
            "name": "drug_stat3_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_beta"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_weighted",
            "role": "additive",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "disease_associated_fibroblast_subtype_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "value",
            "op": "in",
            "expected": [
              "subtype_lambda",
              "subtype_epsilon",
              "subtype_iota",
              "subtype_beta",
              "subtype_eta",
              "subtype_zeta",
              "subtype_not_assessed"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "simple_de_ranking_by_log_fc",
              "marker_based_de_only",
              "not_computed"
            ]
          },
          {
            "name": "top_master_regulator_transcription_factors_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_not_computed"
            ]
          },
          {
            "name": "top_druggable_proteins_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_not_assessed"
            ]
          },
          {
            "name": "cross_axis_circular_marker_in_druggable_proteins",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_gamma",
              "drug_target_delta",
              "drug_target_eta"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-5-claude-code",
      "modelName": "claude-opus-5",
      "harness": "claude-code",
      "task": "gse171524_plasma_bcr_neutralizing_clonotype",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 1,
      "passed": true,
      "score": null,
      "durationS": 1329.84,
      "cost": 10.262113,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dominant_pairing_heavy_gate",
            "role": "gate",
            "field": "$.dominant_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV4-34"
            ]
          },
          {
            "name": "dominant_pairing_light_gate",
            "role": "gate",
            "field": "$.dominant_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGLV3-19"
            ]
          },
          {
            "name": "dominant_neutralizer_match_gate",
            "role": "gate",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_known_neutralizer_match"
            ]
          },
          {
            "name": "neutralizer_pairing_heavy_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV1-18"
            ]
          },
          {
            "name": "neutralizer_pairing_light_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGKV3-20"
            ]
          },
          {
            "name": "neutralizer_recurrence_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "observed_in_2_to_5_donors"
            ]
          },
          {
            "name": "neutralizer_rank_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "top_5_pairing",
              "top_6_to_20_pairing"
            ]
          },
          {
            "name": "dominant_clone_isotype_gate",
            "role": "gate",
            "field": "$.dominant_clone_isotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "IgG"
            ]
          },
          {
            "name": "blineage_class_switch_contrast_gate",
            "role": "gate",
            "field": "$.blineage_class_switch_contrast",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_class_switched_control_higher_unswitched",
              "class_switched_in_both_cohorts"
            ]
          },
          {
            "name": "dominant_match_narrative_trap_or_null",
            "role": "hard_fail",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_rbd_neutralizer",
              "matches_known_autoreactive_clone",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_heavy",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_light",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_recurrence_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_rank_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "cross_axis_found_dominant_but_failed_to_scan_for_embedded_neutralizer",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_pairing_neutralizer_match": {
                "op": "in",
                "values": [
                  "no_known_neutralizer_match"
                ]
              },
              "neutralizer_pairing_heavy": {
                "op": "in",
                "values": [
                  "none_detected"
                ]
              }
            }
          },
          {
            "name": "neutralizer_pairing_not_exactly_the_s309_pair",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "op": "not",
            "expected": {
              "op": "and",
              "args": [
                {
                  "op": "field",
                  "name": "neutralizer_pairing_heavy",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGHV1-18"
                    ]
                  }
                },
                {
                  "op": "field",
                  "name": "neutralizer_pairing_light",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGKV3-20"
                    ]
                  }
                }
              ]
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-5-claude-code",
      "modelName": "claude-opus-5",
      "harness": "claude-code",
      "task": "gse171524_plasma_bcr_neutralizing_clonotype",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 881.18,
      "cost": 2.743596250000001,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dominant_pairing_heavy_gate",
            "role": "gate",
            "field": "$.dominant_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV4-34"
            ]
          },
          {
            "name": "dominant_pairing_light_gate",
            "role": "gate",
            "field": "$.dominant_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGLV3-19"
            ]
          },
          {
            "name": "dominant_neutralizer_match_gate",
            "role": "gate",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_known_neutralizer_match"
            ]
          },
          {
            "name": "neutralizer_pairing_heavy_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV1-18"
            ]
          },
          {
            "name": "neutralizer_pairing_light_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGKV3-20"
            ]
          },
          {
            "name": "neutralizer_recurrence_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "observed_in_2_to_5_donors"
            ]
          },
          {
            "name": "neutralizer_rank_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "top_5_pairing",
              "top_6_to_20_pairing"
            ]
          },
          {
            "name": "dominant_clone_isotype_gate",
            "role": "gate",
            "field": "$.dominant_clone_isotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "IgG"
            ]
          },
          {
            "name": "blineage_class_switch_contrast_gate",
            "role": "gate",
            "field": "$.blineage_class_switch_contrast",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_class_switched_control_higher_unswitched",
              "class_switched_in_both_cohorts"
            ]
          },
          {
            "name": "dominant_match_narrative_trap_or_null",
            "role": "hard_fail",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_rbd_neutralizer",
              "matches_known_autoreactive_clone",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_heavy",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_light",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_recurrence_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_rank_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "cross_axis_found_dominant_but_failed_to_scan_for_embedded_neutralizer",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_pairing_neutralizer_match": {
                "op": "in",
                "values": [
                  "no_known_neutralizer_match"
                ]
              },
              "neutralizer_pairing_heavy": {
                "op": "in",
                "values": [
                  "none_detected"
                ]
              }
            }
          },
          {
            "name": "neutralizer_pairing_not_exactly_the_s309_pair",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "op": "not",
            "expected": {
              "op": "and",
              "args": [
                {
                  "op": "field",
                  "name": "neutralizer_pairing_heavy",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGHV1-18"
                    ]
                  }
                },
                {
                  "op": "field",
                  "name": "neutralizer_pairing_light",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGKV3-20"
                    ]
                  }
                }
              ]
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-5-claude-code",
      "modelName": "claude-opus-5",
      "harness": "claude-code",
      "task": "gse171524_plasma_bcr_neutralizing_clonotype",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 3121.93,
      "cost": 16.666481500000003,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dominant_pairing_heavy_gate",
            "role": "gate",
            "field": "$.dominant_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV4-34"
            ]
          },
          {
            "name": "dominant_pairing_light_gate",
            "role": "gate",
            "field": "$.dominant_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGLV3-19"
            ]
          },
          {
            "name": "dominant_neutralizer_match_gate",
            "role": "gate",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_known_neutralizer_match"
            ]
          },
          {
            "name": "neutralizer_pairing_heavy_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV1-18"
            ]
          },
          {
            "name": "neutralizer_pairing_light_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGKV3-20"
            ]
          },
          {
            "name": "neutralizer_recurrence_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "observed_in_2_to_5_donors"
            ]
          },
          {
            "name": "neutralizer_rank_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "top_5_pairing",
              "top_6_to_20_pairing"
            ]
          },
          {
            "name": "dominant_clone_isotype_gate",
            "role": "gate",
            "field": "$.dominant_clone_isotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "IgG"
            ]
          },
          {
            "name": "blineage_class_switch_contrast_gate",
            "role": "gate",
            "field": "$.blineage_class_switch_contrast",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_class_switched_control_higher_unswitched",
              "class_switched_in_both_cohorts"
            ]
          },
          {
            "name": "dominant_match_narrative_trap_or_null",
            "role": "hard_fail",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_rbd_neutralizer",
              "matches_known_autoreactive_clone",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_heavy",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_light",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_recurrence_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_rank_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "cross_axis_found_dominant_but_failed_to_scan_for_embedded_neutralizer",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_pairing_neutralizer_match": {
                "op": "in",
                "values": [
                  "no_known_neutralizer_match"
                ]
              },
              "neutralizer_pairing_heavy": {
                "op": "in",
                "values": [
                  "none_detected"
                ]
              }
            }
          },
          {
            "name": "neutralizer_pairing_not_exactly_the_s309_pair",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "op": "not",
            "expected": {
              "op": "and",
              "args": [
                {
                  "op": "field",
                  "name": "neutralizer_pairing_heavy",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGHV1-18"
                    ]
                  }
                },
                {
                  "op": "field",
                  "name": "neutralizer_pairing_light",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGKV3-20"
                    ]
                  }
                }
              ]
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-5-claude-code",
      "modelName": "claude-opus-5",
      "harness": "claude-code",
      "task": "q6_myeloid_subset_restricted_de",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 1,
      "passed": true,
      "score": null,
      "durationS": 2933.09,
      "cost": 5.787970499999998,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "myeloid_subset_with_strongest_change_gate",
            "role": "gate",
            "field": "$.myeloid_subset_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "subset_gamma"
            ]
          },
          {
            "name": "gene_with_strongest_change_gate",
            "role": "gate",
            "field": "$.gene_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "AXL",
              "PROS1"
            ]
          },
          {
            "name": "direction_of_change_gate",
            "role": "gate",
            "field": "$.direction_of_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "down_in_covid"
            ]
          },
          {
            "name": "biological_function_interpretation_gate",
            "role": "gate",
            "field": "$.biological_function_interpretation",
            "kind": "value",
            "op": "in",
            "expected": [
              "efferocytosis",
              "phagocytosis"
            ]
          },
          {
            "name": "myeloid_compartment_abundance_gate",
            "role": "gate",
            "field": "$.myeloid_compartment_abundance",
            "kind": "value",
            "op": "in",
            "expected": [
              "expands_in_covid"
            ]
          },
          {
            "name": "myeloid_expansion_localization_gate",
            "role": "gate",
            "field": "$.myeloid_expansion_localization",
            "kind": "value",
            "op": "in",
            "expected": [
              "recruited_driven_resident_dysfunction"
            ]
          },
          {
            "name": "dominant_signature_gate",
            "role": "gate",
            "field": "$.dominant_signature",
            "kind": "value",
            "op": "in",
            "expected": [
              "technical_or_agonal_artifact"
            ]
          },
          {
            "name": "dominant_signature_direction_gate",
            "role": "gate",
            "field": "$.dominant_signature_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_in_covid"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-5-claude-code",
      "modelName": "claude-opus-5",
      "harness": "claude-code",
      "task": "q6_myeloid_subset_restricted_de",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 2,
      "passed": true,
      "score": null,
      "durationS": 3093.75,
      "cost": 9.6037675,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "myeloid_subset_with_strongest_change_gate",
            "role": "gate",
            "field": "$.myeloid_subset_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "subset_gamma"
            ]
          },
          {
            "name": "gene_with_strongest_change_gate",
            "role": "gate",
            "field": "$.gene_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "AXL",
              "PROS1"
            ]
          },
          {
            "name": "direction_of_change_gate",
            "role": "gate",
            "field": "$.direction_of_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "down_in_covid"
            ]
          },
          {
            "name": "biological_function_interpretation_gate",
            "role": "gate",
            "field": "$.biological_function_interpretation",
            "kind": "value",
            "op": "in",
            "expected": [
              "efferocytosis",
              "phagocytosis"
            ]
          },
          {
            "name": "myeloid_compartment_abundance_gate",
            "role": "gate",
            "field": "$.myeloid_compartment_abundance",
            "kind": "value",
            "op": "in",
            "expected": [
              "expands_in_covid"
            ]
          },
          {
            "name": "myeloid_expansion_localization_gate",
            "role": "gate",
            "field": "$.myeloid_expansion_localization",
            "kind": "value",
            "op": "in",
            "expected": [
              "recruited_driven_resident_dysfunction"
            ]
          },
          {
            "name": "dominant_signature_gate",
            "role": "gate",
            "field": "$.dominant_signature",
            "kind": "value",
            "op": "in",
            "expected": [
              "technical_or_agonal_artifact"
            ]
          },
          {
            "name": "dominant_signature_direction_gate",
            "role": "gate",
            "field": "$.dominant_signature_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_in_covid"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-5-claude-code",
      "modelName": "claude-opus-5",
      "harness": "claude-code",
      "task": "q6_myeloid_subset_restricted_de",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 18189.64,
      "cost": 9.936704499999998,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "myeloid_subset_with_strongest_change_gate",
            "role": "gate",
            "field": "$.myeloid_subset_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "subset_gamma"
            ]
          },
          {
            "name": "gene_with_strongest_change_gate",
            "role": "gate",
            "field": "$.gene_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "AXL",
              "PROS1"
            ]
          },
          {
            "name": "direction_of_change_gate",
            "role": "gate",
            "field": "$.direction_of_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "down_in_covid"
            ]
          },
          {
            "name": "biological_function_interpretation_gate",
            "role": "gate",
            "field": "$.biological_function_interpretation",
            "kind": "value",
            "op": "in",
            "expected": [
              "efferocytosis",
              "phagocytosis"
            ]
          },
          {
            "name": "myeloid_compartment_abundance_gate",
            "role": "gate",
            "field": "$.myeloid_compartment_abundance",
            "kind": "value",
            "op": "in",
            "expected": [
              "expands_in_covid"
            ]
          },
          {
            "name": "myeloid_expansion_localization_gate",
            "role": "gate",
            "field": "$.myeloid_expansion_localization",
            "kind": "value",
            "op": "in",
            "expected": [
              "recruited_driven_resident_dysfunction"
            ]
          },
          {
            "name": "dominant_signature_gate",
            "role": "gate",
            "field": "$.dominant_signature",
            "kind": "value",
            "op": "in",
            "expected": [
              "technical_or_agonal_artifact"
            ]
          },
          {
            "name": "dominant_signature_direction_gate",
            "role": "gate",
            "field": "$.dominant_signature_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_in_covid"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-5-claude-code",
      "modelName": "claude-opus-5",
      "harness": "claude-code",
      "task": "snrnaseq_per_donor_composition_shifts_lethal_covid_lung",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": null,
      "cost": null,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dict_match",
            "role": "gate",
            "field": "cell_type_direction_map",
            "kind": "dict",
            "expected": {
              "Myeloid": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "Epithelia": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Fibroblasts": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "T_NK": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "B_plasma": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Endothelial": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Mast": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "APC": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Neuronal": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              }
            }
          },
          {
            "name": "over_merge_immune_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Immune",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_leukocyte_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Leukocyte",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_lymphoid_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Lymphoid",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_stromal_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Stromal",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-5-claude-code",
      "modelName": "claude-opus-5",
      "harness": "claude-code",
      "task": "snrnaseq_per_donor_composition_shifts_lethal_covid_lung",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 4268.1,
      "cost": 6.48255125,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dict_match",
            "role": "gate",
            "field": "cell_type_direction_map",
            "kind": "dict",
            "expected": {
              "Myeloid": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "Epithelia": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Fibroblasts": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "T_NK": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "B_plasma": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Endothelial": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Mast": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "APC": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Neuronal": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              }
            }
          },
          {
            "name": "over_merge_immune_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Immune",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_leukocyte_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Leukocyte",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_lymphoid_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Lymphoid",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_stromal_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Stromal",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-5-claude-code",
      "modelName": "claude-opus-5",
      "harness": "claude-code",
      "task": "snrnaseq_per_donor_composition_shifts_lethal_covid_lung",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 2764.39,
      "cost": 26.835416250000005,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dict_match",
            "role": "gate",
            "field": "cell_type_direction_map",
            "kind": "dict",
            "expected": {
              "Myeloid": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "Epithelia": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Fibroblasts": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "T_NK": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "B_plasma": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Endothelial": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Mast": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "APC": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Neuronal": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              }
            }
          },
          {
            "name": "over_merge_immune_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Immune",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_leukocyte_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Leukocyte",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_lymphoid_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Lymphoid",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_stromal_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Stromal",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-5-claude-code",
      "modelName": "claude-opus-5",
      "harness": "claude-code",
      "task": "10x_5p_melanoma_cluster_enriched_apc_states_v2",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 1,
      "passed": true,
      "score": null,
      "durationS": 3461.87,
      "cost": 9.9900475,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "top_3_cluster_enriched_apc_states",
            "kind": "list",
            "matchKey": "apc_state",
            "k": 3,
            "expected": [
              {
                "key": "plasmacytoid_DC",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "T_cell_stimulatory_role_supported",
                      "role_direction_unresolved"
                    ],
                    "canonical": "T_cell_stimulatory_role_supported"
                  }
                }
              },
              {
                "key": "C1qhigh_lipid_associated_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "C1qhigh_inflammatory_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "plasma_cell",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "cohort_significant_per_patient_inconsistent"
                    ],
                    "canonical": "cohort_significant_per_patient_inconsistent"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CD80_CD86": {
                "op": "in",
                "value": [
                  "mreg_DC",
                  "shared_across_APC_states"
                ]
              },
              "PD_L1_in_APC_clusters": {
                "op": "in",
                "value": [
                  "C1qhigh_macrophage_either_subtype",
                  "shared_across_APC_states",
                  "mreg_DC"
                ]
              },
              "type_I_IFN": {
                "op": "in",
                "value": [
                  "plasmacytoid_DC",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "cf_mreg_dc_claimed_enriched",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "mreg_DC",
              "mature_DC_enriched_in_immunoregulatory_molecules"
            ]
          },
          {
            "name": "cf_framework_offcompartment_or_generic_substitution",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "cDC1",
              "Maier_mregDC",
              "Cheng_C1QC_TAM",
              "Cheng_SPP1_TAM",
              "Cheng_FOLR2_TAM",
              "Villani_pDC",
              "Zhang_inflammatory_monocyte",
              "LAMP3_pos_DC",
              "myeloid_derived_suppressor_cell",
              "MDSC",
              "tumor_associated_neutrophil",
              "mast_cell",
              "APC",
              "antigen_presenting_cell",
              "macrophage",
              "dendritic_cell",
              "tumor_associated_macrophage",
              "TAM"
            ]
          },
          {
            "name": "cf_direction_commit_on_unresolvable_slot",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_not_resolvable_hedge",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].functional_attribution_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_type_I_IFN_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.type_I_IFN",
            "kind": "value",
            "op": "in",
            "expected": [
              "mreg_DC",
              "C1qhigh_macrophage_either_subtype"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-5-claude-code",
      "modelName": "claude-opus-5",
      "harness": "claude-code",
      "task": "10x_5p_melanoma_cluster_enriched_apc_states_v2",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 2596.61,
      "cost": 7.5345027500000015,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "top_3_cluster_enriched_apc_states",
            "kind": "list",
            "matchKey": "apc_state",
            "k": 3,
            "expected": [
              {
                "key": "plasmacytoid_DC",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "T_cell_stimulatory_role_supported",
                      "role_direction_unresolved"
                    ],
                    "canonical": "T_cell_stimulatory_role_supported"
                  }
                }
              },
              {
                "key": "C1qhigh_lipid_associated_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "C1qhigh_inflammatory_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "plasma_cell",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "cohort_significant_per_patient_inconsistent"
                    ],
                    "canonical": "cohort_significant_per_patient_inconsistent"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CD80_CD86": {
                "op": "in",
                "value": [
                  "mreg_DC",
                  "shared_across_APC_states"
                ]
              },
              "PD_L1_in_APC_clusters": {
                "op": "in",
                "value": [
                  "C1qhigh_macrophage_either_subtype",
                  "shared_across_APC_states",
                  "mreg_DC"
                ]
              },
              "type_I_IFN": {
                "op": "in",
                "value": [
                  "plasmacytoid_DC",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "cf_mreg_dc_claimed_enriched",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "mreg_DC",
              "mature_DC_enriched_in_immunoregulatory_molecules"
            ]
          },
          {
            "name": "cf_framework_offcompartment_or_generic_substitution",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "cDC1",
              "Maier_mregDC",
              "Cheng_C1QC_TAM",
              "Cheng_SPP1_TAM",
              "Cheng_FOLR2_TAM",
              "Villani_pDC",
              "Zhang_inflammatory_monocyte",
              "LAMP3_pos_DC",
              "myeloid_derived_suppressor_cell",
              "MDSC",
              "tumor_associated_neutrophil",
              "mast_cell",
              "APC",
              "antigen_presenting_cell",
              "macrophage",
              "dendritic_cell",
              "tumor_associated_macrophage",
              "TAM"
            ]
          },
          {
            "name": "cf_direction_commit_on_unresolvable_slot",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_not_resolvable_hedge",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].functional_attribution_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_type_I_IFN_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.type_I_IFN",
            "kind": "value",
            "op": "in",
            "expected": [
              "mreg_DC",
              "C1qhigh_macrophage_either_subtype"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-5-claude-code",
      "modelName": "claude-opus-5",
      "harness": "claude-code",
      "task": "10x_5p_melanoma_cluster_enriched_apc_states_v2",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 3773.63,
      "cost": 8.7971685,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "top_3_cluster_enriched_apc_states",
            "kind": "list",
            "matchKey": "apc_state",
            "k": 3,
            "expected": [
              {
                "key": "plasmacytoid_DC",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "T_cell_stimulatory_role_supported",
                      "role_direction_unresolved"
                    ],
                    "canonical": "T_cell_stimulatory_role_supported"
                  }
                }
              },
              {
                "key": "C1qhigh_lipid_associated_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "C1qhigh_inflammatory_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "plasma_cell",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "cohort_significant_per_patient_inconsistent"
                    ],
                    "canonical": "cohort_significant_per_patient_inconsistent"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CD80_CD86": {
                "op": "in",
                "value": [
                  "mreg_DC",
                  "shared_across_APC_states"
                ]
              },
              "PD_L1_in_APC_clusters": {
                "op": "in",
                "value": [
                  "C1qhigh_macrophage_either_subtype",
                  "shared_across_APC_states",
                  "mreg_DC"
                ]
              },
              "type_I_IFN": {
                "op": "in",
                "value": [
                  "plasmacytoid_DC",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "cf_mreg_dc_claimed_enriched",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "mreg_DC",
              "mature_DC_enriched_in_immunoregulatory_molecules"
            ]
          },
          {
            "name": "cf_framework_offcompartment_or_generic_substitution",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "cDC1",
              "Maier_mregDC",
              "Cheng_C1QC_TAM",
              "Cheng_SPP1_TAM",
              "Cheng_FOLR2_TAM",
              "Villani_pDC",
              "Zhang_inflammatory_monocyte",
              "LAMP3_pos_DC",
              "myeloid_derived_suppressor_cell",
              "MDSC",
              "tumor_associated_neutrophil",
              "mast_cell",
              "APC",
              "antigen_presenting_cell",
              "macrophage",
              "dendritic_cell",
              "tumor_associated_macrophage",
              "TAM"
            ]
          },
          {
            "name": "cf_direction_commit_on_unresolvable_slot",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_not_resolvable_hedge",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].functional_attribution_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_type_I_IFN_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.type_I_IFN",
            "kind": "value",
            "op": "in",
            "expected": [
              "mreg_DC",
              "C1qhigh_macrophage_either_subtype"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-5-claude-code",
      "modelName": "claude-opus-5",
      "harness": "claude-code",
      "task": "10x_5p_melanoma_cluster_tumor_state_causation_v5",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 1,
      "passed": true,
      "score": null,
      "durationS": 2893.77,
      "cost": 13.380729500000005,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "enriched_tumor_states",
            "kind": "list",
            "matchKey": "tumor_state",
            "k": 3,
            "expected": [
              {
                "key": "immune_response",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "direction_unresolved_cross_sectional"
                    ],
                    "canonical": "direction_unresolved_cross_sectional"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "IFN_response_axis",
                      "broad_antigen_presentation_MHC"
                    ],
                    "canonical": "IFN_response_axis"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "enrichment_diminished_by_decontamination",
                      "threshold_sensitive"
                    ],
                    "canonical": "enrichment_diminished_by_decontamination"
                  }
                }
              },
              {
                "key": "stress_hypoxia",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "data_supports_cell_intrinsic_cause"
                    ],
                    "canonical": "data_supports_cell_intrinsic_cause"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "HIF_hypoxia_program_specific"
                    ],
                    "canonical": "HIF_hypoxia_program_specific"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "robust_across_methods",
                      "robust_after_decontamination"
                    ],
                    "canonical": "robust_across_methods"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CXCL9_CXCL10": {
                "op": "in",
                "value": [
                  "immune_response_tumor_state"
                ]
              },
              "CCL5": {
                "op": "in",
                "value": [
                  "shared_across_tumor_states",
                  "partner_lymphocyte_derived",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "gate_conjugate_gate_confounder",
            "role": "gate",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "partner_lymphocyte_capture"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "antigen_presentation_decomposition",
            "kind": "dict",
            "expected": {
              "MHC_class_I": {
                "op": "in",
                "value": [
                  "co_regulated_with_dominant_program",
                  "tumor_intrinsic_enriched_independent"
                ]
              },
              "MHC_class_II": {
                "op": "in",
                "value": [
                  "partner_cell_attributable",
                  "flat_not_enriched",
                  "method_sensitive_no_robust_call",
                  "co_regulated_with_dominant_program"
                ]
              }
            }
          },
          {
            "name": "cf_non_robust_state_claimed_enriched",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "transitory_melanocytic",
              "intermediate_melanocytic",
              "melanocytic",
              "MITF_high",
              "neural_crest_like",
              "mitotic",
              "stress_p53",
              "undifferentiated",
              "dedifferentiated",
              "mesenchymal_tumor",
              "AXL_high_tumor",
              "invasive_tumor",
              "senescence_like",
              "EMT_transition",
              "cancer_stem_cell_like",
              "oxidative_stress_tumor"
            ]
          },
          {
            "name": "cf_framework_generic_or_offtissue_substitution",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "Tsoi_undifferentiated",
              "Tsoi_neural_crest_like",
              "Tsoi_transitory",
              "Tsoi_melanocytic",
              "Rambow_NCSC",
              "Wouters_mesenchymal",
              "Pozniak_TCF4_dependent",
              "tumor_cell",
              "melanoma_cell",
              "malignant_cell",
              "tumor_subpopulation",
              "keratinocyte",
              "hepatocyte",
              "lung_adenocarcinoma_cell",
              "colon_adenocarcinoma_cell",
              "Tirosh_cell_cycle_program",
              "Jerby_Arnon_program",
              "glioblastoma_cell",
              "breast_carcinoma_cell"
            ]
          },
          {
            "name": "cf_immune_response_direction_committed",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_immune_robustness_recall",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_ambiguous_hedge",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].causation_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_stress_generic_stress_response_evidence",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_conjugate_confounder_contamination_denied",
            "role": "hard_fail",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected"
            ]
          },
          {
            "name": "cf_ccl5_recall_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CCL5",
            "kind": "avoid",
            "expected": [
              "immune_response_tumor_state"
            ]
          },
          {
            "name": "cf_cxcl9_cxcl10_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CXCL9_CXCL10",
            "kind": "avoid",
            "expected": [
              "stress_hypoxia_tumor_state"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-5-claude-code",
      "modelName": "claude-opus-5",
      "harness": "claude-code",
      "task": "10x_5p_melanoma_cluster_tumor_state_causation_v5",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 3833.24,
      "cost": 7.211267499999999,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "enriched_tumor_states",
            "kind": "list",
            "matchKey": "tumor_state",
            "k": 3,
            "expected": [
              {
                "key": "immune_response",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "direction_unresolved_cross_sectional"
                    ],
                    "canonical": "direction_unresolved_cross_sectional"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "IFN_response_axis",
                      "broad_antigen_presentation_MHC"
                    ],
                    "canonical": "IFN_response_axis"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "enrichment_diminished_by_decontamination",
                      "threshold_sensitive"
                    ],
                    "canonical": "enrichment_diminished_by_decontamination"
                  }
                }
              },
              {
                "key": "stress_hypoxia",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "data_supports_cell_intrinsic_cause"
                    ],
                    "canonical": "data_supports_cell_intrinsic_cause"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "HIF_hypoxia_program_specific"
                    ],
                    "canonical": "HIF_hypoxia_program_specific"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "robust_across_methods",
                      "robust_after_decontamination"
                    ],
                    "canonical": "robust_across_methods"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CXCL9_CXCL10": {
                "op": "in",
                "value": [
                  "immune_response_tumor_state"
                ]
              },
              "CCL5": {
                "op": "in",
                "value": [
                  "shared_across_tumor_states",
                  "partner_lymphocyte_derived",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "gate_conjugate_gate_confounder",
            "role": "gate",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "partner_lymphocyte_capture"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "antigen_presentation_decomposition",
            "kind": "dict",
            "expected": {
              "MHC_class_I": {
                "op": "in",
                "value": [
                  "co_regulated_with_dominant_program",
                  "tumor_intrinsic_enriched_independent"
                ]
              },
              "MHC_class_II": {
                "op": "in",
                "value": [
                  "partner_cell_attributable",
                  "flat_not_enriched",
                  "method_sensitive_no_robust_call",
                  "co_regulated_with_dominant_program"
                ]
              }
            }
          },
          {
            "name": "cf_non_robust_state_claimed_enriched",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "transitory_melanocytic",
              "intermediate_melanocytic",
              "melanocytic",
              "MITF_high",
              "neural_crest_like",
              "mitotic",
              "stress_p53",
              "undifferentiated",
              "dedifferentiated",
              "mesenchymal_tumor",
              "AXL_high_tumor",
              "invasive_tumor",
              "senescence_like",
              "EMT_transition",
              "cancer_stem_cell_like",
              "oxidative_stress_tumor"
            ]
          },
          {
            "name": "cf_framework_generic_or_offtissue_substitution",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "Tsoi_undifferentiated",
              "Tsoi_neural_crest_like",
              "Tsoi_transitory",
              "Tsoi_melanocytic",
              "Rambow_NCSC",
              "Wouters_mesenchymal",
              "Pozniak_TCF4_dependent",
              "tumor_cell",
              "melanoma_cell",
              "malignant_cell",
              "tumor_subpopulation",
              "keratinocyte",
              "hepatocyte",
              "lung_adenocarcinoma_cell",
              "colon_adenocarcinoma_cell",
              "Tirosh_cell_cycle_program",
              "Jerby_Arnon_program",
              "glioblastoma_cell",
              "breast_carcinoma_cell"
            ]
          },
          {
            "name": "cf_immune_response_direction_committed",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_immune_robustness_recall",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_ambiguous_hedge",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].causation_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_stress_generic_stress_response_evidence",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_conjugate_confounder_contamination_denied",
            "role": "hard_fail",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected"
            ]
          },
          {
            "name": "cf_ccl5_recall_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CCL5",
            "kind": "avoid",
            "expected": [
              "immune_response_tumor_state"
            ]
          },
          {
            "name": "cf_cxcl9_cxcl10_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CXCL9_CXCL10",
            "kind": "avoid",
            "expected": [
              "stress_hypoxia_tumor_state"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-5-claude-code",
      "modelName": "claude-opus-5",
      "harness": "claude-code",
      "task": "10x_5p_melanoma_cluster_tumor_state_causation_v5",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 3.07,
      "cost": 0,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "enriched_tumor_states",
            "kind": "list",
            "matchKey": "tumor_state",
            "k": 3,
            "expected": [
              {
                "key": "immune_response",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "direction_unresolved_cross_sectional"
                    ],
                    "canonical": "direction_unresolved_cross_sectional"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "IFN_response_axis",
                      "broad_antigen_presentation_MHC"
                    ],
                    "canonical": "IFN_response_axis"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "enrichment_diminished_by_decontamination",
                      "threshold_sensitive"
                    ],
                    "canonical": "enrichment_diminished_by_decontamination"
                  }
                }
              },
              {
                "key": "stress_hypoxia",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "data_supports_cell_intrinsic_cause"
                    ],
                    "canonical": "data_supports_cell_intrinsic_cause"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "HIF_hypoxia_program_specific"
                    ],
                    "canonical": "HIF_hypoxia_program_specific"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "robust_across_methods",
                      "robust_after_decontamination"
                    ],
                    "canonical": "robust_across_methods"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CXCL9_CXCL10": {
                "op": "in",
                "value": [
                  "immune_response_tumor_state"
                ]
              },
              "CCL5": {
                "op": "in",
                "value": [
                  "shared_across_tumor_states",
                  "partner_lymphocyte_derived",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "gate_conjugate_gate_confounder",
            "role": "gate",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "partner_lymphocyte_capture"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "antigen_presentation_decomposition",
            "kind": "dict",
            "expected": {
              "MHC_class_I": {
                "op": "in",
                "value": [
                  "co_regulated_with_dominant_program",
                  "tumor_intrinsic_enriched_independent"
                ]
              },
              "MHC_class_II": {
                "op": "in",
                "value": [
                  "partner_cell_attributable",
                  "flat_not_enriched",
                  "method_sensitive_no_robust_call",
                  "co_regulated_with_dominant_program"
                ]
              }
            }
          },
          {
            "name": "cf_non_robust_state_claimed_enriched",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "transitory_melanocytic",
              "intermediate_melanocytic",
              "melanocytic",
              "MITF_high",
              "neural_crest_like",
              "mitotic",
              "stress_p53",
              "undifferentiated",
              "dedifferentiated",
              "mesenchymal_tumor",
              "AXL_high_tumor",
              "invasive_tumor",
              "senescence_like",
              "EMT_transition",
              "cancer_stem_cell_like",
              "oxidative_stress_tumor"
            ]
          },
          {
            "name": "cf_framework_generic_or_offtissue_substitution",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "Tsoi_undifferentiated",
              "Tsoi_neural_crest_like",
              "Tsoi_transitory",
              "Tsoi_melanocytic",
              "Rambow_NCSC",
              "Wouters_mesenchymal",
              "Pozniak_TCF4_dependent",
              "tumor_cell",
              "melanoma_cell",
              "malignant_cell",
              "tumor_subpopulation",
              "keratinocyte",
              "hepatocyte",
              "lung_adenocarcinoma_cell",
              "colon_adenocarcinoma_cell",
              "Tirosh_cell_cycle_program",
              "Jerby_Arnon_program",
              "glioblastoma_cell",
              "breast_carcinoma_cell"
            ]
          },
          {
            "name": "cf_immune_response_direction_committed",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_immune_robustness_recall",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_ambiguous_hedge",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].causation_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_stress_generic_stress_response_evidence",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_conjugate_confounder_contamination_denied",
            "role": "hard_fail",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected"
            ]
          },
          {
            "name": "cf_ccl5_recall_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CCL5",
            "kind": "avoid",
            "expected": [
              "immune_response_tumor_state"
            ]
          },
          {
            "name": "cf_cxcl9_cxcl10_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CXCL9_CXCL10",
            "kind": "avoid",
            "expected": [
              "stress_hypoxia_tumor_state"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-5-claude-code",
      "modelName": "claude-opus-5",
      "harness": "claude-code",
      "task": "10x_5p_melanoma_cluster_vs_cd39_tumor_reactivity_dominance",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 1,
      "passed": true,
      "score": null,
      "durationS": 1466.27,
      "cost": 3.5485582500000006,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "population_findings",
            "kind": "list",
            "matchKey": "capture_gate",
            "k": 4,
            "expected": [
              {
                "key": "tumor_cell_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "apc_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "combined_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "unselected_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "naive",
                      "central_memory",
                      "effector_memory"
                    ],
                    "canonical": "naive"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "low"
                    ],
                    "canonical": "low"
                  },
                  "purity_caveat": {
                    "op": "equals",
                    "values": [
                      "pure_T_cell"
                    ],
                    "canonical": "pure_T_cell"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "no_reactivity_evidence"
                    ],
                    "canonical": "no_reactivity_evidence"
                  }
                }
              },
              {
                "key": "marker_sorted_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "equals",
                    "values": [
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "terminal_exhausted_LAG3_high"
                  },
                  "clonal_expansion_level": {
                    "op": "in",
                    "values": [
                      "high",
                      "intermediate"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "marker_on_non_T_lineages",
                      "single_cell_gating_loses_target"
                    ],
                    "canonical": "marker_on_non_T_lineages"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "marker_or_exhaustion_defined_only"
                    ],
                    "canonical": "marker_or_exhaustion_defined_only"
                  }
                }
              }
            ]
          },
          {
            "name": "best_capture_approach_gate",
            "role": "gate",
            "field": "best_capture_approach",
            "kind": "value",
            "expected": "heterotypic_cell_conjugate"
          },
          {
            "name": "best_selection_marker_gate",
            "role": "gate",
            "field": "best_selection_marker",
            "kind": "value",
            "expected": "none"
          },
          {
            "name": "reactivity_criterion_gate",
            "role": "gate",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "clonal_expansion_and_physical_engagement"
          },
          {
            "name": "cross_patient_robustness_gate",
            "role": "gate",
            "field": "cross_patient_robustness",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_across_patients",
              "marginal_in_some_patients"
            ]
          },
          {
            "name": "marker_selection_consequence_gate",
            "role": "gate",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "discards_clonally_expanded_reactive_cells"
          },
          {
            "name": "superiority_claim_basis_gate",
            "role": "gate",
            "field": "superiority_claim_basis",
            "kind": "value",
            "expected": "requires_functional_reactivity_assay"
          },
          {
            "name": "conjugate_enriched_state_gate",
            "role": "gate",
            "field": "conjugate_enriched_state",
            "kind": "value",
            "expected": "progenitor_exhausted_TCF7_pos_stem_like"
          },
          {
            "name": "cf_circular_exhaustion_criterion",
            "role": "hard_fail",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "exhaustion_signature_intensity"
          },
          {
            "name": "cf_marker_sort_prior",
            "role": "hard_fail",
            "field": "best_selection_marker",
            "kind": "value",
            "op": "in",
            "expected": [
              "CD39",
              "PD1",
              "PD1_high",
              "TIM3",
              "PD1_CD39",
              "CD39_CD103"
            ]
          },
          {
            "name": "cf_single_cell_capture_prior",
            "role": "hard_fail",
            "field": "best_capture_approach",
            "kind": "value",
            "op": "in",
            "expected": [
              "dissociated_single_cell",
              "dissociated_bulk_population"
            ]
          },
          {
            "name": "cf_off_data_capture_gate",
            "role": "hard_fail",
            "field": "$.population_findings[*].capture_gate",
            "kind": "avoid",
            "expected": [
              "dissociated_bulk",
              "matched_peripheral_blood",
              "matched_lymph_node",
              "post_hoc_expanded_clonotype_group"
            ]
          },
          {
            "name": "cf_conjugate_mischaracterized",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_marker_singlet_called_engaged",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "contains",
            "matchKey": "capture_gate",
            "entity": "marker_sorted_singlet",
            "valueField": "reactivity_evidence",
            "expected": [
              "engagement_and_clonal_expansion"
            ]
          },
          {
            "name": "cf_no_engaged_reactive_population",
            "role": "hard_fail",
            "field": "$.population_findings[*].reactivity_evidence",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_marker_captures_all_reactive",
            "role": "hard_fail",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "captures_full_reactive_repertoire"
          },
          {
            "name": "cf_superiority_overclaimed_from_phenotype",
            "role": "hard_fail",
            "field": "superiority_claim_basis",
            "kind": "value",
            "op": "in",
            "expected": [
              "established_by_scrna_phenotype",
              "established_by_reactivity_proxy_scores"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-5-claude-code",
      "modelName": "claude-opus-5",
      "harness": "claude-code",
      "task": "10x_5p_melanoma_cluster_vs_cd39_tumor_reactivity_dominance",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 2,
      "passed": true,
      "score": null,
      "durationS": 2162.12,
      "cost": 6.1016675,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "population_findings",
            "kind": "list",
            "matchKey": "capture_gate",
            "k": 4,
            "expected": [
              {
                "key": "tumor_cell_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "apc_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "combined_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "unselected_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "naive",
                      "central_memory",
                      "effector_memory"
                    ],
                    "canonical": "naive"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "low"
                    ],
                    "canonical": "low"
                  },
                  "purity_caveat": {
                    "op": "equals",
                    "values": [
                      "pure_T_cell"
                    ],
                    "canonical": "pure_T_cell"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "no_reactivity_evidence"
                    ],
                    "canonical": "no_reactivity_evidence"
                  }
                }
              },
              {
                "key": "marker_sorted_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "equals",
                    "values": [
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "terminal_exhausted_LAG3_high"
                  },
                  "clonal_expansion_level": {
                    "op": "in",
                    "values": [
                      "high",
                      "intermediate"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "marker_on_non_T_lineages",
                      "single_cell_gating_loses_target"
                    ],
                    "canonical": "marker_on_non_T_lineages"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "marker_or_exhaustion_defined_only"
                    ],
                    "canonical": "marker_or_exhaustion_defined_only"
                  }
                }
              }
            ]
          },
          {
            "name": "best_capture_approach_gate",
            "role": "gate",
            "field": "best_capture_approach",
            "kind": "value",
            "expected": "heterotypic_cell_conjugate"
          },
          {
            "name": "best_selection_marker_gate",
            "role": "gate",
            "field": "best_selection_marker",
            "kind": "value",
            "expected": "none"
          },
          {
            "name": "reactivity_criterion_gate",
            "role": "gate",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "clonal_expansion_and_physical_engagement"
          },
          {
            "name": "cross_patient_robustness_gate",
            "role": "gate",
            "field": "cross_patient_robustness",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_across_patients",
              "marginal_in_some_patients"
            ]
          },
          {
            "name": "marker_selection_consequence_gate",
            "role": "gate",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "discards_clonally_expanded_reactive_cells"
          },
          {
            "name": "superiority_claim_basis_gate",
            "role": "gate",
            "field": "superiority_claim_basis",
            "kind": "value",
            "expected": "requires_functional_reactivity_assay"
          },
          {
            "name": "conjugate_enriched_state_gate",
            "role": "gate",
            "field": "conjugate_enriched_state",
            "kind": "value",
            "expected": "progenitor_exhausted_TCF7_pos_stem_like"
          },
          {
            "name": "cf_circular_exhaustion_criterion",
            "role": "hard_fail",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "exhaustion_signature_intensity"
          },
          {
            "name": "cf_marker_sort_prior",
            "role": "hard_fail",
            "field": "best_selection_marker",
            "kind": "value",
            "op": "in",
            "expected": [
              "CD39",
              "PD1",
              "PD1_high",
              "TIM3",
              "PD1_CD39",
              "CD39_CD103"
            ]
          },
          {
            "name": "cf_single_cell_capture_prior",
            "role": "hard_fail",
            "field": "best_capture_approach",
            "kind": "value",
            "op": "in",
            "expected": [
              "dissociated_single_cell",
              "dissociated_bulk_population"
            ]
          },
          {
            "name": "cf_off_data_capture_gate",
            "role": "hard_fail",
            "field": "$.population_findings[*].capture_gate",
            "kind": "avoid",
            "expected": [
              "dissociated_bulk",
              "matched_peripheral_blood",
              "matched_lymph_node",
              "post_hoc_expanded_clonotype_group"
            ]
          },
          {
            "name": "cf_conjugate_mischaracterized",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_marker_singlet_called_engaged",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "contains",
            "matchKey": "capture_gate",
            "entity": "marker_sorted_singlet",
            "valueField": "reactivity_evidence",
            "expected": [
              "engagement_and_clonal_expansion"
            ]
          },
          {
            "name": "cf_no_engaged_reactive_population",
            "role": "hard_fail",
            "field": "$.population_findings[*].reactivity_evidence",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_marker_captures_all_reactive",
            "role": "hard_fail",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "captures_full_reactive_repertoire"
          },
          {
            "name": "cf_superiority_overclaimed_from_phenotype",
            "role": "hard_fail",
            "field": "superiority_claim_basis",
            "kind": "value",
            "op": "in",
            "expected": [
              "established_by_scrna_phenotype",
              "established_by_reactivity_proxy_scores"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-5-claude-code",
      "modelName": "claude-opus-5",
      "harness": "claude-code",
      "task": "10x_5p_melanoma_cluster_vs_cd39_tumor_reactivity_dominance",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": true,
      "score": null,
      "durationS": 1917.77,
      "cost": 7.962293750000001,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "population_findings",
            "kind": "list",
            "matchKey": "capture_gate",
            "k": 4,
            "expected": [
              {
                "key": "tumor_cell_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "apc_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "combined_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "unselected_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "naive",
                      "central_memory",
                      "effector_memory"
                    ],
                    "canonical": "naive"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "low"
                    ],
                    "canonical": "low"
                  },
                  "purity_caveat": {
                    "op": "equals",
                    "values": [
                      "pure_T_cell"
                    ],
                    "canonical": "pure_T_cell"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "no_reactivity_evidence"
                    ],
                    "canonical": "no_reactivity_evidence"
                  }
                }
              },
              {
                "key": "marker_sorted_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "equals",
                    "values": [
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "terminal_exhausted_LAG3_high"
                  },
                  "clonal_expansion_level": {
                    "op": "in",
                    "values": [
                      "high",
                      "intermediate"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "marker_on_non_T_lineages",
                      "single_cell_gating_loses_target"
                    ],
                    "canonical": "marker_on_non_T_lineages"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "marker_or_exhaustion_defined_only"
                    ],
                    "canonical": "marker_or_exhaustion_defined_only"
                  }
                }
              }
            ]
          },
          {
            "name": "best_capture_approach_gate",
            "role": "gate",
            "field": "best_capture_approach",
            "kind": "value",
            "expected": "heterotypic_cell_conjugate"
          },
          {
            "name": "best_selection_marker_gate",
            "role": "gate",
            "field": "best_selection_marker",
            "kind": "value",
            "expected": "none"
          },
          {
            "name": "reactivity_criterion_gate",
            "role": "gate",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "clonal_expansion_and_physical_engagement"
          },
          {
            "name": "cross_patient_robustness_gate",
            "role": "gate",
            "field": "cross_patient_robustness",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_across_patients",
              "marginal_in_some_patients"
            ]
          },
          {
            "name": "marker_selection_consequence_gate",
            "role": "gate",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "discards_clonally_expanded_reactive_cells"
          },
          {
            "name": "superiority_claim_basis_gate",
            "role": "gate",
            "field": "superiority_claim_basis",
            "kind": "value",
            "expected": "requires_functional_reactivity_assay"
          },
          {
            "name": "conjugate_enriched_state_gate",
            "role": "gate",
            "field": "conjugate_enriched_state",
            "kind": "value",
            "expected": "progenitor_exhausted_TCF7_pos_stem_like"
          },
          {
            "name": "cf_circular_exhaustion_criterion",
            "role": "hard_fail",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "exhaustion_signature_intensity"
          },
          {
            "name": "cf_marker_sort_prior",
            "role": "hard_fail",
            "field": "best_selection_marker",
            "kind": "value",
            "op": "in",
            "expected": [
              "CD39",
              "PD1",
              "PD1_high",
              "TIM3",
              "PD1_CD39",
              "CD39_CD103"
            ]
          },
          {
            "name": "cf_single_cell_capture_prior",
            "role": "hard_fail",
            "field": "best_capture_approach",
            "kind": "value",
            "op": "in",
            "expected": [
              "dissociated_single_cell",
              "dissociated_bulk_population"
            ]
          },
          {
            "name": "cf_off_data_capture_gate",
            "role": "hard_fail",
            "field": "$.population_findings[*].capture_gate",
            "kind": "avoid",
            "expected": [
              "dissociated_bulk",
              "matched_peripheral_blood",
              "matched_lymph_node",
              "post_hoc_expanded_clonotype_group"
            ]
          },
          {
            "name": "cf_conjugate_mischaracterized",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_marker_singlet_called_engaged",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "contains",
            "matchKey": "capture_gate",
            "entity": "marker_sorted_singlet",
            "valueField": "reactivity_evidence",
            "expected": [
              "engagement_and_clonal_expansion"
            ]
          },
          {
            "name": "cf_no_engaged_reactive_population",
            "role": "hard_fail",
            "field": "$.population_findings[*].reactivity_evidence",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_marker_captures_all_reactive",
            "role": "hard_fail",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "captures_full_reactive_repertoire"
          },
          {
            "name": "cf_superiority_overclaimed_from_phenotype",
            "role": "hard_fail",
            "field": "superiority_claim_basis",
            "kind": "value",
            "op": "in",
            "expected": [
              "established_by_scrna_phenotype",
              "established_by_reactivity_proxy_scores"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-5-claude-code",
      "modelName": "claude-opus-5",
      "harness": "claude-code",
      "task": "10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 1,
      "passed": true,
      "score": null,
      "durationS": 3033.22,
      "cost": 19.84855425,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_singlet_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "TOXhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "LAG3high_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "GZMKhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "intermediate_exhausted_gzmk"
                    ],
                    "canonical": "intermediate_exhausted_gzmk"
                  }
                }
              },
              {
                "key": "TCF7_stem_like_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "stem_like_exhausted"
                    ],
                    "canonical": "stem_like_exhausted"
                  }
                }
              },
              {
                "key": "MKI67_pos_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              },
              {
                "key": "MKI67high_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_tumor_cluster_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "Tn",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "Tn_Tmem",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "early_Tem_GZMK",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              },
              {
                "key": "Tem_GZMH",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              }
            ]
          },
          {
            "name": "clonotype_expansion_direction_data_true",
            "role": "gate",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_substantially_more_expanded_than_cluster",
              "singlet_modestly_more_expanded_than_cluster"
            ]
          },
          {
            "name": "biological_inference_data_true",
            "role": "gate",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_antigen_experienced_cluster_bystander"
            ]
          },
          {
            "name": "singlet_top3_contains_naive_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Tn",
              "Tn_Tmem",
              "early_Tem_GZMK",
              "Tem_GZMH"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "TOXhigh_Tex",
              "LAG3high_Tex",
              "GZMKhigh_Tex",
              "TCF7_stem_like_Tex",
              "MKI67_pos_Tex_Tprol",
              "MKI67high_Tex_Tprol"
            ]
          },
          {
            "name": "singlet_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "TOXhigh_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "TCF7_stem_like_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "MKI67_state_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "Tn_or_Tn_Tmem_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_stemlike_precursor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_reported_canonical_expansion_direction",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_substantially_more_expanded_than_singlet",
              "cluster_modestly_more_expanded_than_singlet"
            ]
          },
          {
            "name": "cf_expansion_lazy_defer",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "expansion_pattern_not_detectable"
            ]
          },
          {
            "name": "cf_reported_canonical_expected_biology",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_antigen_experienced_singlet_bystander"
            ]
          },
          {
            "name": "cf_biological_inference_null_artifact_or_lineage",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "differences_reflect_dissociation_or_sorting_artifact",
              "cluster_and_singlet_capture_same_biology",
              "cluster_and_singlet_capture_distinct_cd8_lineages"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "biological_relations",
            "kind": "dict",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_greater_than",
                  "greater_than"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "less_than",
                  "much_less_than"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than",
                  "approximately_equal"
                ]
              },
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal"
                ]
              },
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal",
                  "greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_exhaustion_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_naive_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_proliferation_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_expansion_richness_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_multichain_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_intermediate_gzmk_overgeneralized",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_melanocytic_conjugate_overcall",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "values": [
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_relations_mass_hedge",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-5-claude-code",
      "modelName": "claude-opus-5",
      "harness": "claude-code",
      "task": "10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 2338.92,
      "cost": 5.5986600000000015,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_singlet_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "TOXhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "LAG3high_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "GZMKhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "intermediate_exhausted_gzmk"
                    ],
                    "canonical": "intermediate_exhausted_gzmk"
                  }
                }
              },
              {
                "key": "TCF7_stem_like_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "stem_like_exhausted"
                    ],
                    "canonical": "stem_like_exhausted"
                  }
                }
              },
              {
                "key": "MKI67_pos_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              },
              {
                "key": "MKI67high_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_tumor_cluster_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "Tn",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "Tn_Tmem",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "early_Tem_GZMK",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              },
              {
                "key": "Tem_GZMH",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              }
            ]
          },
          {
            "name": "clonotype_expansion_direction_data_true",
            "role": "gate",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_substantially_more_expanded_than_cluster",
              "singlet_modestly_more_expanded_than_cluster"
            ]
          },
          {
            "name": "biological_inference_data_true",
            "role": "gate",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_antigen_experienced_cluster_bystander"
            ]
          },
          {
            "name": "singlet_top3_contains_naive_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Tn",
              "Tn_Tmem",
              "early_Tem_GZMK",
              "Tem_GZMH"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "TOXhigh_Tex",
              "LAG3high_Tex",
              "GZMKhigh_Tex",
              "TCF7_stem_like_Tex",
              "MKI67_pos_Tex_Tprol",
              "MKI67high_Tex_Tprol"
            ]
          },
          {
            "name": "singlet_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "TOXhigh_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "TCF7_stem_like_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "MKI67_state_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "Tn_or_Tn_Tmem_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_stemlike_precursor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_reported_canonical_expansion_direction",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_substantially_more_expanded_than_singlet",
              "cluster_modestly_more_expanded_than_singlet"
            ]
          },
          {
            "name": "cf_expansion_lazy_defer",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "expansion_pattern_not_detectable"
            ]
          },
          {
            "name": "cf_reported_canonical_expected_biology",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_antigen_experienced_singlet_bystander"
            ]
          },
          {
            "name": "cf_biological_inference_null_artifact_or_lineage",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "differences_reflect_dissociation_or_sorting_artifact",
              "cluster_and_singlet_capture_same_biology",
              "cluster_and_singlet_capture_distinct_cd8_lineages"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "biological_relations",
            "kind": "dict",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_greater_than",
                  "greater_than"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "less_than",
                  "much_less_than"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than",
                  "approximately_equal"
                ]
              },
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal"
                ]
              },
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal",
                  "greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_exhaustion_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_naive_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_proliferation_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_expansion_richness_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_multichain_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_intermediate_gzmk_overgeneralized",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_melanocytic_conjugate_overcall",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "values": [
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_relations_mass_hedge",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-5-claude-code",
      "modelName": "claude-opus-5",
      "harness": "claude-code",
      "task": "10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 3685.73,
      "cost": 11.093810250000002,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_singlet_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "TOXhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "LAG3high_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "GZMKhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "intermediate_exhausted_gzmk"
                    ],
                    "canonical": "intermediate_exhausted_gzmk"
                  }
                }
              },
              {
                "key": "TCF7_stem_like_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "stem_like_exhausted"
                    ],
                    "canonical": "stem_like_exhausted"
                  }
                }
              },
              {
                "key": "MKI67_pos_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              },
              {
                "key": "MKI67high_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_tumor_cluster_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "Tn",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "Tn_Tmem",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "early_Tem_GZMK",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              },
              {
                "key": "Tem_GZMH",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              }
            ]
          },
          {
            "name": "clonotype_expansion_direction_data_true",
            "role": "gate",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_substantially_more_expanded_than_cluster",
              "singlet_modestly_more_expanded_than_cluster"
            ]
          },
          {
            "name": "biological_inference_data_true",
            "role": "gate",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_antigen_experienced_cluster_bystander"
            ]
          },
          {
            "name": "singlet_top3_contains_naive_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Tn",
              "Tn_Tmem",
              "early_Tem_GZMK",
              "Tem_GZMH"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "TOXhigh_Tex",
              "LAG3high_Tex",
              "GZMKhigh_Tex",
              "TCF7_stem_like_Tex",
              "MKI67_pos_Tex_Tprol",
              "MKI67high_Tex_Tprol"
            ]
          },
          {
            "name": "singlet_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "TOXhigh_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "TCF7_stem_like_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "MKI67_state_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "Tn_or_Tn_Tmem_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_stemlike_precursor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_reported_canonical_expansion_direction",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_substantially_more_expanded_than_singlet",
              "cluster_modestly_more_expanded_than_singlet"
            ]
          },
          {
            "name": "cf_expansion_lazy_defer",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "expansion_pattern_not_detectable"
            ]
          },
          {
            "name": "cf_reported_canonical_expected_biology",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_antigen_experienced_singlet_bystander"
            ]
          },
          {
            "name": "cf_biological_inference_null_artifact_or_lineage",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "differences_reflect_dissociation_or_sorting_artifact",
              "cluster_and_singlet_capture_same_biology",
              "cluster_and_singlet_capture_distinct_cd8_lineages"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "biological_relations",
            "kind": "dict",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_greater_than",
                  "greater_than"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "less_than",
                  "much_less_than"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than",
                  "approximately_equal"
                ]
              },
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal"
                ]
              },
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal",
                  "greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_exhaustion_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_naive_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_proliferation_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_expansion_richness_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_multichain_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_intermediate_gzmk_overgeneralized",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_melanocytic_conjugate_overcall",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "values": [
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_relations_mass_hedge",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-5-claude-code",
      "modelName": "claude-opus-5",
      "harness": "claude-code",
      "task": "10x_5p_melanoma_tcr_matched_apc_vs_tumor_exhaustion",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 1,
      "passed": true,
      "score": null,
      "durationS": 1739.31,
      "cost": 3.6226250000000007,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "state_discriminating_genes",
            "kind": "list",
            "matchKey": "gene",
            "k": 3,
            "expected": [
              {
                "key": "CTLA4",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TOX",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "ENTPD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "HAVCR2",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TIGIT",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "PDCD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "difference_attributable_to",
            "role": "gate",
            "field": "difference_attributable_to",
            "kind": "value",
            "expected": "covaries_with_apc_conjugate_state"
          },
          {
            "name": "causal_status",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "direction_unresolved_selection_confound"
          },
          {
            "name": "functional_interpretation",
            "role": "gate",
            "field": "functional_interpretation",
            "kind": "value",
            "expected": "costimulation_signature_function_unresolved"
          },
          {
            "name": "cf_contamination_gene_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "MLANA",
              "PMEL",
              "DCT",
              "TYR",
              "TYRP1",
              "SOX10",
              "MITF",
              "MAGEA4"
            ]
          },
          {
            "name": "cf_apc_partner_contamination_gene",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "HLA-DRA",
              "CD68",
              "LYZ",
              "CD14",
              "C1QA",
              "ITGAX",
              "CD1C"
            ]
          },
          {
            "name": "cf_ligand_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "CD274",
              "PDCD1LG2",
              "CD80",
              "CD86"
            ]
          },
          {
            "name": "cf_attribution_causal_or_baseline",
            "role": "hard_fail",
            "field": "$.difference_attributable_to",
            "kind": "avoid",
            "expected": [
              "apc_engagement",
              "tumor_cell_engagement",
              "clonotype_intrinsic_baseline"
            ]
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "engagement_induces_exhausted_state",
              "pre_exhausted_cells_select_into_apc_conjugates"
            ]
          },
          {
            "name": "cf_causal_fabrication",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "resolved_by_pseudotemporal_ordering",
              "resolved_by_perturbation_experiment"
            ]
          },
          {
            "name": "cf_functional_overclaim",
            "role": "hard_fail",
            "field": "$.functional_interpretation",
            "kind": "avoid",
            "expected": [
              "costimulation_licensed_reinvigoration_competent",
              "costimulation_licensed_checkpoint_responsive",
              "tumor_driven_terminal_exhaustion"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-5-claude-code",
      "modelName": "claude-opus-5",
      "harness": "claude-code",
      "task": "10x_5p_melanoma_tcr_matched_apc_vs_tumor_exhaustion",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 1519.62,
      "cost": 2.7433047500000005,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "state_discriminating_genes",
            "kind": "list",
            "matchKey": "gene",
            "k": 3,
            "expected": [
              {
                "key": "CTLA4",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TOX",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "ENTPD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "HAVCR2",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TIGIT",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "PDCD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "difference_attributable_to",
            "role": "gate",
            "field": "difference_attributable_to",
            "kind": "value",
            "expected": "covaries_with_apc_conjugate_state"
          },
          {
            "name": "causal_status",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "direction_unresolved_selection_confound"
          },
          {
            "name": "functional_interpretation",
            "role": "gate",
            "field": "functional_interpretation",
            "kind": "value",
            "expected": "costimulation_signature_function_unresolved"
          },
          {
            "name": "cf_contamination_gene_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "MLANA",
              "PMEL",
              "DCT",
              "TYR",
              "TYRP1",
              "SOX10",
              "MITF",
              "MAGEA4"
            ]
          },
          {
            "name": "cf_apc_partner_contamination_gene",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "HLA-DRA",
              "CD68",
              "LYZ",
              "CD14",
              "C1QA",
              "ITGAX",
              "CD1C"
            ]
          },
          {
            "name": "cf_ligand_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "CD274",
              "PDCD1LG2",
              "CD80",
              "CD86"
            ]
          },
          {
            "name": "cf_attribution_causal_or_baseline",
            "role": "hard_fail",
            "field": "$.difference_attributable_to",
            "kind": "avoid",
            "expected": [
              "apc_engagement",
              "tumor_cell_engagement",
              "clonotype_intrinsic_baseline"
            ]
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "engagement_induces_exhausted_state",
              "pre_exhausted_cells_select_into_apc_conjugates"
            ]
          },
          {
            "name": "cf_causal_fabrication",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "resolved_by_pseudotemporal_ordering",
              "resolved_by_perturbation_experiment"
            ]
          },
          {
            "name": "cf_functional_overclaim",
            "role": "hard_fail",
            "field": "$.functional_interpretation",
            "kind": "avoid",
            "expected": [
              "costimulation_licensed_reinvigoration_competent",
              "costimulation_licensed_checkpoint_responsive",
              "tumor_driven_terminal_exhaustion"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-5-claude-code",
      "modelName": "claude-opus-5",
      "harness": "claude-code",
      "task": "10x_5p_melanoma_tcr_matched_apc_vs_tumor_exhaustion",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1528.36,
      "cost": 4.20022325,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "state_discriminating_genes",
            "kind": "list",
            "matchKey": "gene",
            "k": 3,
            "expected": [
              {
                "key": "CTLA4",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TOX",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "ENTPD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "HAVCR2",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TIGIT",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "PDCD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "difference_attributable_to",
            "role": "gate",
            "field": "difference_attributable_to",
            "kind": "value",
            "expected": "covaries_with_apc_conjugate_state"
          },
          {
            "name": "causal_status",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "direction_unresolved_selection_confound"
          },
          {
            "name": "functional_interpretation",
            "role": "gate",
            "field": "functional_interpretation",
            "kind": "value",
            "expected": "costimulation_signature_function_unresolved"
          },
          {
            "name": "cf_contamination_gene_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "MLANA",
              "PMEL",
              "DCT",
              "TYR",
              "TYRP1",
              "SOX10",
              "MITF",
              "MAGEA4"
            ]
          },
          {
            "name": "cf_apc_partner_contamination_gene",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "HLA-DRA",
              "CD68",
              "LYZ",
              "CD14",
              "C1QA",
              "ITGAX",
              "CD1C"
            ]
          },
          {
            "name": "cf_ligand_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "CD274",
              "PDCD1LG2",
              "CD80",
              "CD86"
            ]
          },
          {
            "name": "cf_attribution_causal_or_baseline",
            "role": "hard_fail",
            "field": "$.difference_attributable_to",
            "kind": "avoid",
            "expected": [
              "apc_engagement",
              "tumor_cell_engagement",
              "clonotype_intrinsic_baseline"
            ]
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "engagement_induces_exhausted_state",
              "pre_exhausted_cells_select_into_apc_conjugates"
            ]
          },
          {
            "name": "cf_causal_fabrication",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "resolved_by_pseudotemporal_ordering",
              "resolved_by_perturbation_experiment"
            ]
          },
          {
            "name": "cf_functional_overclaim",
            "role": "hard_fail",
            "field": "$.functional_interpretation",
            "kind": "avoid",
            "expected": [
              "costimulation_licensed_reinvigoration_competent",
              "costimulation_licensed_checkpoint_responsive",
              "tumor_driven_terminal_exhaustion"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-5-claude-code",
      "modelName": "claude-opus-5",
      "harness": "claude-code",
      "task": "green_2_tme_context_dependence",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": null,
      "cost": null,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "program_map",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "sig_includes_itgae",
            "role": "gate",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Itgae"
            ]
          },
          {
            "name": "declared_a_known_confounder",
            "role": "gate",
            "field": "$.excluded_confounders[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "cf_signature_gut_contaminant",
            "role": "hard_fail",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "generic_signature_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd103_program_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "<"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "per_model_regulatory_state",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "poised_unfired"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "late_activation"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "closing_committed"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_tuples",
            "kind": "list",
            "matchKey": "regulator",
            "k": 1,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "role": {
                    "op": "equals",
                    "values": [
                      "repressor"
                    ]
                  },
                  "gating_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  },
                  "evidence_basis": {
                    "op": "equals",
                    "values": [
                      "target_program_behavior"
                    ]
                  },
                  "expr_vs_activity": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "locus_differential_gate",
            "role": "gate",
            "field": "locus_differential",
            "kind": "value",
            "expected": "uniform_across_models"
          },
          {
            "name": "formation_limited_by_gate",
            "role": "gate",
            "field": "formation_limited_by",
            "kind": "value",
            "expected": "absent_induction"
          },
          {
            "name": "locus_competent_in_naive_gate",
            "role": "gate",
            "field": "locus_competent_in_naive",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "ready_silent_fraction_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 3,
            "expected": [
              {
                "key": "B16_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_KPC_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "activator_chromatin_footprint_gate",
            "role": "gate",
            "field": "activator_chromatin_footprint",
            "kind": "value",
            "expected": "inconsistent_across_formers"
          },
          {
            "name": "exhaustion_comparable_gate",
            "role": "gate",
            "field": "exhaustion_comparable_across_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_nondeterminant_gate",
            "role": "gate",
            "field": "exhaustion_explains_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_separable_program_gate",
            "role": "gate",
            "field": "exhaustion_separable_program",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "exhaustion_counterexample_gate",
            "role": "gate",
            "field": "exhaustion_counterexample_model",
            "kind": "value",
            "expected": "KPC_GP"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "exhaustion_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "cf_inconsistent_present_without_activator",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "program",
            "entity": "present",
            "valueField": "activator_state",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_no_block_but_absent",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "none",
            "valueField": "program",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_late_block_without_poised_locus",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "late_activation",
            "valueField": "locus_state",
            "expected": [
              "closing_committed",
              "active"
            ]
          },
          {
            "name": "cf_inconsistent_early_block_with_brake_released",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "early_release",
            "valueField": "brake_state",
            "expected": [
              "released"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-5-claude-code",
      "modelName": "claude-opus-5",
      "harness": "claude-code",
      "task": "green_2_tme_context_dependence",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 2527.12,
      "cost": 7.900859500000003,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "program_map",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "sig_includes_itgae",
            "role": "gate",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Itgae"
            ]
          },
          {
            "name": "declared_a_known_confounder",
            "role": "gate",
            "field": "$.excluded_confounders[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "cf_signature_gut_contaminant",
            "role": "hard_fail",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "generic_signature_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd103_program_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "<"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "per_model_regulatory_state",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "poised_unfired"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "late_activation"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "closing_committed"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_tuples",
            "kind": "list",
            "matchKey": "regulator",
            "k": 1,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "role": {
                    "op": "equals",
                    "values": [
                      "repressor"
                    ]
                  },
                  "gating_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  },
                  "evidence_basis": {
                    "op": "equals",
                    "values": [
                      "target_program_behavior"
                    ]
                  },
                  "expr_vs_activity": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "locus_differential_gate",
            "role": "gate",
            "field": "locus_differential",
            "kind": "value",
            "expected": "uniform_across_models"
          },
          {
            "name": "formation_limited_by_gate",
            "role": "gate",
            "field": "formation_limited_by",
            "kind": "value",
            "expected": "absent_induction"
          },
          {
            "name": "locus_competent_in_naive_gate",
            "role": "gate",
            "field": "locus_competent_in_naive",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "ready_silent_fraction_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 3,
            "expected": [
              {
                "key": "B16_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_KPC_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "activator_chromatin_footprint_gate",
            "role": "gate",
            "field": "activator_chromatin_footprint",
            "kind": "value",
            "expected": "inconsistent_across_formers"
          },
          {
            "name": "exhaustion_comparable_gate",
            "role": "gate",
            "field": "exhaustion_comparable_across_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_nondeterminant_gate",
            "role": "gate",
            "field": "exhaustion_explains_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_separable_program_gate",
            "role": "gate",
            "field": "exhaustion_separable_program",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "exhaustion_counterexample_gate",
            "role": "gate",
            "field": "exhaustion_counterexample_model",
            "kind": "value",
            "expected": "KPC_GP"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "exhaustion_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "cf_inconsistent_present_without_activator",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "program",
            "entity": "present",
            "valueField": "activator_state",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_no_block_but_absent",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "none",
            "valueField": "program",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_late_block_without_poised_locus",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "late_activation",
            "valueField": "locus_state",
            "expected": [
              "closing_committed",
              "active"
            ]
          },
          {
            "name": "cf_inconsistent_early_block_with_brake_released",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "early_release",
            "valueField": "brake_state",
            "expected": [
              "released"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-5-claude-code",
      "modelName": "claude-opus-5",
      "harness": "claude-code",
      "task": "green_2_tme_context_dependence",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 3042.69,
      "cost": 11.840701,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "program_map",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "sig_includes_itgae",
            "role": "gate",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Itgae"
            ]
          },
          {
            "name": "declared_a_known_confounder",
            "role": "gate",
            "field": "$.excluded_confounders[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "cf_signature_gut_contaminant",
            "role": "hard_fail",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "generic_signature_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd103_program_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "<"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "per_model_regulatory_state",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "poised_unfired"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "late_activation"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "closing_committed"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_tuples",
            "kind": "list",
            "matchKey": "regulator",
            "k": 1,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "role": {
                    "op": "equals",
                    "values": [
                      "repressor"
                    ]
                  },
                  "gating_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  },
                  "evidence_basis": {
                    "op": "equals",
                    "values": [
                      "target_program_behavior"
                    ]
                  },
                  "expr_vs_activity": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "locus_differential_gate",
            "role": "gate",
            "field": "locus_differential",
            "kind": "value",
            "expected": "uniform_across_models"
          },
          {
            "name": "formation_limited_by_gate",
            "role": "gate",
            "field": "formation_limited_by",
            "kind": "value",
            "expected": "absent_induction"
          },
          {
            "name": "locus_competent_in_naive_gate",
            "role": "gate",
            "field": "locus_competent_in_naive",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "ready_silent_fraction_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 3,
            "expected": [
              {
                "key": "B16_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_KPC_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "activator_chromatin_footprint_gate",
            "role": "gate",
            "field": "activator_chromatin_footprint",
            "kind": "value",
            "expected": "inconsistent_across_formers"
          },
          {
            "name": "exhaustion_comparable_gate",
            "role": "gate",
            "field": "exhaustion_comparable_across_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_nondeterminant_gate",
            "role": "gate",
            "field": "exhaustion_explains_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_separable_program_gate",
            "role": "gate",
            "field": "exhaustion_separable_program",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "exhaustion_counterexample_gate",
            "role": "gate",
            "field": "exhaustion_counterexample_model",
            "kind": "value",
            "expected": "KPC_GP"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "exhaustion_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "cf_inconsistent_present_without_activator",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "program",
            "entity": "present",
            "valueField": "activator_state",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_no_block_but_absent",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "none",
            "valueField": "program",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_late_block_without_poised_locus",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "late_activation",
            "valueField": "locus_state",
            "expected": [
              "closing_committed",
              "active"
            ]
          },
          {
            "name": "cf_inconsistent_early_block_with_brake_released",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "early_release",
            "valueField": "brake_state",
            "expected": [
              "released"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-5-claude-code",
      "modelName": "claude-opus-5",
      "harness": "claude-code",
      "task": "green_klf2_hidden_repressor_of_trm_like_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 3350.8,
      "cost": 10.174584500000002,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "regulator_gate",
            "role": "gate",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "KLF2",
              "Klf2"
            ]
          },
          {
            "name": "controlling_node_character_gate",
            "role": "gate",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "high_differential_expression"
          },
          {
            "name": "expression_vs_activity_gate",
            "role": "gate",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "expression_exceeds_activity"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "target_program_partition",
            "kind": "list",
            "matchKey": "gene",
            "k": 7,
            "expected": [
              {
                "key": "S1pr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "S1pr4",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Klrg1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Cx3cr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Itgae",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Hic1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "P2rx7",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              }
            ]
          },
          {
            "name": "causal_status_gate",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "association_only_sufficiency_requires_perturbation"
          },
          {
            "name": "residency_control_logic_gate",
            "role": "gate",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_release_necessary_not_sufficient_multi_regulon"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "trm_like_program_by_model",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_absent"
                    ],
                    "canonical": "niche_absent"
                  }
                }
              }
            ]
          },
          {
            "name": "cf_regulator_activator_or_wrong_node",
            "role": "hard_fail",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "RUNX3",
              "Runx3",
              "PRDM1",
              "Prdm1",
              "BLIMP1",
              "Blimp1",
              "BHLHE40",
              "Bhlhe40",
              "ZNF683",
              "Zfp683",
              "HOBIT",
              "Hobit",
              "AHR",
              "Ahr",
              "BATF",
              "Batf",
              "TOX",
              "Tox",
              "NR4A1",
              "Nr4a1",
              "NR4A2",
              "Nr4a2",
              "NR4A3",
              "Nr4a3",
              "TCF7",
              "Tcf7",
              "TCF1",
              "Tcf1",
              "BACH2",
              "Bach2",
              "TBX21",
              "Tbx21",
              "TBET",
              "Tbet",
              "FLI1",
              "Fli1",
              "NR3C1",
              "Nr3c1",
              "EOMES",
              "Eomes",
              "ZEB2",
              "Zeb2",
              "ID2",
              "Id2",
              "ID3",
              "Id3",
              "IRF4",
              "Irf4"
            ]
          },
          {
            "name": "cf_node_character_hidden_driver_recall",
            "role": "hard_fail",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "hidden_driver_activity_not_expression"
          },
          {
            "name": "cf_effect_relation_hidden_pattern",
            "role": "hard_fail",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "activity_exceeds_expression"
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "causal_status",
            "kind": "value",
            "op": "in",
            "expected": [
              "sufficiency_established",
              "necessity_established",
              "causal_established_in_dataset",
              "sufficiency_via_perturbation_in_dataset"
            ]
          },
          {
            "name": "cf_residency_logic_activator_prior",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "residency_activator_induction_primary"
          },
          {
            "name": "cf_residency_logic_single_regulon",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_controls_entire_residency_program"
          },
          {
            "name": "cf_decoy_called_target",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_distractor_model_reported",
            "role": "hard_fail",
            "field": "$.trm_like_program_by_model[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_chromatin_only_decoy",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-5-claude-code",
      "modelName": "claude-opus-5",
      "harness": "claude-code",
      "task": "green_klf2_hidden_repressor_of_trm_like_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 2138.45,
      "cost": 7.060943249999998,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "regulator_gate",
            "role": "gate",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "KLF2",
              "Klf2"
            ]
          },
          {
            "name": "controlling_node_character_gate",
            "role": "gate",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "high_differential_expression"
          },
          {
            "name": "expression_vs_activity_gate",
            "role": "gate",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "expression_exceeds_activity"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "target_program_partition",
            "kind": "list",
            "matchKey": "gene",
            "k": 7,
            "expected": [
              {
                "key": "S1pr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "S1pr4",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Klrg1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Cx3cr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Itgae",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Hic1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "P2rx7",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              }
            ]
          },
          {
            "name": "causal_status_gate",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "association_only_sufficiency_requires_perturbation"
          },
          {
            "name": "residency_control_logic_gate",
            "role": "gate",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_release_necessary_not_sufficient_multi_regulon"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "trm_like_program_by_model",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_absent"
                    ],
                    "canonical": "niche_absent"
                  }
                }
              }
            ]
          },
          {
            "name": "cf_regulator_activator_or_wrong_node",
            "role": "hard_fail",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "RUNX3",
              "Runx3",
              "PRDM1",
              "Prdm1",
              "BLIMP1",
              "Blimp1",
              "BHLHE40",
              "Bhlhe40",
              "ZNF683",
              "Zfp683",
              "HOBIT",
              "Hobit",
              "AHR",
              "Ahr",
              "BATF",
              "Batf",
              "TOX",
              "Tox",
              "NR4A1",
              "Nr4a1",
              "NR4A2",
              "Nr4a2",
              "NR4A3",
              "Nr4a3",
              "TCF7",
              "Tcf7",
              "TCF1",
              "Tcf1",
              "BACH2",
              "Bach2",
              "TBX21",
              "Tbx21",
              "TBET",
              "Tbet",
              "FLI1",
              "Fli1",
              "NR3C1",
              "Nr3c1",
              "EOMES",
              "Eomes",
              "ZEB2",
              "Zeb2",
              "ID2",
              "Id2",
              "ID3",
              "Id3",
              "IRF4",
              "Irf4"
            ]
          },
          {
            "name": "cf_node_character_hidden_driver_recall",
            "role": "hard_fail",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "hidden_driver_activity_not_expression"
          },
          {
            "name": "cf_effect_relation_hidden_pattern",
            "role": "hard_fail",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "activity_exceeds_expression"
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "causal_status",
            "kind": "value",
            "op": "in",
            "expected": [
              "sufficiency_established",
              "necessity_established",
              "causal_established_in_dataset",
              "sufficiency_via_perturbation_in_dataset"
            ]
          },
          {
            "name": "cf_residency_logic_activator_prior",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "residency_activator_induction_primary"
          },
          {
            "name": "cf_residency_logic_single_regulon",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_controls_entire_residency_program"
          },
          {
            "name": "cf_decoy_called_target",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_distractor_model_reported",
            "role": "hard_fail",
            "field": "$.trm_like_program_by_model[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_chromatin_only_decoy",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-5-claude-code",
      "modelName": "claude-opus-5",
      "harness": "claude-code",
      "task": "green_klf2_hidden_repressor_of_trm_like_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 3689.4,
      "cost": 22.89046124999999,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "regulator_gate",
            "role": "gate",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "KLF2",
              "Klf2"
            ]
          },
          {
            "name": "controlling_node_character_gate",
            "role": "gate",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "high_differential_expression"
          },
          {
            "name": "expression_vs_activity_gate",
            "role": "gate",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "expression_exceeds_activity"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "target_program_partition",
            "kind": "list",
            "matchKey": "gene",
            "k": 7,
            "expected": [
              {
                "key": "S1pr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "S1pr4",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Klrg1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Cx3cr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Itgae",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Hic1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "P2rx7",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              }
            ]
          },
          {
            "name": "causal_status_gate",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "association_only_sufficiency_requires_perturbation"
          },
          {
            "name": "residency_control_logic_gate",
            "role": "gate",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_release_necessary_not_sufficient_multi_regulon"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "trm_like_program_by_model",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_absent"
                    ],
                    "canonical": "niche_absent"
                  }
                }
              }
            ]
          },
          {
            "name": "cf_regulator_activator_or_wrong_node",
            "role": "hard_fail",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "RUNX3",
              "Runx3",
              "PRDM1",
              "Prdm1",
              "BLIMP1",
              "Blimp1",
              "BHLHE40",
              "Bhlhe40",
              "ZNF683",
              "Zfp683",
              "HOBIT",
              "Hobit",
              "AHR",
              "Ahr",
              "BATF",
              "Batf",
              "TOX",
              "Tox",
              "NR4A1",
              "Nr4a1",
              "NR4A2",
              "Nr4a2",
              "NR4A3",
              "Nr4a3",
              "TCF7",
              "Tcf7",
              "TCF1",
              "Tcf1",
              "BACH2",
              "Bach2",
              "TBX21",
              "Tbx21",
              "TBET",
              "Tbet",
              "FLI1",
              "Fli1",
              "NR3C1",
              "Nr3c1",
              "EOMES",
              "Eomes",
              "ZEB2",
              "Zeb2",
              "ID2",
              "Id2",
              "ID3",
              "Id3",
              "IRF4",
              "Irf4"
            ]
          },
          {
            "name": "cf_node_character_hidden_driver_recall",
            "role": "hard_fail",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "hidden_driver_activity_not_expression"
          },
          {
            "name": "cf_effect_relation_hidden_pattern",
            "role": "hard_fail",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "activity_exceeds_expression"
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "causal_status",
            "kind": "value",
            "op": "in",
            "expected": [
              "sufficiency_established",
              "necessity_established",
              "causal_established_in_dataset",
              "sufficiency_via_perturbation_in_dataset"
            ]
          },
          {
            "name": "cf_residency_logic_activator_prior",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "residency_activator_induction_primary"
          },
          {
            "name": "cf_residency_logic_single_regulon",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_controls_entire_residency_program"
          },
          {
            "name": "cf_decoy_called_target",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_distractor_model_reported",
            "role": "hard_fail",
            "field": "$.trm_like_program_by_model[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_chromatin_only_decoy",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-5-claude-code",
      "modelName": "claude-opus-5",
      "harness": "claude-code",
      "task": "green_shared_vs_distinct_regulators_trm_vs_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 1,
      "passed": true,
      "score": null,
      "durationS": 3134.64,
      "cost": 19.91642425,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_program_map",
            "kind": "list",
            "matchKey": "regulator",
            "k": 3,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "both"
                    ],
                    "canonical": "both"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "represses"
                    ],
                    "canonical": "represses"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Batf",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "trm_like_til_specific"
                    ],
                    "canonical": "trm_like_til_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "chromatin_over_rna"
                    ],
                    "canonical": "chromatin_over_rna"
                  }
                }
              },
              {
                "key": "Ahr",
                "fields": {
                  "program_specificity": {
                    "op": "in",
                    "values": [
                      "trm_specific",
                      "both"
                    ],
                    "canonical": "trm_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "rna_over_chromatin"
                    ],
                    "canonical": "rna_over_chromatin"
                  }
                }
              }
            ]
          },
          {
            "name": "regulator_set_no_extras_gate",
            "role": "gate",
            "field": "$.regulator_program_map[*]",
            "kind": "other",
            "expected": null
          },
          {
            "name": "program_identity_conclusion_gate",
            "role": "gate",
            "field": "program_identity_conclusion",
            "kind": "value",
            "expected": "distinct_state_shared_brake_divergent_activators"
          },
          {
            "name": "residency_install_logic_gate",
            "role": "gate",
            "field": "residency_install_logic",
            "kind": "value",
            "expected": "brake_release_permits_context_matched_activator"
          },
          {
            "name": "activator_context_dependence_gate",
            "role": "gate",
            "field": "activator_context_dependence",
            "kind": "value",
            "expected": "context_matched_crossover"
          },
          {
            "name": "evidence_ceiling_gate",
            "role": "gate",
            "field": "evidence_ceiling",
            "kind": "value",
            "expected": "specificity_dissociations_require_perturbation"
          },
          {
            "name": "cf_install_logic_flat_or_single_step",
            "role": "hard_fail",
            "field": "residency_install_logic",
            "kind": "value",
            "op": "in",
            "expected": [
              "flat_shared_residency_toolkit",
              "activator_induction_sufficient",
              "brake_release_alone_sufficient"
            ]
          },
          {
            "name": "cf_program_identity_lumped_or_wrong",
            "role": "hard_fail",
            "field": "program_identity_conclusion",
            "kind": "value",
            "op": "in",
            "expected": [
              "trm_like_til_is_tumor_trm",
              "trm_like_til_is_exhausted_subset",
              "trm_like_til_is_effector_state",
              "trm_like_til_is_transitional_intermediate",
              "trm_like_til_is_shared_program_distinct_niche"
            ]
          },
          {
            "name": "cf_ahr_assigned_to_til",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Ahr",
            "valueField": "program_specificity",
            "expected": [
              "trm_like_til_specific"
            ]
          },
          {
            "name": "cf_batf_assigned_to_trm_or_shared",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Batf",
            "valueField": "program_specificity",
            "expected": [
              "trm_specific",
              "both"
            ]
          },
          {
            "name": "cf_klf2_called_activator",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Klf2",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          },
          {
            "name": "cf_evidence_ceiling_overclaim",
            "role": "hard_fail",
            "field": "evidence_ceiling",
            "kind": "value",
            "op": "in",
            "expected": [
              "specificity_established_from_regulon_activity",
              "specificity_established_in_dataset",
              "causation_established_by_cross_sectional_concordance",
              "necessity_and_sufficiency_established"
            ]
          },
          {
            "name": "cf_activator_context_shared_or_single",
            "role": "hard_fail",
            "field": "activator_context_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "shared_activators_both_contexts",
              "single_global_activator",
              "redundant_activators"
            ]
          },
          {
            "name": "cf_single_modality_rule",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_hic1_motif_brake_mislabel",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_runx3_anticanonical_as_controller",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Runx3",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-5-claude-code",
      "modelName": "claude-opus-5",
      "harness": "claude-code",
      "task": "green_shared_vs_distinct_regulators_trm_vs_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 2,
      "passed": true,
      "score": null,
      "durationS": 4162.04,
      "cost": 13.749157250000003,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_program_map",
            "kind": "list",
            "matchKey": "regulator",
            "k": 3,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "both"
                    ],
                    "canonical": "both"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "represses"
                    ],
                    "canonical": "represses"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Batf",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "trm_like_til_specific"
                    ],
                    "canonical": "trm_like_til_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "chromatin_over_rna"
                    ],
                    "canonical": "chromatin_over_rna"
                  }
                }
              },
              {
                "key": "Ahr",
                "fields": {
                  "program_specificity": {
                    "op": "in",
                    "values": [
                      "trm_specific",
                      "both"
                    ],
                    "canonical": "trm_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "rna_over_chromatin"
                    ],
                    "canonical": "rna_over_chromatin"
                  }
                }
              }
            ]
          },
          {
            "name": "regulator_set_no_extras_gate",
            "role": "gate",
            "field": "$.regulator_program_map[*]",
            "kind": "other",
            "expected": null
          },
          {
            "name": "program_identity_conclusion_gate",
            "role": "gate",
            "field": "program_identity_conclusion",
            "kind": "value",
            "expected": "distinct_state_shared_brake_divergent_activators"
          },
          {
            "name": "residency_install_logic_gate",
            "role": "gate",
            "field": "residency_install_logic",
            "kind": "value",
            "expected": "brake_release_permits_context_matched_activator"
          },
          {
            "name": "activator_context_dependence_gate",
            "role": "gate",
            "field": "activator_context_dependence",
            "kind": "value",
            "expected": "context_matched_crossover"
          },
          {
            "name": "evidence_ceiling_gate",
            "role": "gate",
            "field": "evidence_ceiling",
            "kind": "value",
            "expected": "specificity_dissociations_require_perturbation"
          },
          {
            "name": "cf_install_logic_flat_or_single_step",
            "role": "hard_fail",
            "field": "residency_install_logic",
            "kind": "value",
            "op": "in",
            "expected": [
              "flat_shared_residency_toolkit",
              "activator_induction_sufficient",
              "brake_release_alone_sufficient"
            ]
          },
          {
            "name": "cf_program_identity_lumped_or_wrong",
            "role": "hard_fail",
            "field": "program_identity_conclusion",
            "kind": "value",
            "op": "in",
            "expected": [
              "trm_like_til_is_tumor_trm",
              "trm_like_til_is_exhausted_subset",
              "trm_like_til_is_effector_state",
              "trm_like_til_is_transitional_intermediate",
              "trm_like_til_is_shared_program_distinct_niche"
            ]
          },
          {
            "name": "cf_ahr_assigned_to_til",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Ahr",
            "valueField": "program_specificity",
            "expected": [
              "trm_like_til_specific"
            ]
          },
          {
            "name": "cf_batf_assigned_to_trm_or_shared",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Batf",
            "valueField": "program_specificity",
            "expected": [
              "trm_specific",
              "both"
            ]
          },
          {
            "name": "cf_klf2_called_activator",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Klf2",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          },
          {
            "name": "cf_evidence_ceiling_overclaim",
            "role": "hard_fail",
            "field": "evidence_ceiling",
            "kind": "value",
            "op": "in",
            "expected": [
              "specificity_established_from_regulon_activity",
              "specificity_established_in_dataset",
              "causation_established_by_cross_sectional_concordance",
              "necessity_and_sufficiency_established"
            ]
          },
          {
            "name": "cf_activator_context_shared_or_single",
            "role": "hard_fail",
            "field": "activator_context_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "shared_activators_both_contexts",
              "single_global_activator",
              "redundant_activators"
            ]
          },
          {
            "name": "cf_single_modality_rule",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_hic1_motif_brake_mislabel",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_runx3_anticanonical_as_controller",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Runx3",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-5-claude-code",
      "modelName": "claude-opus-5",
      "harness": "claude-code",
      "task": "green_shared_vs_distinct_regulators_trm_vs_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 13201.34,
      "cost": 23.57537025,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_program_map",
            "kind": "list",
            "matchKey": "regulator",
            "k": 3,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "both"
                    ],
                    "canonical": "both"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "represses"
                    ],
                    "canonical": "represses"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Batf",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "trm_like_til_specific"
                    ],
                    "canonical": "trm_like_til_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "chromatin_over_rna"
                    ],
                    "canonical": "chromatin_over_rna"
                  }
                }
              },
              {
                "key": "Ahr",
                "fields": {
                  "program_specificity": {
                    "op": "in",
                    "values": [
                      "trm_specific",
                      "both"
                    ],
                    "canonical": "trm_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "rna_over_chromatin"
                    ],
                    "canonical": "rna_over_chromatin"
                  }
                }
              }
            ]
          },
          {
            "name": "regulator_set_no_extras_gate",
            "role": "gate",
            "field": "$.regulator_program_map[*]",
            "kind": "other",
            "expected": null
          },
          {
            "name": "program_identity_conclusion_gate",
            "role": "gate",
            "field": "program_identity_conclusion",
            "kind": "value",
            "expected": "distinct_state_shared_brake_divergent_activators"
          },
          {
            "name": "residency_install_logic_gate",
            "role": "gate",
            "field": "residency_install_logic",
            "kind": "value",
            "expected": "brake_release_permits_context_matched_activator"
          },
          {
            "name": "activator_context_dependence_gate",
            "role": "gate",
            "field": "activator_context_dependence",
            "kind": "value",
            "expected": "context_matched_crossover"
          },
          {
            "name": "evidence_ceiling_gate",
            "role": "gate",
            "field": "evidence_ceiling",
            "kind": "value",
            "expected": "specificity_dissociations_require_perturbation"
          },
          {
            "name": "cf_install_logic_flat_or_single_step",
            "role": "hard_fail",
            "field": "residency_install_logic",
            "kind": "value",
            "op": "in",
            "expected": [
              "flat_shared_residency_toolkit",
              "activator_induction_sufficient",
              "brake_release_alone_sufficient"
            ]
          },
          {
            "name": "cf_program_identity_lumped_or_wrong",
            "role": "hard_fail",
            "field": "program_identity_conclusion",
            "kind": "value",
            "op": "in",
            "expected": [
              "trm_like_til_is_tumor_trm",
              "trm_like_til_is_exhausted_subset",
              "trm_like_til_is_effector_state",
              "trm_like_til_is_transitional_intermediate",
              "trm_like_til_is_shared_program_distinct_niche"
            ]
          },
          {
            "name": "cf_ahr_assigned_to_til",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Ahr",
            "valueField": "program_specificity",
            "expected": [
              "trm_like_til_specific"
            ]
          },
          {
            "name": "cf_batf_assigned_to_trm_or_shared",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Batf",
            "valueField": "program_specificity",
            "expected": [
              "trm_specific",
              "both"
            ]
          },
          {
            "name": "cf_klf2_called_activator",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Klf2",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          },
          {
            "name": "cf_evidence_ceiling_overclaim",
            "role": "hard_fail",
            "field": "evidence_ceiling",
            "kind": "value",
            "op": "in",
            "expected": [
              "specificity_established_from_regulon_activity",
              "specificity_established_in_dataset",
              "causation_established_by_cross_sectional_concordance",
              "necessity_and_sufficiency_established"
            ]
          },
          {
            "name": "cf_activator_context_shared_or_single",
            "role": "hard_fail",
            "field": "activator_context_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "shared_activators_both_contexts",
              "single_global_activator",
              "redundant_activators"
            ]
          },
          {
            "name": "cf_single_modality_rule",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_hic1_motif_brake_mislabel",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_runx3_anticanonical_as_controller",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Runx3",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-5-claude-code",
      "modelName": "claude-opus-5",
      "harness": "claude-code",
      "task": "gse297023_q2_senescence_not_mechanism",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 1,
      "passed": true,
      "score": null,
      "durationS": null,
      "cost": null,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "candidate_is_senescence",
            "role": "gate",
            "field": "$.candidate_mechanism",
            "kind": "value",
            "op": "in",
            "expected": [
              "cellular_senescence",
              "oncogene_induced_senescence",
              "replicative_senescence"
            ]
          },
          {
            "name": "arrest_core_not_engaged",
            "role": "gate",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_not_engaged"
            ]
          },
          {
            "name": "secretory_arm_is_inflammaging",
            "role": "gate",
            "field": "$.secretory_arm_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "acute_phase_inflammaging_response"
            ]
          },
          {
            "name": "secretory_arm_broadly_elevated",
            "role": "gate",
            "field": "$.secretory_arm_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "broadly_elevated"
            ]
          },
          {
            "name": "carries_aging_program",
            "role": "gate",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "carries_bona_fide_aging_program_distinct_from_nominated"
            ]
          },
          {
            "name": "conclusion_senescence_unlikely",
            "role": "gate",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "unlikely_major_contributor"
            ]
          },
          {
            "name": "cf_arrest_core_engaged_recall",
            "role": "hard_fail",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_engaged"
            ]
          },
          {
            "name": "cf_aging_program_conflation_or_explain_away",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "aging_program_indistinguishable_from_nominated_program",
              "aging_program_is_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_no_aging_program_sensitivity_failure",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_detectable_aging_program"
            ]
          },
          {
            "name": "cf_senescence_driver_conclusion",
            "role": "hard_fail",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "likely_primary_driver",
              "plausible_supported_contributor"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-5-claude-code",
      "modelName": "claude-opus-5",
      "harness": "claude-code",
      "task": "gse297023_q2_senescence_not_mechanism",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 5622.69,
      "cost": 6.11465225,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "candidate_is_senescence",
            "role": "gate",
            "field": "$.candidate_mechanism",
            "kind": "value",
            "op": "in",
            "expected": [
              "cellular_senescence",
              "oncogene_induced_senescence",
              "replicative_senescence"
            ]
          },
          {
            "name": "arrest_core_not_engaged",
            "role": "gate",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_not_engaged"
            ]
          },
          {
            "name": "secretory_arm_is_inflammaging",
            "role": "gate",
            "field": "$.secretory_arm_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "acute_phase_inflammaging_response"
            ]
          },
          {
            "name": "secretory_arm_broadly_elevated",
            "role": "gate",
            "field": "$.secretory_arm_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "broadly_elevated"
            ]
          },
          {
            "name": "carries_aging_program",
            "role": "gate",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "carries_bona_fide_aging_program_distinct_from_nominated"
            ]
          },
          {
            "name": "conclusion_senescence_unlikely",
            "role": "gate",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "unlikely_major_contributor"
            ]
          },
          {
            "name": "cf_arrest_core_engaged_recall",
            "role": "hard_fail",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_engaged"
            ]
          },
          {
            "name": "cf_aging_program_conflation_or_explain_away",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "aging_program_indistinguishable_from_nominated_program",
              "aging_program_is_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_no_aging_program_sensitivity_failure",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_detectable_aging_program"
            ]
          },
          {
            "name": "cf_senescence_driver_conclusion",
            "role": "hard_fail",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "likely_primary_driver",
              "plausible_supported_contributor"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-5-claude-code",
      "modelName": "claude-opus-5",
      "harness": "claude-code",
      "task": "gse297023_q2_senescence_not_mechanism",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": null,
      "cost": null,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "candidate_is_senescence",
            "role": "gate",
            "field": "$.candidate_mechanism",
            "kind": "value",
            "op": "in",
            "expected": [
              "cellular_senescence",
              "oncogene_induced_senescence",
              "replicative_senescence"
            ]
          },
          {
            "name": "arrest_core_not_engaged",
            "role": "gate",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_not_engaged"
            ]
          },
          {
            "name": "secretory_arm_is_inflammaging",
            "role": "gate",
            "field": "$.secretory_arm_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "acute_phase_inflammaging_response"
            ]
          },
          {
            "name": "secretory_arm_broadly_elevated",
            "role": "gate",
            "field": "$.secretory_arm_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "broadly_elevated"
            ]
          },
          {
            "name": "carries_aging_program",
            "role": "gate",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "carries_bona_fide_aging_program_distinct_from_nominated"
            ]
          },
          {
            "name": "conclusion_senescence_unlikely",
            "role": "gate",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "unlikely_major_contributor"
            ]
          },
          {
            "name": "cf_arrest_core_engaged_recall",
            "role": "hard_fail",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_engaged"
            ]
          },
          {
            "name": "cf_aging_program_conflation_or_explain_away",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "aging_program_indistinguishable_from_nominated_program",
              "aging_program_is_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_no_aging_program_sensitivity_failure",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_detectable_aging_program"
            ]
          },
          {
            "name": "cf_senescence_driver_conclusion",
            "role": "hard_fail",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "likely_primary_driver",
              "plausible_supported_contributor"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-5-claude-code",
      "modelName": "claude-opus-5",
      "harness": "claude-code",
      "task": "gse297023_q3_age_retained_mapk_reduced",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 1,
      "passed": true,
      "score": null,
      "durationS": null,
      "cost": null,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "age_retained",
            "role": "gate",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_retained"
            ]
          },
          {
            "name": "identity_is_aging",
            "role": "gate",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_aging_signatures"
            ]
          },
          {
            "name": "nature_state_shift",
            "role": "gate",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_state_expression_shift"
            ]
          },
          {
            "name": "pathway_is_mapk",
            "role": "gate",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "mapk"
            ]
          },
          {
            "name": "pathway_reduced_in_aged",
            "role": "gate",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "reduced_in_aged"
            ]
          },
          {
            "name": "cf_mapk_increased_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_in_aged"
            ]
          },
          {
            "name": "cf_age_erased",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_erased"
            ]
          },
          {
            "name": "cf_pervasive_overcall",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_pervasive_artifactual"
            ]
          },
          {
            "name": "cf_explain_away_aging",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_compositional_or_identity",
            "role": "hard_fail",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_proportion_shift",
              "cell_identity_shift",
              "composition_dominant_state_secondary",
              "both_state_and_composition_equally"
            ]
          },
          {
            "name": "cf_biggest_inflammatory_signal_not_mapk",
            "role": "hard_fail",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "jak_stat",
              "nfkb",
              "tnfa"
            ]
          },
          {
            "name": "cf_senescence_misattribution",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_senescence_signature_specifically"
            ]
          },
          {
            "name": "cf_dismiss_as_batch_artifact",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_batch_or_technical_artifact"
            ]
          },
          {
            "name": "cf_mapk_comparable_constitutive_kras_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "comparable_across_ages"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-5-claude-code",
      "modelName": "claude-opus-5",
      "harness": "claude-code",
      "task": "gse297023_q3_age_retained_mapk_reduced",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 2,
      "passed": true,
      "score": null,
      "durationS": 13158.77,
      "cost": null,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "age_retained",
            "role": "gate",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_retained"
            ]
          },
          {
            "name": "identity_is_aging",
            "role": "gate",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_aging_signatures"
            ]
          },
          {
            "name": "nature_state_shift",
            "role": "gate",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_state_expression_shift"
            ]
          },
          {
            "name": "pathway_is_mapk",
            "role": "gate",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "mapk"
            ]
          },
          {
            "name": "pathway_reduced_in_aged",
            "role": "gate",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "reduced_in_aged"
            ]
          },
          {
            "name": "cf_mapk_increased_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_in_aged"
            ]
          },
          {
            "name": "cf_age_erased",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_erased"
            ]
          },
          {
            "name": "cf_pervasive_overcall",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_pervasive_artifactual"
            ]
          },
          {
            "name": "cf_explain_away_aging",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_compositional_or_identity",
            "role": "hard_fail",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_proportion_shift",
              "cell_identity_shift",
              "composition_dominant_state_secondary",
              "both_state_and_composition_equally"
            ]
          },
          {
            "name": "cf_biggest_inflammatory_signal_not_mapk",
            "role": "hard_fail",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "jak_stat",
              "nfkb",
              "tnfa"
            ]
          },
          {
            "name": "cf_senescence_misattribution",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_senescence_signature_specifically"
            ]
          },
          {
            "name": "cf_dismiss_as_batch_artifact",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_batch_or_technical_artifact"
            ]
          },
          {
            "name": "cf_mapk_comparable_constitutive_kras_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "comparable_across_ages"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-5-claude-code",
      "modelName": "claude-opus-5",
      "harness": "claude-code",
      "task": "gse297023_q3_age_retained_mapk_reduced",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 12587.13,
      "cost": 17.854533749999998,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "age_retained",
            "role": "gate",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_retained"
            ]
          },
          {
            "name": "identity_is_aging",
            "role": "gate",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_aging_signatures"
            ]
          },
          {
            "name": "nature_state_shift",
            "role": "gate",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_state_expression_shift"
            ]
          },
          {
            "name": "pathway_is_mapk",
            "role": "gate",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "mapk"
            ]
          },
          {
            "name": "pathway_reduced_in_aged",
            "role": "gate",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "reduced_in_aged"
            ]
          },
          {
            "name": "cf_mapk_increased_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_in_aged"
            ]
          },
          {
            "name": "cf_age_erased",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_erased"
            ]
          },
          {
            "name": "cf_pervasive_overcall",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_pervasive_artifactual"
            ]
          },
          {
            "name": "cf_explain_away_aging",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_compositional_or_identity",
            "role": "hard_fail",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_proportion_shift",
              "cell_identity_shift",
              "composition_dominant_state_secondary",
              "both_state_and_composition_equally"
            ]
          },
          {
            "name": "cf_biggest_inflammatory_signal_not_mapk",
            "role": "hard_fail",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "jak_stat",
              "nfkb",
              "tnfa"
            ]
          },
          {
            "name": "cf_senescence_misattribution",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_senescence_signature_specifically"
            ]
          },
          {
            "name": "cf_dismiss_as_batch_artifact",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_batch_or_technical_artifact"
            ]
          },
          {
            "name": "cf_mapk_comparable_constitutive_kras_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "comparable_across_ages"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-5-claude-code",
      "modelName": "claude-opus-5",
      "harness": "claude-code",
      "task": "gse297023_q5_tme_age_dependent_remodeling",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 1,
      "passed": true,
      "score": null,
      "durationS": 11155.03,
      "cost": 30.603046000000013,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "perturbation_is_pten",
            "role": "gate",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "pten"
            ]
          },
          {
            "name": "tme_non_cell_autonomous",
            "role": "gate",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "substantial_non_cell_autonomous_remodeling"
            ]
          },
          {
            "name": "tme_state_not_composition",
            "role": "gate",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "tme_age_dependent",
            "role": "gate",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_dependent"
            ]
          },
          {
            "name": "tme_response_broader_in_aged",
            "role": "gate",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_aged"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "most_responsive_noncancer_cell_types",
            "kind": "list",
            "matchKey": "cell_type",
            "k": 2,
            "expected": [
              {
                "key": "alveolar_macrophage",
                "fields": {}
              },
              {
                "key": "endothelial",
                "fields": {}
              }
            ]
          },
          {
            "name": "cf_cell_intrinsic_only",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "negligible_cell_intrinsic_only_effect",
              "remodeling_unresolved",
              "modest_remodeling_few_cell_types"
            ]
          },
          {
            "name": "cf_remodeling_is_contamination",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "apparent_remodeling_reflects_cancer_contamination"
            ]
          },
          {
            "name": "cf_perturbation_design_misread",
            "role": "hard_fail",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "multiple_tumor_suppressors_pooled",
              "no_additional_gene_difference_kras_only",
              "rb1_cdkn2a_double",
              "pcna_essential_gene_control",
              "difference_is_oncogene_not_suppressor"
            ]
          },
          {
            "name": "cf_age_independent_tme",
            "role": "hard_fail",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_independent"
            ]
          },
          {
            "name": "cf_compositional_shift",
            "role": "hard_fail",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_frequency_shift",
              "compositional_immune_infiltration",
              "composition_dominant_state_secondary",
              "both_state_and_composition"
            ]
          },
          {
            "name": "cf_epithelial_or_cancer_substitution",
            "role": "hard_fail",
            "field": "$.most_responsive_noncancer_cell_types[*].cell_type",
            "kind": "avoid",
            "expected": [
              "AT2_cell",
              "AT1_cell",
              "cancer_cell",
              "ciliated",
              "secretory",
              "club_cell",
              "basal_cell",
              "goblet_cell"
            ]
          },
          {
            "name": "cf_tme_breadth_broader_in_young",
            "role": "hard_fail",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_young"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-5-claude-code",
      "modelName": "claude-opus-5",
      "harness": "claude-code",
      "task": "gse297023_q5_tme_age_dependent_remodeling",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": null,
      "cost": null,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "perturbation_is_pten",
            "role": "gate",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "pten"
            ]
          },
          {
            "name": "tme_non_cell_autonomous",
            "role": "gate",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "substantial_non_cell_autonomous_remodeling"
            ]
          },
          {
            "name": "tme_state_not_composition",
            "role": "gate",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "tme_age_dependent",
            "role": "gate",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_dependent"
            ]
          },
          {
            "name": "tme_response_broader_in_aged",
            "role": "gate",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_aged"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "most_responsive_noncancer_cell_types",
            "kind": "list",
            "matchKey": "cell_type",
            "k": 2,
            "expected": [
              {
                "key": "alveolar_macrophage",
                "fields": {}
              },
              {
                "key": "endothelial",
                "fields": {}
              }
            ]
          },
          {
            "name": "cf_cell_intrinsic_only",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "negligible_cell_intrinsic_only_effect",
              "remodeling_unresolved",
              "modest_remodeling_few_cell_types"
            ]
          },
          {
            "name": "cf_remodeling_is_contamination",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "apparent_remodeling_reflects_cancer_contamination"
            ]
          },
          {
            "name": "cf_perturbation_design_misread",
            "role": "hard_fail",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "multiple_tumor_suppressors_pooled",
              "no_additional_gene_difference_kras_only",
              "rb1_cdkn2a_double",
              "pcna_essential_gene_control",
              "difference_is_oncogene_not_suppressor"
            ]
          },
          {
            "name": "cf_age_independent_tme",
            "role": "hard_fail",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_independent"
            ]
          },
          {
            "name": "cf_compositional_shift",
            "role": "hard_fail",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_frequency_shift",
              "compositional_immune_infiltration",
              "composition_dominant_state_secondary",
              "both_state_and_composition"
            ]
          },
          {
            "name": "cf_epithelial_or_cancer_substitution",
            "role": "hard_fail",
            "field": "$.most_responsive_noncancer_cell_types[*].cell_type",
            "kind": "avoid",
            "expected": [
              "AT2_cell",
              "AT1_cell",
              "cancer_cell",
              "ciliated",
              "secretory",
              "club_cell",
              "basal_cell",
              "goblet_cell"
            ]
          },
          {
            "name": "cf_tme_breadth_broader_in_young",
            "role": "hard_fail",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_young"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-5-claude-code",
      "modelName": "claude-opus-5",
      "harness": "claude-code",
      "task": "gse297023_q5_tme_age_dependent_remodeling",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": null,
      "cost": null,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "perturbation_is_pten",
            "role": "gate",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "pten"
            ]
          },
          {
            "name": "tme_non_cell_autonomous",
            "role": "gate",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "substantial_non_cell_autonomous_remodeling"
            ]
          },
          {
            "name": "tme_state_not_composition",
            "role": "gate",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "tme_age_dependent",
            "role": "gate",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_dependent"
            ]
          },
          {
            "name": "tme_response_broader_in_aged",
            "role": "gate",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_aged"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "most_responsive_noncancer_cell_types",
            "kind": "list",
            "matchKey": "cell_type",
            "k": 2,
            "expected": [
              {
                "key": "alveolar_macrophage",
                "fields": {}
              },
              {
                "key": "endothelial",
                "fields": {}
              }
            ]
          },
          {
            "name": "cf_cell_intrinsic_only",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "negligible_cell_intrinsic_only_effect",
              "remodeling_unresolved",
              "modest_remodeling_few_cell_types"
            ]
          },
          {
            "name": "cf_remodeling_is_contamination",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "apparent_remodeling_reflects_cancer_contamination"
            ]
          },
          {
            "name": "cf_perturbation_design_misread",
            "role": "hard_fail",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "multiple_tumor_suppressors_pooled",
              "no_additional_gene_difference_kras_only",
              "rb1_cdkn2a_double",
              "pcna_essential_gene_control",
              "difference_is_oncogene_not_suppressor"
            ]
          },
          {
            "name": "cf_age_independent_tme",
            "role": "hard_fail",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_independent"
            ]
          },
          {
            "name": "cf_compositional_shift",
            "role": "hard_fail",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_frequency_shift",
              "compositional_immune_infiltration",
              "composition_dominant_state_secondary",
              "both_state_and_composition"
            ]
          },
          {
            "name": "cf_epithelial_or_cancer_substitution",
            "role": "hard_fail",
            "field": "$.most_responsive_noncancer_cell_types[*].cell_type",
            "kind": "avoid",
            "expected": [
              "AT2_cell",
              "AT1_cell",
              "cancer_cell",
              "ciliated",
              "secretory",
              "club_cell",
              "basal_cell",
              "goblet_cell"
            ]
          },
          {
            "name": "cf_tme_breadth_broader_in_young",
            "role": "hard_fail",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_young"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-5-claude-code",
      "modelName": "claude-opus-5",
      "harness": "claude-code",
      "task": "gse297023_q6_age_immune_infiltration",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 1,
      "passed": true,
      "score": null,
      "durationS": 15193.44,
      "cost": 7.275424999999999,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "composition_stable_with_age",
            "role": "gate",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "frequencies_largely_stable"
            ]
          },
          {
            "name": "age_signal_is_state_not_composition",
            "role": "gate",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "frequencies_uninformative_for_infiltration",
            "role": "gate",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_cannot_establish_infiltration"
            ]
          },
          {
            "name": "infiltration_increases_with_age",
            "role": "gate",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_with_age"
            ]
          },
          {
            "name": "infiltration_strength_calibrated",
            "role": "gate",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "significant_increase_strongest_markers_only",
              "consistent_increase_not_statistically_robust"
            ]
          },
          {
            "name": "genotype_marker_fraction_lower_in_pten",
            "role": "gate",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "lower_in_pten_deficient"
            ]
          },
          {
            "name": "cf_composition_shift_reading",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "major_compositional_shift_with_age",
              "increased_immune_fraction_with_age"
            ]
          },
          {
            "name": "cf_lineage_specific_composition_claim",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "myeloid_expansion_with_age",
              "lymphoid_expansion_with_age",
              "minor_shifts_in_some_cell_types"
            ]
          },
          {
            "name": "cf_immunosenescence_depletion_prior",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "immune_depletion_with_age"
            ]
          },
          {
            "name": "cf_age_modality_compositional",
            "role": "hard_fail",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_type_composition_shift",
              "both_equally",
              "composition_dominant_state_secondary"
            ]
          },
          {
            "name": "cf_overread_frequencies_as_infiltration",
            "role": "hard_fail",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_shows_no_infiltration_change",
              "single_cell_quantifies_infiltration",
              "frequencies_show_increased_infiltration",
              "frequencies_show_decreased_infiltration"
            ]
          },
          {
            "name": "cf_infiltration_decreases_immunosenescence",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "decreased_with_age"
            ]
          },
          {
            "name": "cf_infiltration_unchanged_scrna_overread",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "unchanged_with_age"
            ]
          },
          {
            "name": "cf_infiltration_indeterminate_unused_node",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "direction_indeterminate_from_data"
            ]
          },
          {
            "name": "cf_infiltration_mixed_direction",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "mixed_by_immune_cell_type"
            ]
          },
          {
            "name": "cf_infiltration_genotype_confusion",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_only_in_pten_deficient",
              "decreased_only_in_pten_deficient"
            ]
          },
          {
            "name": "cf_pten_infiltration_paper_text_recall",
            "role": "hard_fail",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "higher_in_pten_deficient",
              "higher_in_pten_deficient_only_in_young",
              "higher_in_pten_deficient_only_in_aged"
            ]
          },
          {
            "name": "cf_pseudoreplication_overclaim",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_increase_across_markers"
            ]
          },
          {
            "name": "cf_infiltration_strength_decrease_direction",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_decrease_across_markers",
              "significant_decrease_strongest_markers_only",
              "consistent_decrease_not_statistically_robust"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-5-claude-code",
      "modelName": "claude-opus-5",
      "harness": "claude-code",
      "task": "gse297023_q6_age_immune_infiltration",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 3612.27,
      "cost": 4.286095250000001,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "composition_stable_with_age",
            "role": "gate",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "frequencies_largely_stable"
            ]
          },
          {
            "name": "age_signal_is_state_not_composition",
            "role": "gate",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "frequencies_uninformative_for_infiltration",
            "role": "gate",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_cannot_establish_infiltration"
            ]
          },
          {
            "name": "infiltration_increases_with_age",
            "role": "gate",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_with_age"
            ]
          },
          {
            "name": "infiltration_strength_calibrated",
            "role": "gate",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "significant_increase_strongest_markers_only",
              "consistent_increase_not_statistically_robust"
            ]
          },
          {
            "name": "genotype_marker_fraction_lower_in_pten",
            "role": "gate",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "lower_in_pten_deficient"
            ]
          },
          {
            "name": "cf_composition_shift_reading",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "major_compositional_shift_with_age",
              "increased_immune_fraction_with_age"
            ]
          },
          {
            "name": "cf_lineage_specific_composition_claim",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "myeloid_expansion_with_age",
              "lymphoid_expansion_with_age",
              "minor_shifts_in_some_cell_types"
            ]
          },
          {
            "name": "cf_immunosenescence_depletion_prior",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "immune_depletion_with_age"
            ]
          },
          {
            "name": "cf_age_modality_compositional",
            "role": "hard_fail",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_type_composition_shift",
              "both_equally",
              "composition_dominant_state_secondary"
            ]
          },
          {
            "name": "cf_overread_frequencies_as_infiltration",
            "role": "hard_fail",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_shows_no_infiltration_change",
              "single_cell_quantifies_infiltration",
              "frequencies_show_increased_infiltration",
              "frequencies_show_decreased_infiltration"
            ]
          },
          {
            "name": "cf_infiltration_decreases_immunosenescence",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "decreased_with_age"
            ]
          },
          {
            "name": "cf_infiltration_unchanged_scrna_overread",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "unchanged_with_age"
            ]
          },
          {
            "name": "cf_infiltration_indeterminate_unused_node",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "direction_indeterminate_from_data"
            ]
          },
          {
            "name": "cf_infiltration_mixed_direction",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "mixed_by_immune_cell_type"
            ]
          },
          {
            "name": "cf_infiltration_genotype_confusion",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_only_in_pten_deficient",
              "decreased_only_in_pten_deficient"
            ]
          },
          {
            "name": "cf_pten_infiltration_paper_text_recall",
            "role": "hard_fail",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "higher_in_pten_deficient",
              "higher_in_pten_deficient_only_in_young",
              "higher_in_pten_deficient_only_in_aged"
            ]
          },
          {
            "name": "cf_pseudoreplication_overclaim",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_increase_across_markers"
            ]
          },
          {
            "name": "cf_infiltration_strength_decrease_direction",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_decrease_across_markers",
              "significant_decrease_strongest_markers_only",
              "consistent_decrease_not_statistically_robust"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-5-claude-code",
      "modelName": "claude-opus-5",
      "harness": "claude-code",
      "task": "gse297023_q6_age_immune_infiltration",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": null,
      "cost": null,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "composition_stable_with_age",
            "role": "gate",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "frequencies_largely_stable"
            ]
          },
          {
            "name": "age_signal_is_state_not_composition",
            "role": "gate",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "frequencies_uninformative_for_infiltration",
            "role": "gate",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_cannot_establish_infiltration"
            ]
          },
          {
            "name": "infiltration_increases_with_age",
            "role": "gate",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_with_age"
            ]
          },
          {
            "name": "infiltration_strength_calibrated",
            "role": "gate",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "significant_increase_strongest_markers_only",
              "consistent_increase_not_statistically_robust"
            ]
          },
          {
            "name": "genotype_marker_fraction_lower_in_pten",
            "role": "gate",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "lower_in_pten_deficient"
            ]
          },
          {
            "name": "cf_composition_shift_reading",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "major_compositional_shift_with_age",
              "increased_immune_fraction_with_age"
            ]
          },
          {
            "name": "cf_lineage_specific_composition_claim",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "myeloid_expansion_with_age",
              "lymphoid_expansion_with_age",
              "minor_shifts_in_some_cell_types"
            ]
          },
          {
            "name": "cf_immunosenescence_depletion_prior",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "immune_depletion_with_age"
            ]
          },
          {
            "name": "cf_age_modality_compositional",
            "role": "hard_fail",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_type_composition_shift",
              "both_equally",
              "composition_dominant_state_secondary"
            ]
          },
          {
            "name": "cf_overread_frequencies_as_infiltration",
            "role": "hard_fail",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_shows_no_infiltration_change",
              "single_cell_quantifies_infiltration",
              "frequencies_show_increased_infiltration",
              "frequencies_show_decreased_infiltration"
            ]
          },
          {
            "name": "cf_infiltration_decreases_immunosenescence",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "decreased_with_age"
            ]
          },
          {
            "name": "cf_infiltration_unchanged_scrna_overread",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "unchanged_with_age"
            ]
          },
          {
            "name": "cf_infiltration_indeterminate_unused_node",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "direction_indeterminate_from_data"
            ]
          },
          {
            "name": "cf_infiltration_mixed_direction",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "mixed_by_immune_cell_type"
            ]
          },
          {
            "name": "cf_infiltration_genotype_confusion",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_only_in_pten_deficient",
              "decreased_only_in_pten_deficient"
            ]
          },
          {
            "name": "cf_pten_infiltration_paper_text_recall",
            "role": "hard_fail",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "higher_in_pten_deficient",
              "higher_in_pten_deficient_only_in_young",
              "higher_in_pten_deficient_only_in_aged"
            ]
          },
          {
            "name": "cf_pseudoreplication_overclaim",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_increase_across_markers"
            ]
          },
          {
            "name": "cf_infiltration_strength_decrease_direction",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_decrease_across_markers",
              "significant_decrease_strongest_markers_only",
              "consistent_decrease_not_statistically_robust"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-5",
      "modelName": "claude-opus-5",
      "harness": "pi",
      "task": "gse155381_q4_pseudotime_delay",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 1,
      "passed": true,
      "score": null,
      "durationS": 806.64,
      "cost": 2.3046534999999997,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "pseudotime_direction_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "E17_chronological_match_weighted",
            "role": "additive",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pluripotency_exit_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pseudotime_direction_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_synchronous_with_monkey_EPI",
              "chimera_human_EPI_developmentally_advanced",
              "chimera_human_EPI_delayed_at_pre_implantation_only_resolved_post_implantation"
            ]
          },
          {
            "name": "E17_chronological_match_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "resembles_monkey_E17_Gast_EPI",
              "resembles_chimera_monkey_E17_EPI_specifically",
              "chimera_human_E17_EPI_more_advanced_than_monkey_E17_Gast"
            ]
          },
          {
            "name": "pluripotency_exit_paper_inverted_or_prior_driven",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_advances_pluripotency_exit_prematurely",
              "naive_pluripotency_program_specifically_engaged",
              "pluripotency_core_retained_via_KLF17_naive_marker_specifically"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-5",
      "modelName": "claude-opus-5",
      "harness": "pi",
      "task": "gse155381_q4_pseudotime_delay",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 2,
      "passed": true,
      "score": null,
      "durationS": 891.59,
      "cost": 2.4210597500000004,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "pseudotime_direction_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "E17_chronological_match_weighted",
            "role": "additive",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pluripotency_exit_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pseudotime_direction_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_synchronous_with_monkey_EPI",
              "chimera_human_EPI_developmentally_advanced",
              "chimera_human_EPI_delayed_at_pre_implantation_only_resolved_post_implantation"
            ]
          },
          {
            "name": "E17_chronological_match_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "resembles_monkey_E17_Gast_EPI",
              "resembles_chimera_monkey_E17_EPI_specifically",
              "chimera_human_E17_EPI_more_advanced_than_monkey_E17_Gast"
            ]
          },
          {
            "name": "pluripotency_exit_paper_inverted_or_prior_driven",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_advances_pluripotency_exit_prematurely",
              "naive_pluripotency_program_specifically_engaged",
              "pluripotency_core_retained_via_KLF17_naive_marker_specifically"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-5",
      "modelName": "claude-opus-5",
      "harness": "pi",
      "task": "gse155381_q4_pseudotime_delay",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 3,
      "passed": true,
      "score": null,
      "durationS": 1009.67,
      "cost": 2.8771850000000003,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "pseudotime_direction_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "E17_chronological_match_weighted",
            "role": "additive",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pluripotency_exit_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pseudotime_direction_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_synchronous_with_monkey_EPI",
              "chimera_human_EPI_developmentally_advanced",
              "chimera_human_EPI_delayed_at_pre_implantation_only_resolved_post_implantation"
            ]
          },
          {
            "name": "E17_chronological_match_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "resembles_monkey_E17_Gast_EPI",
              "resembles_chimera_monkey_E17_EPI_specifically",
              "chimera_human_E17_EPI_more_advanced_than_monkey_E17_Gast"
            ]
          },
          {
            "name": "pluripotency_exit_paper_inverted_or_prior_driven",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_advances_pluripotency_exit_prematurely",
              "naive_pluripotency_program_specifically_engaged",
              "pluripotency_core_retained_via_KLF17_naive_marker_specifically"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-5",
      "modelName": "claude-opus-5",
      "harness": "pi",
      "task": "gse155381_q5_chimera_monkey_perturbation",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 728.13,
      "cost": 1.5261922499999998,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "deg_direction_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "comparison_subset_choice_weighted",
            "role": "additive",
            "field": "$.comparison_subset_choice",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "panel_resolved_strong_target_ccn1_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN1",
              "CYR61"
            ]
          },
          {
            "name": "panel_resolved_strong_target_ccn2_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN2",
              "CTGF"
            ]
          },
          {
            "name": "hippo_pathway_weighted",
            "role": "additive",
            "field": "$.hippo_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "tgf_beta_pathway_weighted",
            "role": "additive",
            "field": "$.tgf_beta_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "deg_direction_batch_effect_uncorrected",
            "role": "hard_fail",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_dominated_global",
              "up_dominated_in_chosen_subset"
            ]
          },
          {
            "name": "silent_panel_dropout_paired_with_directional_pathway_claim",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "hippo_pathway_direction": {
                "op": "in",
                "values": [
                  "hippo_pathway_upregulated",
                  "hippo_pathway_unchanged",
                  "wnt_pathway_specifically_perturbed",
                  "stress_response_pathway_dominantly_engaged",
                  "no_pathway_signal_above_confound",
                  "yap_tead_direct_target_panel_downregulated",
                  "yap_tead_panel_dependent_direction",
                  "hippo_pathway_downregulated"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-5",
      "modelName": "claude-opus-5",
      "harness": "pi",
      "task": "gse155381_q5_chimera_monkey_perturbation",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 1011.4,
      "cost": 2.03250625,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "deg_direction_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "comparison_subset_choice_weighted",
            "role": "additive",
            "field": "$.comparison_subset_choice",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "panel_resolved_strong_target_ccn1_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN1",
              "CYR61"
            ]
          },
          {
            "name": "panel_resolved_strong_target_ccn2_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN2",
              "CTGF"
            ]
          },
          {
            "name": "hippo_pathway_weighted",
            "role": "additive",
            "field": "$.hippo_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "tgf_beta_pathway_weighted",
            "role": "additive",
            "field": "$.tgf_beta_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "deg_direction_batch_effect_uncorrected",
            "role": "hard_fail",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_dominated_global",
              "up_dominated_in_chosen_subset"
            ]
          },
          {
            "name": "silent_panel_dropout_paired_with_directional_pathway_claim",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "hippo_pathway_direction": {
                "op": "in",
                "values": [
                  "hippo_pathway_upregulated",
                  "hippo_pathway_unchanged",
                  "wnt_pathway_specifically_perturbed",
                  "stress_response_pathway_dominantly_engaged",
                  "no_pathway_signal_above_confound",
                  "yap_tead_direct_target_panel_downregulated",
                  "yap_tead_panel_dependent_direction",
                  "hippo_pathway_downregulated"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-5",
      "modelName": "claude-opus-5",
      "harness": "pi",
      "task": "gse155381_q5_chimera_monkey_perturbation",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 890.19,
      "cost": 1.874474,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "deg_direction_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "comparison_subset_choice_weighted",
            "role": "additive",
            "field": "$.comparison_subset_choice",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "panel_resolved_strong_target_ccn1_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN1",
              "CYR61"
            ]
          },
          {
            "name": "panel_resolved_strong_target_ccn2_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN2",
              "CTGF"
            ]
          },
          {
            "name": "hippo_pathway_weighted",
            "role": "additive",
            "field": "$.hippo_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "tgf_beta_pathway_weighted",
            "role": "additive",
            "field": "$.tgf_beta_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "deg_direction_batch_effect_uncorrected",
            "role": "hard_fail",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_dominated_global",
              "up_dominated_in_chosen_subset"
            ]
          },
          {
            "name": "silent_panel_dropout_paired_with_directional_pathway_claim",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "hippo_pathway_direction": {
                "op": "in",
                "values": [
                  "hippo_pathway_upregulated",
                  "hippo_pathway_unchanged",
                  "wnt_pathway_specifically_perturbed",
                  "stress_response_pathway_dominantly_engaged",
                  "no_pathway_signal_above_confound",
                  "yap_tead_direct_target_panel_downregulated",
                  "yap_tead_panel_dependent_direction",
                  "hippo_pathway_downregulated"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-5",
      "modelName": "claude-opus-5",
      "harness": "pi",
      "task": "gse155381_q7_xspecies_epi_communication",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 1012.81,
      "cost": 3.03327675,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "richness_weighted",
            "role": "additive",
            "field": "$.xspecies_signaling_richness",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_weighted",
            "role": "additive",
            "field": "$.dominant_signaling_axis",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_wnt_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "WNT_FZD_LRP_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_fgf_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "FGF_FGFR_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_notch_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "NOTCH_DLL_JAG_family"
                ]
              }
            }
          },
          {
            "name": "signaling_directionality_weighted",
            "role": "additive",
            "field": "$.signaling_directionality",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "top_signal_interaction_class_weighted",
            "role": "additive",
            "field": "$.top_signal_interaction_class",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-5",
      "modelName": "claude-opus-5",
      "harness": "pi",
      "task": "gse155381_q7_xspecies_epi_communication",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 850.46,
      "cost": 2.3369025,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "richness_weighted",
            "role": "additive",
            "field": "$.xspecies_signaling_richness",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_weighted",
            "role": "additive",
            "field": "$.dominant_signaling_axis",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_wnt_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "WNT_FZD_LRP_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_fgf_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "FGF_FGFR_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_notch_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "NOTCH_DLL_JAG_family"
                ]
              }
            }
          },
          {
            "name": "signaling_directionality_weighted",
            "role": "additive",
            "field": "$.signaling_directionality",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "top_signal_interaction_class_weighted",
            "role": "additive",
            "field": "$.top_signal_interaction_class",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-5",
      "modelName": "claude-opus-5",
      "harness": "pi",
      "task": "gse155381_q7_xspecies_epi_communication",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 669.16,
      "cost": 1.819512,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "richness_weighted",
            "role": "additive",
            "field": "$.xspecies_signaling_richness",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_weighted",
            "role": "additive",
            "field": "$.dominant_signaling_axis",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_wnt_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "WNT_FZD_LRP_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_fgf_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "FGF_FGFR_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_notch_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "NOTCH_DLL_JAG_family"
                ]
              }
            }
          },
          {
            "name": "signaling_directionality_weighted",
            "role": "additive",
            "field": "$.signaling_directionality",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "top_signal_interaction_class_weighted",
            "role": "additive",
            "field": "$.top_signal_interaction_class",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-5",
      "modelName": "claude-opus-5",
      "harness": "pi",
      "task": "gse155381_q8_bidirectional_convergence",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 2087.03,
      "cost": 5.8288445,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "chimera_human_identity_shift_target_weighted",
            "role": "additive",
            "field": "$.chimera_human_identity_shift_target",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_in_vivo_state_assignment_weighted",
            "role": "additive",
            "field": "$.chimera_human_in_vivo_state_assignment",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_shift_magnitude_weighted",
            "role": "additive",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_monkey_identity_shift_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "both_arms_null_contradicted_by_calibration",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "chimera_human_identity_shift_target": {
                "op": "equals",
                "values": [
                  "chimera_human_maintains_origin_identity_no_measurable_shift"
                ]
              },
              "chimera_monkey_identity_shift": {
                "op": "equals",
                "values": [
                  "chimera_monkey_unchanged_relative_to_control"
                ]
              }
            }
          },
          {
            "name": "chimera_monkey_paper_claim_on_confounded_input",
            "role": "hard_fail",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "value",
            "expected": "chimera_monkey_shifts_toward_chimera_human_identity"
          },
          {
            "name": "chimera_human_magnitude_wrong_attribution_signature",
            "role": "hard_fail",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "value",
            "op": "in",
            "expected": [
              "dramatic_complete_replacement_of_origin_identity",
              "magnitude_dominated_by_culture_or_metabolic_axis",
              "magnitude_dominated_by_developmental_stage_axis",
              "magnitude_dominated_by_cell_cycle_axis",
              "magnitude_dominated_by_batch_effect_axis"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-5",
      "modelName": "claude-opus-5",
      "harness": "pi",
      "task": "gse155381_q8_bidirectional_convergence",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 2242.86,
      "cost": 8.544175,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "chimera_human_identity_shift_target_weighted",
            "role": "additive",
            "field": "$.chimera_human_identity_shift_target",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_in_vivo_state_assignment_weighted",
            "role": "additive",
            "field": "$.chimera_human_in_vivo_state_assignment",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_shift_magnitude_weighted",
            "role": "additive",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_monkey_identity_shift_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "both_arms_null_contradicted_by_calibration",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "chimera_human_identity_shift_target": {
                "op": "equals",
                "values": [
                  "chimera_human_maintains_origin_identity_no_measurable_shift"
                ]
              },
              "chimera_monkey_identity_shift": {
                "op": "equals",
                "values": [
                  "chimera_monkey_unchanged_relative_to_control"
                ]
              }
            }
          },
          {
            "name": "chimera_monkey_paper_claim_on_confounded_input",
            "role": "hard_fail",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "value",
            "expected": "chimera_monkey_shifts_toward_chimera_human_identity"
          },
          {
            "name": "chimera_human_magnitude_wrong_attribution_signature",
            "role": "hard_fail",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "value",
            "op": "in",
            "expected": [
              "dramatic_complete_replacement_of_origin_identity",
              "magnitude_dominated_by_culture_or_metabolic_axis",
              "magnitude_dominated_by_developmental_stage_axis",
              "magnitude_dominated_by_cell_cycle_axis",
              "magnitude_dominated_by_batch_effect_axis"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-5",
      "modelName": "claude-opus-5",
      "harness": "pi",
      "task": "gse155381_q8_bidirectional_convergence",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1686.72,
      "cost": 5.871976999999999,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "chimera_human_identity_shift_target_weighted",
            "role": "additive",
            "field": "$.chimera_human_identity_shift_target",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_in_vivo_state_assignment_weighted",
            "role": "additive",
            "field": "$.chimera_human_in_vivo_state_assignment",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_shift_magnitude_weighted",
            "role": "additive",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_monkey_identity_shift_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "both_arms_null_contradicted_by_calibration",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "chimera_human_identity_shift_target": {
                "op": "equals",
                "values": [
                  "chimera_human_maintains_origin_identity_no_measurable_shift"
                ]
              },
              "chimera_monkey_identity_shift": {
                "op": "equals",
                "values": [
                  "chimera_monkey_unchanged_relative_to_control"
                ]
              }
            }
          },
          {
            "name": "chimera_monkey_paper_claim_on_confounded_input",
            "role": "hard_fail",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "value",
            "expected": "chimera_monkey_shifts_toward_chimera_human_identity"
          },
          {
            "name": "chimera_human_magnitude_wrong_attribution_signature",
            "role": "hard_fail",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "value",
            "op": "in",
            "expected": [
              "dramatic_complete_replacement_of_origin_identity",
              "magnitude_dominated_by_culture_or_metabolic_axis",
              "magnitude_dominated_by_developmental_stage_axis",
              "magnitude_dominated_by_cell_cycle_axis",
              "magnitude_dominated_by_batch_effect_axis"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-5",
      "modelName": "claude-opus-5",
      "harness": "pi",
      "task": "gse171524_ectopic_airway_rare_cell",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 1,
      "passed": true,
      "score": null,
      "durationS": 3635.81,
      "cost": 6.167368500000001,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "focus_population_identity_gate",
            "role": "gate",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "chemosensory_marker_expressing_population_alpha"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "focus_population_marker_calls",
            "kind": "list",
            "matchKey": "marker",
            "k": 3,
            "expected": [
              {
                "key": "POU2F3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "CHAT",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "TRPM5",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "LRMP",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNAT3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "AVIL",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNG13",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              }
            ]
          },
          {
            "name": "focus_population_condition_distribution_gate",
            "role": "gate",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_enriched_with_substantial_control_presence",
              "covid_predominant",
              "covid_only",
              "covid_enriched_trend_not_significant"
            ]
          },
          {
            "name": "focus_population_location_gate",
            "role": "gate",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "ectopic_parenchyma_in_covid",
              "parenchyma_only",
              "bronchiolized_or_peribronchial_parenchyma",
              "airway_and_parenchyma"
            ]
          },
          {
            "name": "focus_population_recovery_mode_gate",
            "role": "gate",
            "field": "$.focus_population_recovery_mode",
            "kind": "value",
            "op": "in",
            "expected": [
              "continuum_not_discrete_cluster"
            ]
          },
          {
            "name": "focus_population_identity_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "ionocyte_like_beta",
              "neuroendocrine_like_gamma",
              "basal_stem_progenitor_like_delta",
              "aberrant_basaloid_transitional_like_epsilon",
              "neuronal_or_neural_gene_minor_population_zeta",
              "deuterosomal_ciliated_progenitor_like_eta",
              "hillock_transitional_like_theta",
              "low_quality_doublet_or_debris",
              "no_rare_cell_recovered",
              "not_assessed"
            ]
          },
          {
            "name": "cf_wrong_lineage_marker_called_positive",
            "role": "hard_fail",
            "field": "$.focus_population_marker_calls[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "focus_population_condition_distribution_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "equally_distributed",
              "control_predominant",
              "not_assessed"
            ]
          },
          {
            "name": "focus_population_location_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "airway_only",
              "not_assessed"
            ]
          },
          {
            "name": "cross_axis_canonical_identity_with_airway_only_location_incoherence",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "focus_population_identity": {
                "op": "in",
                "values": [
                  "chemosensory_marker_expressing_population_alpha"
                ]
              },
              "focus_population_location": {
                "op": "in",
                "values": [
                  "airway_only"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-5",
      "modelName": "claude-opus-5",
      "harness": "pi",
      "task": "gse171524_ectopic_airway_rare_cell",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 2,
      "passed": true,
      "score": null,
      "durationS": 2551.43,
      "cost": 3.5138335000000014,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "focus_population_identity_gate",
            "role": "gate",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "chemosensory_marker_expressing_population_alpha"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "focus_population_marker_calls",
            "kind": "list",
            "matchKey": "marker",
            "k": 3,
            "expected": [
              {
                "key": "POU2F3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "CHAT",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "TRPM5",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "LRMP",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNAT3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "AVIL",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNG13",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              }
            ]
          },
          {
            "name": "focus_population_condition_distribution_gate",
            "role": "gate",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_enriched_with_substantial_control_presence",
              "covid_predominant",
              "covid_only",
              "covid_enriched_trend_not_significant"
            ]
          },
          {
            "name": "focus_population_location_gate",
            "role": "gate",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "ectopic_parenchyma_in_covid",
              "parenchyma_only",
              "bronchiolized_or_peribronchial_parenchyma",
              "airway_and_parenchyma"
            ]
          },
          {
            "name": "focus_population_recovery_mode_gate",
            "role": "gate",
            "field": "$.focus_population_recovery_mode",
            "kind": "value",
            "op": "in",
            "expected": [
              "continuum_not_discrete_cluster"
            ]
          },
          {
            "name": "focus_population_identity_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "ionocyte_like_beta",
              "neuroendocrine_like_gamma",
              "basal_stem_progenitor_like_delta",
              "aberrant_basaloid_transitional_like_epsilon",
              "neuronal_or_neural_gene_minor_population_zeta",
              "deuterosomal_ciliated_progenitor_like_eta",
              "hillock_transitional_like_theta",
              "low_quality_doublet_or_debris",
              "no_rare_cell_recovered",
              "not_assessed"
            ]
          },
          {
            "name": "cf_wrong_lineage_marker_called_positive",
            "role": "hard_fail",
            "field": "$.focus_population_marker_calls[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "focus_population_condition_distribution_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "equally_distributed",
              "control_predominant",
              "not_assessed"
            ]
          },
          {
            "name": "focus_population_location_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "airway_only",
              "not_assessed"
            ]
          },
          {
            "name": "cross_axis_canonical_identity_with_airway_only_location_incoherence",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "focus_population_identity": {
                "op": "in",
                "values": [
                  "chemosensory_marker_expressing_population_alpha"
                ]
              },
              "focus_population_location": {
                "op": "in",
                "values": [
                  "airway_only"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-5",
      "modelName": "claude-opus-5",
      "harness": "pi",
      "task": "gse171524_ectopic_airway_rare_cell",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1946.76,
      "cost": 4.49578025,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "focus_population_identity_gate",
            "role": "gate",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "chemosensory_marker_expressing_population_alpha"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "focus_population_marker_calls",
            "kind": "list",
            "matchKey": "marker",
            "k": 3,
            "expected": [
              {
                "key": "POU2F3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "CHAT",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "TRPM5",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "LRMP",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNAT3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "AVIL",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNG13",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              }
            ]
          },
          {
            "name": "focus_population_condition_distribution_gate",
            "role": "gate",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_enriched_with_substantial_control_presence",
              "covid_predominant",
              "covid_only",
              "covid_enriched_trend_not_significant"
            ]
          },
          {
            "name": "focus_population_location_gate",
            "role": "gate",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "ectopic_parenchyma_in_covid",
              "parenchyma_only",
              "bronchiolized_or_peribronchial_parenchyma",
              "airway_and_parenchyma"
            ]
          },
          {
            "name": "focus_population_recovery_mode_gate",
            "role": "gate",
            "field": "$.focus_population_recovery_mode",
            "kind": "value",
            "op": "in",
            "expected": [
              "continuum_not_discrete_cluster"
            ]
          },
          {
            "name": "focus_population_identity_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "ionocyte_like_beta",
              "neuroendocrine_like_gamma",
              "basal_stem_progenitor_like_delta",
              "aberrant_basaloid_transitional_like_epsilon",
              "neuronal_or_neural_gene_minor_population_zeta",
              "deuterosomal_ciliated_progenitor_like_eta",
              "hillock_transitional_like_theta",
              "low_quality_doublet_or_debris",
              "no_rare_cell_recovered",
              "not_assessed"
            ]
          },
          {
            "name": "cf_wrong_lineage_marker_called_positive",
            "role": "hard_fail",
            "field": "$.focus_population_marker_calls[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "focus_population_condition_distribution_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "equally_distributed",
              "control_predominant",
              "not_assessed"
            ]
          },
          {
            "name": "focus_population_location_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "airway_only",
              "not_assessed"
            ]
          },
          {
            "name": "cross_axis_canonical_identity_with_airway_only_location_incoherence",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "focus_population_identity": {
                "op": "in",
                "values": [
                  "chemosensory_marker_expressing_population_alpha"
                ]
              },
              "focus_population_location": {
                "op": "in",
                "values": [
                  "airway_only"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-5",
      "modelName": "claude-opus-5",
      "harness": "pi",
      "task": "gse171524_pathological_fibroblast_master_regulator_druggable_targets",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 1,
      "passed": true,
      "score": null,
      "durationS": 14134.34,
      "cost": 12.15531475,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "disease_associated_fibroblast_subtype_weighted",
            "role": "additive",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "mr_tf_junb_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_alpha"
            ]
          },
          {
            "name": "mr_tf_jund_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_beta"
            ]
          },
          {
            "name": "drug_mmp14_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_alpha"
            ]
          },
          {
            "name": "drug_stat3_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_beta"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_weighted",
            "role": "additive",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "disease_associated_fibroblast_subtype_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "value",
            "op": "in",
            "expected": [
              "subtype_lambda",
              "subtype_epsilon",
              "subtype_iota",
              "subtype_beta",
              "subtype_eta",
              "subtype_zeta",
              "subtype_not_assessed"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "simple_de_ranking_by_log_fc",
              "marker_based_de_only",
              "not_computed"
            ]
          },
          {
            "name": "top_master_regulator_transcription_factors_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_not_computed"
            ]
          },
          {
            "name": "top_druggable_proteins_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_not_assessed"
            ]
          },
          {
            "name": "cross_axis_circular_marker_in_druggable_proteins",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_gamma",
              "drug_target_delta",
              "drug_target_eta"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-5",
      "modelName": "claude-opus-5",
      "harness": "pi",
      "task": "gse171524_pathological_fibroblast_master_regulator_druggable_targets",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 2,
      "passed": true,
      "score": null,
      "durationS": 1833.96,
      "cost": 1.71447075,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "disease_associated_fibroblast_subtype_weighted",
            "role": "additive",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "mr_tf_junb_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_alpha"
            ]
          },
          {
            "name": "mr_tf_jund_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_beta"
            ]
          },
          {
            "name": "drug_mmp14_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_alpha"
            ]
          },
          {
            "name": "drug_stat3_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_beta"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_weighted",
            "role": "additive",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "disease_associated_fibroblast_subtype_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "value",
            "op": "in",
            "expected": [
              "subtype_lambda",
              "subtype_epsilon",
              "subtype_iota",
              "subtype_beta",
              "subtype_eta",
              "subtype_zeta",
              "subtype_not_assessed"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "simple_de_ranking_by_log_fc",
              "marker_based_de_only",
              "not_computed"
            ]
          },
          {
            "name": "top_master_regulator_transcription_factors_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_not_computed"
            ]
          },
          {
            "name": "top_druggable_proteins_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_not_assessed"
            ]
          },
          {
            "name": "cross_axis_circular_marker_in_druggable_proteins",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_gamma",
              "drug_target_delta",
              "drug_target_eta"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-5",
      "modelName": "claude-opus-5",
      "harness": "pi",
      "task": "gse171524_pathological_fibroblast_master_regulator_druggable_targets",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 3,
      "passed": true,
      "score": null,
      "durationS": 4446.88,
      "cost": 4.408289250000001,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "disease_associated_fibroblast_subtype_weighted",
            "role": "additive",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "mr_tf_junb_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_alpha"
            ]
          },
          {
            "name": "mr_tf_jund_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_beta"
            ]
          },
          {
            "name": "drug_mmp14_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_alpha"
            ]
          },
          {
            "name": "drug_stat3_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_beta"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_weighted",
            "role": "additive",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "disease_associated_fibroblast_subtype_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "value",
            "op": "in",
            "expected": [
              "subtype_lambda",
              "subtype_epsilon",
              "subtype_iota",
              "subtype_beta",
              "subtype_eta",
              "subtype_zeta",
              "subtype_not_assessed"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "simple_de_ranking_by_log_fc",
              "marker_based_de_only",
              "not_computed"
            ]
          },
          {
            "name": "top_master_regulator_transcription_factors_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_not_computed"
            ]
          },
          {
            "name": "top_druggable_proteins_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_not_assessed"
            ]
          },
          {
            "name": "cross_axis_circular_marker_in_druggable_proteins",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_gamma",
              "drug_target_delta",
              "drug_target_eta"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-5",
      "modelName": "claude-opus-5",
      "harness": "pi",
      "task": "gse171524_plasma_bcr_neutralizing_clonotype",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 1,
      "passed": true,
      "score": null,
      "durationS": 931.25,
      "cost": 1.4233645,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dominant_pairing_heavy_gate",
            "role": "gate",
            "field": "$.dominant_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV4-34"
            ]
          },
          {
            "name": "dominant_pairing_light_gate",
            "role": "gate",
            "field": "$.dominant_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGLV3-19"
            ]
          },
          {
            "name": "dominant_neutralizer_match_gate",
            "role": "gate",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_known_neutralizer_match"
            ]
          },
          {
            "name": "neutralizer_pairing_heavy_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV1-18"
            ]
          },
          {
            "name": "neutralizer_pairing_light_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGKV3-20"
            ]
          },
          {
            "name": "neutralizer_recurrence_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "observed_in_2_to_5_donors"
            ]
          },
          {
            "name": "neutralizer_rank_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "top_5_pairing",
              "top_6_to_20_pairing"
            ]
          },
          {
            "name": "dominant_clone_isotype_gate",
            "role": "gate",
            "field": "$.dominant_clone_isotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "IgG"
            ]
          },
          {
            "name": "blineage_class_switch_contrast_gate",
            "role": "gate",
            "field": "$.blineage_class_switch_contrast",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_class_switched_control_higher_unswitched",
              "class_switched_in_both_cohorts"
            ]
          },
          {
            "name": "dominant_match_narrative_trap_or_null",
            "role": "hard_fail",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_rbd_neutralizer",
              "matches_known_autoreactive_clone",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_heavy",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_light",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_recurrence_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_rank_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "cross_axis_found_dominant_but_failed_to_scan_for_embedded_neutralizer",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_pairing_neutralizer_match": {
                "op": "in",
                "values": [
                  "no_known_neutralizer_match"
                ]
              },
              "neutralizer_pairing_heavy": {
                "op": "in",
                "values": [
                  "none_detected"
                ]
              }
            }
          },
          {
            "name": "neutralizer_pairing_not_exactly_the_s309_pair",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "op": "not",
            "expected": {
              "op": "and",
              "args": [
                {
                  "op": "field",
                  "name": "neutralizer_pairing_heavy",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGHV1-18"
                    ]
                  }
                },
                {
                  "op": "field",
                  "name": "neutralizer_pairing_light",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGKV3-20"
                    ]
                  }
                }
              ]
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-5",
      "modelName": "claude-opus-5",
      "harness": "pi",
      "task": "gse171524_plasma_bcr_neutralizing_clonotype",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 2,
      "passed": true,
      "score": null,
      "durationS": 1010.74,
      "cost": 2.2349490000000003,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dominant_pairing_heavy_gate",
            "role": "gate",
            "field": "$.dominant_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV4-34"
            ]
          },
          {
            "name": "dominant_pairing_light_gate",
            "role": "gate",
            "field": "$.dominant_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGLV3-19"
            ]
          },
          {
            "name": "dominant_neutralizer_match_gate",
            "role": "gate",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_known_neutralizer_match"
            ]
          },
          {
            "name": "neutralizer_pairing_heavy_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV1-18"
            ]
          },
          {
            "name": "neutralizer_pairing_light_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGKV3-20"
            ]
          },
          {
            "name": "neutralizer_recurrence_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "observed_in_2_to_5_donors"
            ]
          },
          {
            "name": "neutralizer_rank_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "top_5_pairing",
              "top_6_to_20_pairing"
            ]
          },
          {
            "name": "dominant_clone_isotype_gate",
            "role": "gate",
            "field": "$.dominant_clone_isotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "IgG"
            ]
          },
          {
            "name": "blineage_class_switch_contrast_gate",
            "role": "gate",
            "field": "$.blineage_class_switch_contrast",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_class_switched_control_higher_unswitched",
              "class_switched_in_both_cohorts"
            ]
          },
          {
            "name": "dominant_match_narrative_trap_or_null",
            "role": "hard_fail",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_rbd_neutralizer",
              "matches_known_autoreactive_clone",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_heavy",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_light",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_recurrence_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_rank_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "cross_axis_found_dominant_but_failed_to_scan_for_embedded_neutralizer",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_pairing_neutralizer_match": {
                "op": "in",
                "values": [
                  "no_known_neutralizer_match"
                ]
              },
              "neutralizer_pairing_heavy": {
                "op": "in",
                "values": [
                  "none_detected"
                ]
              }
            }
          },
          {
            "name": "neutralizer_pairing_not_exactly_the_s309_pair",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "op": "not",
            "expected": {
              "op": "and",
              "args": [
                {
                  "op": "field",
                  "name": "neutralizer_pairing_heavy",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGHV1-18"
                    ]
                  }
                },
                {
                  "op": "field",
                  "name": "neutralizer_pairing_light",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGKV3-20"
                    ]
                  }
                }
              ]
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-5",
      "modelName": "claude-opus-5",
      "harness": "pi",
      "task": "gse171524_plasma_bcr_neutralizing_clonotype",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 482.5,
      "cost": 0.8266144999999999,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dominant_pairing_heavy_gate",
            "role": "gate",
            "field": "$.dominant_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV4-34"
            ]
          },
          {
            "name": "dominant_pairing_light_gate",
            "role": "gate",
            "field": "$.dominant_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGLV3-19"
            ]
          },
          {
            "name": "dominant_neutralizer_match_gate",
            "role": "gate",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_known_neutralizer_match"
            ]
          },
          {
            "name": "neutralizer_pairing_heavy_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV1-18"
            ]
          },
          {
            "name": "neutralizer_pairing_light_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGKV3-20"
            ]
          },
          {
            "name": "neutralizer_recurrence_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "observed_in_2_to_5_donors"
            ]
          },
          {
            "name": "neutralizer_rank_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "top_5_pairing",
              "top_6_to_20_pairing"
            ]
          },
          {
            "name": "dominant_clone_isotype_gate",
            "role": "gate",
            "field": "$.dominant_clone_isotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "IgG"
            ]
          },
          {
            "name": "blineage_class_switch_contrast_gate",
            "role": "gate",
            "field": "$.blineage_class_switch_contrast",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_class_switched_control_higher_unswitched",
              "class_switched_in_both_cohorts"
            ]
          },
          {
            "name": "dominant_match_narrative_trap_or_null",
            "role": "hard_fail",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_rbd_neutralizer",
              "matches_known_autoreactive_clone",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_heavy",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_light",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_recurrence_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_rank_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "cross_axis_found_dominant_but_failed_to_scan_for_embedded_neutralizer",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_pairing_neutralizer_match": {
                "op": "in",
                "values": [
                  "no_known_neutralizer_match"
                ]
              },
              "neutralizer_pairing_heavy": {
                "op": "in",
                "values": [
                  "none_detected"
                ]
              }
            }
          },
          {
            "name": "neutralizer_pairing_not_exactly_the_s309_pair",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "op": "not",
            "expected": {
              "op": "and",
              "args": [
                {
                  "op": "field",
                  "name": "neutralizer_pairing_heavy",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGHV1-18"
                    ]
                  }
                },
                {
                  "op": "field",
                  "name": "neutralizer_pairing_light",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGKV3-20"
                    ]
                  }
                }
              ]
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-5",
      "modelName": "claude-opus-5",
      "harness": "pi",
      "task": "q6_myeloid_subset_restricted_de",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 1,
      "passed": true,
      "score": null,
      "durationS": 3173.16,
      "cost": 2.7078904999999995,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "myeloid_subset_with_strongest_change_gate",
            "role": "gate",
            "field": "$.myeloid_subset_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "subset_gamma"
            ]
          },
          {
            "name": "gene_with_strongest_change_gate",
            "role": "gate",
            "field": "$.gene_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "AXL",
              "PROS1"
            ]
          },
          {
            "name": "direction_of_change_gate",
            "role": "gate",
            "field": "$.direction_of_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "down_in_covid"
            ]
          },
          {
            "name": "biological_function_interpretation_gate",
            "role": "gate",
            "field": "$.biological_function_interpretation",
            "kind": "value",
            "op": "in",
            "expected": [
              "efferocytosis",
              "phagocytosis"
            ]
          },
          {
            "name": "myeloid_compartment_abundance_gate",
            "role": "gate",
            "field": "$.myeloid_compartment_abundance",
            "kind": "value",
            "op": "in",
            "expected": [
              "expands_in_covid"
            ]
          },
          {
            "name": "myeloid_expansion_localization_gate",
            "role": "gate",
            "field": "$.myeloid_expansion_localization",
            "kind": "value",
            "op": "in",
            "expected": [
              "recruited_driven_resident_dysfunction"
            ]
          },
          {
            "name": "dominant_signature_gate",
            "role": "gate",
            "field": "$.dominant_signature",
            "kind": "value",
            "op": "in",
            "expected": [
              "technical_or_agonal_artifact"
            ]
          },
          {
            "name": "dominant_signature_direction_gate",
            "role": "gate",
            "field": "$.dominant_signature_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_in_covid"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-5",
      "modelName": "claude-opus-5",
      "harness": "pi",
      "task": "q6_myeloid_subset_restricted_de",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 2,
      "passed": true,
      "score": null,
      "durationS": 1337.67,
      "cost": 2.5579549999999993,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "myeloid_subset_with_strongest_change_gate",
            "role": "gate",
            "field": "$.myeloid_subset_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "subset_gamma"
            ]
          },
          {
            "name": "gene_with_strongest_change_gate",
            "role": "gate",
            "field": "$.gene_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "AXL",
              "PROS1"
            ]
          },
          {
            "name": "direction_of_change_gate",
            "role": "gate",
            "field": "$.direction_of_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "down_in_covid"
            ]
          },
          {
            "name": "biological_function_interpretation_gate",
            "role": "gate",
            "field": "$.biological_function_interpretation",
            "kind": "value",
            "op": "in",
            "expected": [
              "efferocytosis",
              "phagocytosis"
            ]
          },
          {
            "name": "myeloid_compartment_abundance_gate",
            "role": "gate",
            "field": "$.myeloid_compartment_abundance",
            "kind": "value",
            "op": "in",
            "expected": [
              "expands_in_covid"
            ]
          },
          {
            "name": "myeloid_expansion_localization_gate",
            "role": "gate",
            "field": "$.myeloid_expansion_localization",
            "kind": "value",
            "op": "in",
            "expected": [
              "recruited_driven_resident_dysfunction"
            ]
          },
          {
            "name": "dominant_signature_gate",
            "role": "gate",
            "field": "$.dominant_signature",
            "kind": "value",
            "op": "in",
            "expected": [
              "technical_or_agonal_artifact"
            ]
          },
          {
            "name": "dominant_signature_direction_gate",
            "role": "gate",
            "field": "$.dominant_signature_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_in_covid"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-5",
      "modelName": "claude-opus-5",
      "harness": "pi",
      "task": "q6_myeloid_subset_restricted_de",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 3,
      "passed": true,
      "score": null,
      "durationS": 2632.13,
      "cost": 2.5945875000000003,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "myeloid_subset_with_strongest_change_gate",
            "role": "gate",
            "field": "$.myeloid_subset_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "subset_gamma"
            ]
          },
          {
            "name": "gene_with_strongest_change_gate",
            "role": "gate",
            "field": "$.gene_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "AXL",
              "PROS1"
            ]
          },
          {
            "name": "direction_of_change_gate",
            "role": "gate",
            "field": "$.direction_of_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "down_in_covid"
            ]
          },
          {
            "name": "biological_function_interpretation_gate",
            "role": "gate",
            "field": "$.biological_function_interpretation",
            "kind": "value",
            "op": "in",
            "expected": [
              "efferocytosis",
              "phagocytosis"
            ]
          },
          {
            "name": "myeloid_compartment_abundance_gate",
            "role": "gate",
            "field": "$.myeloid_compartment_abundance",
            "kind": "value",
            "op": "in",
            "expected": [
              "expands_in_covid"
            ]
          },
          {
            "name": "myeloid_expansion_localization_gate",
            "role": "gate",
            "field": "$.myeloid_expansion_localization",
            "kind": "value",
            "op": "in",
            "expected": [
              "recruited_driven_resident_dysfunction"
            ]
          },
          {
            "name": "dominant_signature_gate",
            "role": "gate",
            "field": "$.dominant_signature",
            "kind": "value",
            "op": "in",
            "expected": [
              "technical_or_agonal_artifact"
            ]
          },
          {
            "name": "dominant_signature_direction_gate",
            "role": "gate",
            "field": "$.dominant_signature_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_in_covid"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-5",
      "modelName": "claude-opus-5",
      "harness": "pi",
      "task": "snrnaseq_per_donor_composition_shifts_lethal_covid_lung",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 1942.91,
      "cost": 1.0892860000000002,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dict_match",
            "role": "gate",
            "field": "cell_type_direction_map",
            "kind": "dict",
            "expected": {
              "Myeloid": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "Epithelia": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Fibroblasts": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "T_NK": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "B_plasma": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Endothelial": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Mast": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "APC": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Neuronal": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              }
            }
          },
          {
            "name": "over_merge_immune_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Immune",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_leukocyte_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Leukocyte",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_lymphoid_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Lymphoid",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_stromal_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Stromal",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-5",
      "modelName": "claude-opus-5",
      "harness": "pi",
      "task": "snrnaseq_per_donor_composition_shifts_lethal_covid_lung",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 2858.8,
      "cost": 1.6059575000000004,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dict_match",
            "role": "gate",
            "field": "cell_type_direction_map",
            "kind": "dict",
            "expected": {
              "Myeloid": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "Epithelia": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Fibroblasts": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "T_NK": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "B_plasma": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Endothelial": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Mast": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "APC": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Neuronal": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              }
            }
          },
          {
            "name": "over_merge_immune_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Immune",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_leukocyte_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Leukocyte",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_lymphoid_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Lymphoid",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_stromal_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Stromal",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-5",
      "modelName": "claude-opus-5",
      "harness": "pi",
      "task": "snrnaseq_per_donor_composition_shifts_lethal_covid_lung",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 3014.01,
      "cost": 2.466524,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dict_match",
            "role": "gate",
            "field": "cell_type_direction_map",
            "kind": "dict",
            "expected": {
              "Myeloid": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "Epithelia": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Fibroblasts": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "T_NK": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "B_plasma": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Endothelial": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Mast": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "APC": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Neuronal": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              }
            }
          },
          {
            "name": "over_merge_immune_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Immune",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_leukocyte_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Leukocyte",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_lymphoid_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Lymphoid",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_stromal_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Stromal",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-5",
      "modelName": "claude-opus-5",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_enriched_apc_states_v2",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 1,
      "passed": true,
      "score": null,
      "durationS": 2474.54,
      "cost": 5.917551499999998,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "top_3_cluster_enriched_apc_states",
            "kind": "list",
            "matchKey": "apc_state",
            "k": 3,
            "expected": [
              {
                "key": "plasmacytoid_DC",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "T_cell_stimulatory_role_supported",
                      "role_direction_unresolved"
                    ],
                    "canonical": "T_cell_stimulatory_role_supported"
                  }
                }
              },
              {
                "key": "C1qhigh_lipid_associated_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "C1qhigh_inflammatory_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "plasma_cell",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "cohort_significant_per_patient_inconsistent"
                    ],
                    "canonical": "cohort_significant_per_patient_inconsistent"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CD80_CD86": {
                "op": "in",
                "value": [
                  "mreg_DC",
                  "shared_across_APC_states"
                ]
              },
              "PD_L1_in_APC_clusters": {
                "op": "in",
                "value": [
                  "C1qhigh_macrophage_either_subtype",
                  "shared_across_APC_states",
                  "mreg_DC"
                ]
              },
              "type_I_IFN": {
                "op": "in",
                "value": [
                  "plasmacytoid_DC",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "cf_mreg_dc_claimed_enriched",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "mreg_DC",
              "mature_DC_enriched_in_immunoregulatory_molecules"
            ]
          },
          {
            "name": "cf_framework_offcompartment_or_generic_substitution",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "cDC1",
              "Maier_mregDC",
              "Cheng_C1QC_TAM",
              "Cheng_SPP1_TAM",
              "Cheng_FOLR2_TAM",
              "Villani_pDC",
              "Zhang_inflammatory_monocyte",
              "LAMP3_pos_DC",
              "myeloid_derived_suppressor_cell",
              "MDSC",
              "tumor_associated_neutrophil",
              "mast_cell",
              "APC",
              "antigen_presenting_cell",
              "macrophage",
              "dendritic_cell",
              "tumor_associated_macrophage",
              "TAM"
            ]
          },
          {
            "name": "cf_direction_commit_on_unresolvable_slot",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_not_resolvable_hedge",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].functional_attribution_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_type_I_IFN_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.type_I_IFN",
            "kind": "value",
            "op": "in",
            "expected": [
              "mreg_DC",
              "C1qhigh_macrophage_either_subtype"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-5",
      "modelName": "claude-opus-5",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_enriched_apc_states_v2",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 2,
      "passed": true,
      "score": null,
      "durationS": 1846.19,
      "cost": 4.591994250000002,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "top_3_cluster_enriched_apc_states",
            "kind": "list",
            "matchKey": "apc_state",
            "k": 3,
            "expected": [
              {
                "key": "plasmacytoid_DC",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "T_cell_stimulatory_role_supported",
                      "role_direction_unresolved"
                    ],
                    "canonical": "T_cell_stimulatory_role_supported"
                  }
                }
              },
              {
                "key": "C1qhigh_lipid_associated_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "C1qhigh_inflammatory_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "plasma_cell",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "cohort_significant_per_patient_inconsistent"
                    ],
                    "canonical": "cohort_significant_per_patient_inconsistent"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CD80_CD86": {
                "op": "in",
                "value": [
                  "mreg_DC",
                  "shared_across_APC_states"
                ]
              },
              "PD_L1_in_APC_clusters": {
                "op": "in",
                "value": [
                  "C1qhigh_macrophage_either_subtype",
                  "shared_across_APC_states",
                  "mreg_DC"
                ]
              },
              "type_I_IFN": {
                "op": "in",
                "value": [
                  "plasmacytoid_DC",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "cf_mreg_dc_claimed_enriched",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "mreg_DC",
              "mature_DC_enriched_in_immunoregulatory_molecules"
            ]
          },
          {
            "name": "cf_framework_offcompartment_or_generic_substitution",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "cDC1",
              "Maier_mregDC",
              "Cheng_C1QC_TAM",
              "Cheng_SPP1_TAM",
              "Cheng_FOLR2_TAM",
              "Villani_pDC",
              "Zhang_inflammatory_monocyte",
              "LAMP3_pos_DC",
              "myeloid_derived_suppressor_cell",
              "MDSC",
              "tumor_associated_neutrophil",
              "mast_cell",
              "APC",
              "antigen_presenting_cell",
              "macrophage",
              "dendritic_cell",
              "tumor_associated_macrophage",
              "TAM"
            ]
          },
          {
            "name": "cf_direction_commit_on_unresolvable_slot",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_not_resolvable_hedge",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].functional_attribution_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_type_I_IFN_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.type_I_IFN",
            "kind": "value",
            "op": "in",
            "expected": [
              "mreg_DC",
              "C1qhigh_macrophage_either_subtype"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-5",
      "modelName": "claude-opus-5",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_enriched_apc_states_v2",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 2462.09,
      "cost": 5.732581250000001,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "top_3_cluster_enriched_apc_states",
            "kind": "list",
            "matchKey": "apc_state",
            "k": 3,
            "expected": [
              {
                "key": "plasmacytoid_DC",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "T_cell_stimulatory_role_supported",
                      "role_direction_unresolved"
                    ],
                    "canonical": "T_cell_stimulatory_role_supported"
                  }
                }
              },
              {
                "key": "C1qhigh_lipid_associated_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "C1qhigh_inflammatory_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "plasma_cell",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "cohort_significant_per_patient_inconsistent"
                    ],
                    "canonical": "cohort_significant_per_patient_inconsistent"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CD80_CD86": {
                "op": "in",
                "value": [
                  "mreg_DC",
                  "shared_across_APC_states"
                ]
              },
              "PD_L1_in_APC_clusters": {
                "op": "in",
                "value": [
                  "C1qhigh_macrophage_either_subtype",
                  "shared_across_APC_states",
                  "mreg_DC"
                ]
              },
              "type_I_IFN": {
                "op": "in",
                "value": [
                  "plasmacytoid_DC",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "cf_mreg_dc_claimed_enriched",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "mreg_DC",
              "mature_DC_enriched_in_immunoregulatory_molecules"
            ]
          },
          {
            "name": "cf_framework_offcompartment_or_generic_substitution",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "cDC1",
              "Maier_mregDC",
              "Cheng_C1QC_TAM",
              "Cheng_SPP1_TAM",
              "Cheng_FOLR2_TAM",
              "Villani_pDC",
              "Zhang_inflammatory_monocyte",
              "LAMP3_pos_DC",
              "myeloid_derived_suppressor_cell",
              "MDSC",
              "tumor_associated_neutrophil",
              "mast_cell",
              "APC",
              "antigen_presenting_cell",
              "macrophage",
              "dendritic_cell",
              "tumor_associated_macrophage",
              "TAM"
            ]
          },
          {
            "name": "cf_direction_commit_on_unresolvable_slot",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_not_resolvable_hedge",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].functional_attribution_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_type_I_IFN_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.type_I_IFN",
            "kind": "value",
            "op": "in",
            "expected": [
              "mreg_DC",
              "C1qhigh_macrophage_either_subtype"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-5",
      "modelName": "claude-opus-5",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_tumor_state_causation_v5",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 2175.18,
      "cost": 4.020879,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "enriched_tumor_states",
            "kind": "list",
            "matchKey": "tumor_state",
            "k": 3,
            "expected": [
              {
                "key": "immune_response",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "direction_unresolved_cross_sectional"
                    ],
                    "canonical": "direction_unresolved_cross_sectional"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "IFN_response_axis",
                      "broad_antigen_presentation_MHC"
                    ],
                    "canonical": "IFN_response_axis"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "enrichment_diminished_by_decontamination",
                      "threshold_sensitive"
                    ],
                    "canonical": "enrichment_diminished_by_decontamination"
                  }
                }
              },
              {
                "key": "stress_hypoxia",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "data_supports_cell_intrinsic_cause"
                    ],
                    "canonical": "data_supports_cell_intrinsic_cause"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "HIF_hypoxia_program_specific"
                    ],
                    "canonical": "HIF_hypoxia_program_specific"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "robust_across_methods",
                      "robust_after_decontamination"
                    ],
                    "canonical": "robust_across_methods"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CXCL9_CXCL10": {
                "op": "in",
                "value": [
                  "immune_response_tumor_state"
                ]
              },
              "CCL5": {
                "op": "in",
                "value": [
                  "shared_across_tumor_states",
                  "partner_lymphocyte_derived",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "gate_conjugate_gate_confounder",
            "role": "gate",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "partner_lymphocyte_capture"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "antigen_presentation_decomposition",
            "kind": "dict",
            "expected": {
              "MHC_class_I": {
                "op": "in",
                "value": [
                  "co_regulated_with_dominant_program",
                  "tumor_intrinsic_enriched_independent"
                ]
              },
              "MHC_class_II": {
                "op": "in",
                "value": [
                  "partner_cell_attributable",
                  "flat_not_enriched",
                  "method_sensitive_no_robust_call",
                  "co_regulated_with_dominant_program"
                ]
              }
            }
          },
          {
            "name": "cf_non_robust_state_claimed_enriched",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "transitory_melanocytic",
              "intermediate_melanocytic",
              "melanocytic",
              "MITF_high",
              "neural_crest_like",
              "mitotic",
              "stress_p53",
              "undifferentiated",
              "dedifferentiated",
              "mesenchymal_tumor",
              "AXL_high_tumor",
              "invasive_tumor",
              "senescence_like",
              "EMT_transition",
              "cancer_stem_cell_like",
              "oxidative_stress_tumor"
            ]
          },
          {
            "name": "cf_framework_generic_or_offtissue_substitution",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "Tsoi_undifferentiated",
              "Tsoi_neural_crest_like",
              "Tsoi_transitory",
              "Tsoi_melanocytic",
              "Rambow_NCSC",
              "Wouters_mesenchymal",
              "Pozniak_TCF4_dependent",
              "tumor_cell",
              "melanoma_cell",
              "malignant_cell",
              "tumor_subpopulation",
              "keratinocyte",
              "hepatocyte",
              "lung_adenocarcinoma_cell",
              "colon_adenocarcinoma_cell",
              "Tirosh_cell_cycle_program",
              "Jerby_Arnon_program",
              "glioblastoma_cell",
              "breast_carcinoma_cell"
            ]
          },
          {
            "name": "cf_immune_response_direction_committed",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_immune_robustness_recall",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_ambiguous_hedge",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].causation_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_stress_generic_stress_response_evidence",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_conjugate_confounder_contamination_denied",
            "role": "hard_fail",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected"
            ]
          },
          {
            "name": "cf_ccl5_recall_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CCL5",
            "kind": "avoid",
            "expected": [
              "immune_response_tumor_state"
            ]
          },
          {
            "name": "cf_cxcl9_cxcl10_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CXCL9_CXCL10",
            "kind": "avoid",
            "expected": [
              "stress_hypoxia_tumor_state"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-5",
      "modelName": "claude-opus-5",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_tumor_state_causation_v5",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 2933.87,
      "cost": 3.3463725,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "enriched_tumor_states",
            "kind": "list",
            "matchKey": "tumor_state",
            "k": 3,
            "expected": [
              {
                "key": "immune_response",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "direction_unresolved_cross_sectional"
                    ],
                    "canonical": "direction_unresolved_cross_sectional"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "IFN_response_axis",
                      "broad_antigen_presentation_MHC"
                    ],
                    "canonical": "IFN_response_axis"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "enrichment_diminished_by_decontamination",
                      "threshold_sensitive"
                    ],
                    "canonical": "enrichment_diminished_by_decontamination"
                  }
                }
              },
              {
                "key": "stress_hypoxia",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "data_supports_cell_intrinsic_cause"
                    ],
                    "canonical": "data_supports_cell_intrinsic_cause"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "HIF_hypoxia_program_specific"
                    ],
                    "canonical": "HIF_hypoxia_program_specific"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "robust_across_methods",
                      "robust_after_decontamination"
                    ],
                    "canonical": "robust_across_methods"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CXCL9_CXCL10": {
                "op": "in",
                "value": [
                  "immune_response_tumor_state"
                ]
              },
              "CCL5": {
                "op": "in",
                "value": [
                  "shared_across_tumor_states",
                  "partner_lymphocyte_derived",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "gate_conjugate_gate_confounder",
            "role": "gate",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "partner_lymphocyte_capture"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "antigen_presentation_decomposition",
            "kind": "dict",
            "expected": {
              "MHC_class_I": {
                "op": "in",
                "value": [
                  "co_regulated_with_dominant_program",
                  "tumor_intrinsic_enriched_independent"
                ]
              },
              "MHC_class_II": {
                "op": "in",
                "value": [
                  "partner_cell_attributable",
                  "flat_not_enriched",
                  "method_sensitive_no_robust_call",
                  "co_regulated_with_dominant_program"
                ]
              }
            }
          },
          {
            "name": "cf_non_robust_state_claimed_enriched",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "transitory_melanocytic",
              "intermediate_melanocytic",
              "melanocytic",
              "MITF_high",
              "neural_crest_like",
              "mitotic",
              "stress_p53",
              "undifferentiated",
              "dedifferentiated",
              "mesenchymal_tumor",
              "AXL_high_tumor",
              "invasive_tumor",
              "senescence_like",
              "EMT_transition",
              "cancer_stem_cell_like",
              "oxidative_stress_tumor"
            ]
          },
          {
            "name": "cf_framework_generic_or_offtissue_substitution",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "Tsoi_undifferentiated",
              "Tsoi_neural_crest_like",
              "Tsoi_transitory",
              "Tsoi_melanocytic",
              "Rambow_NCSC",
              "Wouters_mesenchymal",
              "Pozniak_TCF4_dependent",
              "tumor_cell",
              "melanoma_cell",
              "malignant_cell",
              "tumor_subpopulation",
              "keratinocyte",
              "hepatocyte",
              "lung_adenocarcinoma_cell",
              "colon_adenocarcinoma_cell",
              "Tirosh_cell_cycle_program",
              "Jerby_Arnon_program",
              "glioblastoma_cell",
              "breast_carcinoma_cell"
            ]
          },
          {
            "name": "cf_immune_response_direction_committed",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_immune_robustness_recall",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_ambiguous_hedge",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].causation_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_stress_generic_stress_response_evidence",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_conjugate_confounder_contamination_denied",
            "role": "hard_fail",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected"
            ]
          },
          {
            "name": "cf_ccl5_recall_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CCL5",
            "kind": "avoid",
            "expected": [
              "immune_response_tumor_state"
            ]
          },
          {
            "name": "cf_cxcl9_cxcl10_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CXCL9_CXCL10",
            "kind": "avoid",
            "expected": [
              "stress_hypoxia_tumor_state"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-5",
      "modelName": "claude-opus-5",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_tumor_state_causation_v5",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 3436.32,
      "cost": 5.118010499999999,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "enriched_tumor_states",
            "kind": "list",
            "matchKey": "tumor_state",
            "k": 3,
            "expected": [
              {
                "key": "immune_response",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "direction_unresolved_cross_sectional"
                    ],
                    "canonical": "direction_unresolved_cross_sectional"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "IFN_response_axis",
                      "broad_antigen_presentation_MHC"
                    ],
                    "canonical": "IFN_response_axis"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "enrichment_diminished_by_decontamination",
                      "threshold_sensitive"
                    ],
                    "canonical": "enrichment_diminished_by_decontamination"
                  }
                }
              },
              {
                "key": "stress_hypoxia",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "data_supports_cell_intrinsic_cause"
                    ],
                    "canonical": "data_supports_cell_intrinsic_cause"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "HIF_hypoxia_program_specific"
                    ],
                    "canonical": "HIF_hypoxia_program_specific"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "robust_across_methods",
                      "robust_after_decontamination"
                    ],
                    "canonical": "robust_across_methods"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CXCL9_CXCL10": {
                "op": "in",
                "value": [
                  "immune_response_tumor_state"
                ]
              },
              "CCL5": {
                "op": "in",
                "value": [
                  "shared_across_tumor_states",
                  "partner_lymphocyte_derived",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "gate_conjugate_gate_confounder",
            "role": "gate",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "partner_lymphocyte_capture"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "antigen_presentation_decomposition",
            "kind": "dict",
            "expected": {
              "MHC_class_I": {
                "op": "in",
                "value": [
                  "co_regulated_with_dominant_program",
                  "tumor_intrinsic_enriched_independent"
                ]
              },
              "MHC_class_II": {
                "op": "in",
                "value": [
                  "partner_cell_attributable",
                  "flat_not_enriched",
                  "method_sensitive_no_robust_call",
                  "co_regulated_with_dominant_program"
                ]
              }
            }
          },
          {
            "name": "cf_non_robust_state_claimed_enriched",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "transitory_melanocytic",
              "intermediate_melanocytic",
              "melanocytic",
              "MITF_high",
              "neural_crest_like",
              "mitotic",
              "stress_p53",
              "undifferentiated",
              "dedifferentiated",
              "mesenchymal_tumor",
              "AXL_high_tumor",
              "invasive_tumor",
              "senescence_like",
              "EMT_transition",
              "cancer_stem_cell_like",
              "oxidative_stress_tumor"
            ]
          },
          {
            "name": "cf_framework_generic_or_offtissue_substitution",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "Tsoi_undifferentiated",
              "Tsoi_neural_crest_like",
              "Tsoi_transitory",
              "Tsoi_melanocytic",
              "Rambow_NCSC",
              "Wouters_mesenchymal",
              "Pozniak_TCF4_dependent",
              "tumor_cell",
              "melanoma_cell",
              "malignant_cell",
              "tumor_subpopulation",
              "keratinocyte",
              "hepatocyte",
              "lung_adenocarcinoma_cell",
              "colon_adenocarcinoma_cell",
              "Tirosh_cell_cycle_program",
              "Jerby_Arnon_program",
              "glioblastoma_cell",
              "breast_carcinoma_cell"
            ]
          },
          {
            "name": "cf_immune_response_direction_committed",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_immune_robustness_recall",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_ambiguous_hedge",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].causation_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_stress_generic_stress_response_evidence",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_conjugate_confounder_contamination_denied",
            "role": "hard_fail",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected"
            ]
          },
          {
            "name": "cf_ccl5_recall_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CCL5",
            "kind": "avoid",
            "expected": [
              "immune_response_tumor_state"
            ]
          },
          {
            "name": "cf_cxcl9_cxcl10_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CXCL9_CXCL10",
            "kind": "avoid",
            "expected": [
              "stress_hypoxia_tumor_state"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-5",
      "modelName": "claude-opus-5",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_vs_cd39_tumor_reactivity_dominance",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 1,
      "passed": true,
      "score": null,
      "durationS": 1453.35,
      "cost": 3.4309142500000003,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "population_findings",
            "kind": "list",
            "matchKey": "capture_gate",
            "k": 4,
            "expected": [
              {
                "key": "tumor_cell_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "apc_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "combined_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "unselected_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "naive",
                      "central_memory",
                      "effector_memory"
                    ],
                    "canonical": "naive"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "low"
                    ],
                    "canonical": "low"
                  },
                  "purity_caveat": {
                    "op": "equals",
                    "values": [
                      "pure_T_cell"
                    ],
                    "canonical": "pure_T_cell"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "no_reactivity_evidence"
                    ],
                    "canonical": "no_reactivity_evidence"
                  }
                }
              },
              {
                "key": "marker_sorted_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "equals",
                    "values": [
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "terminal_exhausted_LAG3_high"
                  },
                  "clonal_expansion_level": {
                    "op": "in",
                    "values": [
                      "high",
                      "intermediate"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "marker_on_non_T_lineages",
                      "single_cell_gating_loses_target"
                    ],
                    "canonical": "marker_on_non_T_lineages"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "marker_or_exhaustion_defined_only"
                    ],
                    "canonical": "marker_or_exhaustion_defined_only"
                  }
                }
              }
            ]
          },
          {
            "name": "best_capture_approach_gate",
            "role": "gate",
            "field": "best_capture_approach",
            "kind": "value",
            "expected": "heterotypic_cell_conjugate"
          },
          {
            "name": "best_selection_marker_gate",
            "role": "gate",
            "field": "best_selection_marker",
            "kind": "value",
            "expected": "none"
          },
          {
            "name": "reactivity_criterion_gate",
            "role": "gate",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "clonal_expansion_and_physical_engagement"
          },
          {
            "name": "cross_patient_robustness_gate",
            "role": "gate",
            "field": "cross_patient_robustness",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_across_patients",
              "marginal_in_some_patients"
            ]
          },
          {
            "name": "marker_selection_consequence_gate",
            "role": "gate",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "discards_clonally_expanded_reactive_cells"
          },
          {
            "name": "superiority_claim_basis_gate",
            "role": "gate",
            "field": "superiority_claim_basis",
            "kind": "value",
            "expected": "requires_functional_reactivity_assay"
          },
          {
            "name": "conjugate_enriched_state_gate",
            "role": "gate",
            "field": "conjugate_enriched_state",
            "kind": "value",
            "expected": "progenitor_exhausted_TCF7_pos_stem_like"
          },
          {
            "name": "cf_circular_exhaustion_criterion",
            "role": "hard_fail",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "exhaustion_signature_intensity"
          },
          {
            "name": "cf_marker_sort_prior",
            "role": "hard_fail",
            "field": "best_selection_marker",
            "kind": "value",
            "op": "in",
            "expected": [
              "CD39",
              "PD1",
              "PD1_high",
              "TIM3",
              "PD1_CD39",
              "CD39_CD103"
            ]
          },
          {
            "name": "cf_single_cell_capture_prior",
            "role": "hard_fail",
            "field": "best_capture_approach",
            "kind": "value",
            "op": "in",
            "expected": [
              "dissociated_single_cell",
              "dissociated_bulk_population"
            ]
          },
          {
            "name": "cf_off_data_capture_gate",
            "role": "hard_fail",
            "field": "$.population_findings[*].capture_gate",
            "kind": "avoid",
            "expected": [
              "dissociated_bulk",
              "matched_peripheral_blood",
              "matched_lymph_node",
              "post_hoc_expanded_clonotype_group"
            ]
          },
          {
            "name": "cf_conjugate_mischaracterized",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_marker_singlet_called_engaged",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "contains",
            "matchKey": "capture_gate",
            "entity": "marker_sorted_singlet",
            "valueField": "reactivity_evidence",
            "expected": [
              "engagement_and_clonal_expansion"
            ]
          },
          {
            "name": "cf_no_engaged_reactive_population",
            "role": "hard_fail",
            "field": "$.population_findings[*].reactivity_evidence",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_marker_captures_all_reactive",
            "role": "hard_fail",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "captures_full_reactive_repertoire"
          },
          {
            "name": "cf_superiority_overclaimed_from_phenotype",
            "role": "hard_fail",
            "field": "superiority_claim_basis",
            "kind": "value",
            "op": "in",
            "expected": [
              "established_by_scrna_phenotype",
              "established_by_reactivity_proxy_scores"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-5",
      "modelName": "claude-opus-5",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_vs_cd39_tumor_reactivity_dominance",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 2,
      "passed": true,
      "score": null,
      "durationS": 1828.45,
      "cost": 2.655991,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "population_findings",
            "kind": "list",
            "matchKey": "capture_gate",
            "k": 4,
            "expected": [
              {
                "key": "tumor_cell_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "apc_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "combined_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "unselected_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "naive",
                      "central_memory",
                      "effector_memory"
                    ],
                    "canonical": "naive"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "low"
                    ],
                    "canonical": "low"
                  },
                  "purity_caveat": {
                    "op": "equals",
                    "values": [
                      "pure_T_cell"
                    ],
                    "canonical": "pure_T_cell"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "no_reactivity_evidence"
                    ],
                    "canonical": "no_reactivity_evidence"
                  }
                }
              },
              {
                "key": "marker_sorted_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "equals",
                    "values": [
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "terminal_exhausted_LAG3_high"
                  },
                  "clonal_expansion_level": {
                    "op": "in",
                    "values": [
                      "high",
                      "intermediate"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "marker_on_non_T_lineages",
                      "single_cell_gating_loses_target"
                    ],
                    "canonical": "marker_on_non_T_lineages"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "marker_or_exhaustion_defined_only"
                    ],
                    "canonical": "marker_or_exhaustion_defined_only"
                  }
                }
              }
            ]
          },
          {
            "name": "best_capture_approach_gate",
            "role": "gate",
            "field": "best_capture_approach",
            "kind": "value",
            "expected": "heterotypic_cell_conjugate"
          },
          {
            "name": "best_selection_marker_gate",
            "role": "gate",
            "field": "best_selection_marker",
            "kind": "value",
            "expected": "none"
          },
          {
            "name": "reactivity_criterion_gate",
            "role": "gate",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "clonal_expansion_and_physical_engagement"
          },
          {
            "name": "cross_patient_robustness_gate",
            "role": "gate",
            "field": "cross_patient_robustness",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_across_patients",
              "marginal_in_some_patients"
            ]
          },
          {
            "name": "marker_selection_consequence_gate",
            "role": "gate",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "discards_clonally_expanded_reactive_cells"
          },
          {
            "name": "superiority_claim_basis_gate",
            "role": "gate",
            "field": "superiority_claim_basis",
            "kind": "value",
            "expected": "requires_functional_reactivity_assay"
          },
          {
            "name": "conjugate_enriched_state_gate",
            "role": "gate",
            "field": "conjugate_enriched_state",
            "kind": "value",
            "expected": "progenitor_exhausted_TCF7_pos_stem_like"
          },
          {
            "name": "cf_circular_exhaustion_criterion",
            "role": "hard_fail",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "exhaustion_signature_intensity"
          },
          {
            "name": "cf_marker_sort_prior",
            "role": "hard_fail",
            "field": "best_selection_marker",
            "kind": "value",
            "op": "in",
            "expected": [
              "CD39",
              "PD1",
              "PD1_high",
              "TIM3",
              "PD1_CD39",
              "CD39_CD103"
            ]
          },
          {
            "name": "cf_single_cell_capture_prior",
            "role": "hard_fail",
            "field": "best_capture_approach",
            "kind": "value",
            "op": "in",
            "expected": [
              "dissociated_single_cell",
              "dissociated_bulk_population"
            ]
          },
          {
            "name": "cf_off_data_capture_gate",
            "role": "hard_fail",
            "field": "$.population_findings[*].capture_gate",
            "kind": "avoid",
            "expected": [
              "dissociated_bulk",
              "matched_peripheral_blood",
              "matched_lymph_node",
              "post_hoc_expanded_clonotype_group"
            ]
          },
          {
            "name": "cf_conjugate_mischaracterized",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_marker_singlet_called_engaged",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "contains",
            "matchKey": "capture_gate",
            "entity": "marker_sorted_singlet",
            "valueField": "reactivity_evidence",
            "expected": [
              "engagement_and_clonal_expansion"
            ]
          },
          {
            "name": "cf_no_engaged_reactive_population",
            "role": "hard_fail",
            "field": "$.population_findings[*].reactivity_evidence",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_marker_captures_all_reactive",
            "role": "hard_fail",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "captures_full_reactive_repertoire"
          },
          {
            "name": "cf_superiority_overclaimed_from_phenotype",
            "role": "hard_fail",
            "field": "superiority_claim_basis",
            "kind": "value",
            "op": "in",
            "expected": [
              "established_by_scrna_phenotype",
              "established_by_reactivity_proxy_scores"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-5",
      "modelName": "claude-opus-5",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_vs_cd39_tumor_reactivity_dominance",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1727.43,
      "cost": 3.1998434999999996,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "population_findings",
            "kind": "list",
            "matchKey": "capture_gate",
            "k": 4,
            "expected": [
              {
                "key": "tumor_cell_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "apc_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "combined_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "unselected_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "naive",
                      "central_memory",
                      "effector_memory"
                    ],
                    "canonical": "naive"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "low"
                    ],
                    "canonical": "low"
                  },
                  "purity_caveat": {
                    "op": "equals",
                    "values": [
                      "pure_T_cell"
                    ],
                    "canonical": "pure_T_cell"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "no_reactivity_evidence"
                    ],
                    "canonical": "no_reactivity_evidence"
                  }
                }
              },
              {
                "key": "marker_sorted_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "equals",
                    "values": [
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "terminal_exhausted_LAG3_high"
                  },
                  "clonal_expansion_level": {
                    "op": "in",
                    "values": [
                      "high",
                      "intermediate"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "marker_on_non_T_lineages",
                      "single_cell_gating_loses_target"
                    ],
                    "canonical": "marker_on_non_T_lineages"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "marker_or_exhaustion_defined_only"
                    ],
                    "canonical": "marker_or_exhaustion_defined_only"
                  }
                }
              }
            ]
          },
          {
            "name": "best_capture_approach_gate",
            "role": "gate",
            "field": "best_capture_approach",
            "kind": "value",
            "expected": "heterotypic_cell_conjugate"
          },
          {
            "name": "best_selection_marker_gate",
            "role": "gate",
            "field": "best_selection_marker",
            "kind": "value",
            "expected": "none"
          },
          {
            "name": "reactivity_criterion_gate",
            "role": "gate",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "clonal_expansion_and_physical_engagement"
          },
          {
            "name": "cross_patient_robustness_gate",
            "role": "gate",
            "field": "cross_patient_robustness",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_across_patients",
              "marginal_in_some_patients"
            ]
          },
          {
            "name": "marker_selection_consequence_gate",
            "role": "gate",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "discards_clonally_expanded_reactive_cells"
          },
          {
            "name": "superiority_claim_basis_gate",
            "role": "gate",
            "field": "superiority_claim_basis",
            "kind": "value",
            "expected": "requires_functional_reactivity_assay"
          },
          {
            "name": "conjugate_enriched_state_gate",
            "role": "gate",
            "field": "conjugate_enriched_state",
            "kind": "value",
            "expected": "progenitor_exhausted_TCF7_pos_stem_like"
          },
          {
            "name": "cf_circular_exhaustion_criterion",
            "role": "hard_fail",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "exhaustion_signature_intensity"
          },
          {
            "name": "cf_marker_sort_prior",
            "role": "hard_fail",
            "field": "best_selection_marker",
            "kind": "value",
            "op": "in",
            "expected": [
              "CD39",
              "PD1",
              "PD1_high",
              "TIM3",
              "PD1_CD39",
              "CD39_CD103"
            ]
          },
          {
            "name": "cf_single_cell_capture_prior",
            "role": "hard_fail",
            "field": "best_capture_approach",
            "kind": "value",
            "op": "in",
            "expected": [
              "dissociated_single_cell",
              "dissociated_bulk_population"
            ]
          },
          {
            "name": "cf_off_data_capture_gate",
            "role": "hard_fail",
            "field": "$.population_findings[*].capture_gate",
            "kind": "avoid",
            "expected": [
              "dissociated_bulk",
              "matched_peripheral_blood",
              "matched_lymph_node",
              "post_hoc_expanded_clonotype_group"
            ]
          },
          {
            "name": "cf_conjugate_mischaracterized",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_marker_singlet_called_engaged",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "contains",
            "matchKey": "capture_gate",
            "entity": "marker_sorted_singlet",
            "valueField": "reactivity_evidence",
            "expected": [
              "engagement_and_clonal_expansion"
            ]
          },
          {
            "name": "cf_no_engaged_reactive_population",
            "role": "hard_fail",
            "field": "$.population_findings[*].reactivity_evidence",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_marker_captures_all_reactive",
            "role": "hard_fail",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "captures_full_reactive_repertoire"
          },
          {
            "name": "cf_superiority_overclaimed_from_phenotype",
            "role": "hard_fail",
            "field": "superiority_claim_basis",
            "kind": "value",
            "op": "in",
            "expected": [
              "established_by_scrna_phenotype",
              "established_by_reactivity_proxy_scores"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-5",
      "modelName": "claude-opus-5",
      "harness": "pi",
      "task": "10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 1,
      "passed": true,
      "score": null,
      "durationS": 1486.11,
      "cost": 3.4903,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_singlet_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "TOXhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "LAG3high_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "GZMKhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "intermediate_exhausted_gzmk"
                    ],
                    "canonical": "intermediate_exhausted_gzmk"
                  }
                }
              },
              {
                "key": "TCF7_stem_like_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "stem_like_exhausted"
                    ],
                    "canonical": "stem_like_exhausted"
                  }
                }
              },
              {
                "key": "MKI67_pos_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              },
              {
                "key": "MKI67high_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_tumor_cluster_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "Tn",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "Tn_Tmem",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "early_Tem_GZMK",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              },
              {
                "key": "Tem_GZMH",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              }
            ]
          },
          {
            "name": "clonotype_expansion_direction_data_true",
            "role": "gate",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_substantially_more_expanded_than_cluster",
              "singlet_modestly_more_expanded_than_cluster"
            ]
          },
          {
            "name": "biological_inference_data_true",
            "role": "gate",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_antigen_experienced_cluster_bystander"
            ]
          },
          {
            "name": "singlet_top3_contains_naive_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Tn",
              "Tn_Tmem",
              "early_Tem_GZMK",
              "Tem_GZMH"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "TOXhigh_Tex",
              "LAG3high_Tex",
              "GZMKhigh_Tex",
              "TCF7_stem_like_Tex",
              "MKI67_pos_Tex_Tprol",
              "MKI67high_Tex_Tprol"
            ]
          },
          {
            "name": "singlet_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "TOXhigh_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "TCF7_stem_like_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "MKI67_state_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "Tn_or_Tn_Tmem_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_stemlike_precursor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_reported_canonical_expansion_direction",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_substantially_more_expanded_than_singlet",
              "cluster_modestly_more_expanded_than_singlet"
            ]
          },
          {
            "name": "cf_expansion_lazy_defer",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "expansion_pattern_not_detectable"
            ]
          },
          {
            "name": "cf_reported_canonical_expected_biology",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_antigen_experienced_singlet_bystander"
            ]
          },
          {
            "name": "cf_biological_inference_null_artifact_or_lineage",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "differences_reflect_dissociation_or_sorting_artifact",
              "cluster_and_singlet_capture_same_biology",
              "cluster_and_singlet_capture_distinct_cd8_lineages"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "biological_relations",
            "kind": "dict",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_greater_than",
                  "greater_than"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "less_than",
                  "much_less_than"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than",
                  "approximately_equal"
                ]
              },
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal"
                ]
              },
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal",
                  "greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_exhaustion_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_naive_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_proliferation_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_expansion_richness_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_multichain_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_intermediate_gzmk_overgeneralized",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_melanocytic_conjugate_overcall",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "values": [
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_relations_mass_hedge",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-5",
      "modelName": "claude-opus-5",
      "harness": "pi",
      "task": "10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 2,
      "passed": true,
      "score": null,
      "durationS": 1804.17,
      "cost": 4.957795750000001,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_singlet_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "TOXhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "LAG3high_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "GZMKhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "intermediate_exhausted_gzmk"
                    ],
                    "canonical": "intermediate_exhausted_gzmk"
                  }
                }
              },
              {
                "key": "TCF7_stem_like_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "stem_like_exhausted"
                    ],
                    "canonical": "stem_like_exhausted"
                  }
                }
              },
              {
                "key": "MKI67_pos_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              },
              {
                "key": "MKI67high_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_tumor_cluster_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "Tn",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "Tn_Tmem",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "early_Tem_GZMK",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              },
              {
                "key": "Tem_GZMH",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              }
            ]
          },
          {
            "name": "clonotype_expansion_direction_data_true",
            "role": "gate",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_substantially_more_expanded_than_cluster",
              "singlet_modestly_more_expanded_than_cluster"
            ]
          },
          {
            "name": "biological_inference_data_true",
            "role": "gate",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_antigen_experienced_cluster_bystander"
            ]
          },
          {
            "name": "singlet_top3_contains_naive_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Tn",
              "Tn_Tmem",
              "early_Tem_GZMK",
              "Tem_GZMH"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "TOXhigh_Tex",
              "LAG3high_Tex",
              "GZMKhigh_Tex",
              "TCF7_stem_like_Tex",
              "MKI67_pos_Tex_Tprol",
              "MKI67high_Tex_Tprol"
            ]
          },
          {
            "name": "singlet_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "TOXhigh_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "TCF7_stem_like_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "MKI67_state_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "Tn_or_Tn_Tmem_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_stemlike_precursor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_reported_canonical_expansion_direction",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_substantially_more_expanded_than_singlet",
              "cluster_modestly_more_expanded_than_singlet"
            ]
          },
          {
            "name": "cf_expansion_lazy_defer",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "expansion_pattern_not_detectable"
            ]
          },
          {
            "name": "cf_reported_canonical_expected_biology",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_antigen_experienced_singlet_bystander"
            ]
          },
          {
            "name": "cf_biological_inference_null_artifact_or_lineage",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "differences_reflect_dissociation_or_sorting_artifact",
              "cluster_and_singlet_capture_same_biology",
              "cluster_and_singlet_capture_distinct_cd8_lineages"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "biological_relations",
            "kind": "dict",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_greater_than",
                  "greater_than"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "less_than",
                  "much_less_than"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than",
                  "approximately_equal"
                ]
              },
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal"
                ]
              },
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal",
                  "greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_exhaustion_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_naive_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_proliferation_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_expansion_richness_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_multichain_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_intermediate_gzmk_overgeneralized",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_melanocytic_conjugate_overcall",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "values": [
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_relations_mass_hedge",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-5",
      "modelName": "claude-opus-5",
      "harness": "pi",
      "task": "10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1774.65,
      "cost": 4.307621749999999,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_singlet_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "TOXhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "LAG3high_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "GZMKhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "intermediate_exhausted_gzmk"
                    ],
                    "canonical": "intermediate_exhausted_gzmk"
                  }
                }
              },
              {
                "key": "TCF7_stem_like_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "stem_like_exhausted"
                    ],
                    "canonical": "stem_like_exhausted"
                  }
                }
              },
              {
                "key": "MKI67_pos_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              },
              {
                "key": "MKI67high_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_tumor_cluster_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "Tn",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "Tn_Tmem",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "early_Tem_GZMK",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              },
              {
                "key": "Tem_GZMH",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              }
            ]
          },
          {
            "name": "clonotype_expansion_direction_data_true",
            "role": "gate",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_substantially_more_expanded_than_cluster",
              "singlet_modestly_more_expanded_than_cluster"
            ]
          },
          {
            "name": "biological_inference_data_true",
            "role": "gate",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_antigen_experienced_cluster_bystander"
            ]
          },
          {
            "name": "singlet_top3_contains_naive_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Tn",
              "Tn_Tmem",
              "early_Tem_GZMK",
              "Tem_GZMH"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "TOXhigh_Tex",
              "LAG3high_Tex",
              "GZMKhigh_Tex",
              "TCF7_stem_like_Tex",
              "MKI67_pos_Tex_Tprol",
              "MKI67high_Tex_Tprol"
            ]
          },
          {
            "name": "singlet_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "TOXhigh_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "TCF7_stem_like_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "MKI67_state_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "Tn_or_Tn_Tmem_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_stemlike_precursor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_reported_canonical_expansion_direction",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_substantially_more_expanded_than_singlet",
              "cluster_modestly_more_expanded_than_singlet"
            ]
          },
          {
            "name": "cf_expansion_lazy_defer",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "expansion_pattern_not_detectable"
            ]
          },
          {
            "name": "cf_reported_canonical_expected_biology",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_antigen_experienced_singlet_bystander"
            ]
          },
          {
            "name": "cf_biological_inference_null_artifact_or_lineage",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "differences_reflect_dissociation_or_sorting_artifact",
              "cluster_and_singlet_capture_same_biology",
              "cluster_and_singlet_capture_distinct_cd8_lineages"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "biological_relations",
            "kind": "dict",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_greater_than",
                  "greater_than"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "less_than",
                  "much_less_than"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than",
                  "approximately_equal"
                ]
              },
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal"
                ]
              },
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal",
                  "greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_exhaustion_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_naive_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_proliferation_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_expansion_richness_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_multichain_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_intermediate_gzmk_overgeneralized",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_melanocytic_conjugate_overcall",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "values": [
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_relations_mass_hedge",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-5",
      "modelName": "claude-opus-5",
      "harness": "pi",
      "task": "10x_5p_melanoma_tcr_matched_apc_vs_tumor_exhaustion",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 1,
      "passed": true,
      "score": null,
      "durationS": 2266.75,
      "cost": 3.3050137499999996,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "state_discriminating_genes",
            "kind": "list",
            "matchKey": "gene",
            "k": 3,
            "expected": [
              {
                "key": "CTLA4",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TOX",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "ENTPD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "HAVCR2",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TIGIT",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "PDCD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "difference_attributable_to",
            "role": "gate",
            "field": "difference_attributable_to",
            "kind": "value",
            "expected": "covaries_with_apc_conjugate_state"
          },
          {
            "name": "causal_status",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "direction_unresolved_selection_confound"
          },
          {
            "name": "functional_interpretation",
            "role": "gate",
            "field": "functional_interpretation",
            "kind": "value",
            "expected": "costimulation_signature_function_unresolved"
          },
          {
            "name": "cf_contamination_gene_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "MLANA",
              "PMEL",
              "DCT",
              "TYR",
              "TYRP1",
              "SOX10",
              "MITF",
              "MAGEA4"
            ]
          },
          {
            "name": "cf_apc_partner_contamination_gene",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "HLA-DRA",
              "CD68",
              "LYZ",
              "CD14",
              "C1QA",
              "ITGAX",
              "CD1C"
            ]
          },
          {
            "name": "cf_ligand_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "CD274",
              "PDCD1LG2",
              "CD80",
              "CD86"
            ]
          },
          {
            "name": "cf_attribution_causal_or_baseline",
            "role": "hard_fail",
            "field": "$.difference_attributable_to",
            "kind": "avoid",
            "expected": [
              "apc_engagement",
              "tumor_cell_engagement",
              "clonotype_intrinsic_baseline"
            ]
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "engagement_induces_exhausted_state",
              "pre_exhausted_cells_select_into_apc_conjugates"
            ]
          },
          {
            "name": "cf_causal_fabrication",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "resolved_by_pseudotemporal_ordering",
              "resolved_by_perturbation_experiment"
            ]
          },
          {
            "name": "cf_functional_overclaim",
            "role": "hard_fail",
            "field": "$.functional_interpretation",
            "kind": "avoid",
            "expected": [
              "costimulation_licensed_reinvigoration_competent",
              "costimulation_licensed_checkpoint_responsive",
              "tumor_driven_terminal_exhaustion"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-5",
      "modelName": "claude-opus-5",
      "harness": "pi",
      "task": "10x_5p_melanoma_tcr_matched_apc_vs_tumor_exhaustion",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 1065.21,
      "cost": 1.9056555,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "state_discriminating_genes",
            "kind": "list",
            "matchKey": "gene",
            "k": 3,
            "expected": [
              {
                "key": "CTLA4",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TOX",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "ENTPD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "HAVCR2",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TIGIT",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "PDCD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "difference_attributable_to",
            "role": "gate",
            "field": "difference_attributable_to",
            "kind": "value",
            "expected": "covaries_with_apc_conjugate_state"
          },
          {
            "name": "causal_status",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "direction_unresolved_selection_confound"
          },
          {
            "name": "functional_interpretation",
            "role": "gate",
            "field": "functional_interpretation",
            "kind": "value",
            "expected": "costimulation_signature_function_unresolved"
          },
          {
            "name": "cf_contamination_gene_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "MLANA",
              "PMEL",
              "DCT",
              "TYR",
              "TYRP1",
              "SOX10",
              "MITF",
              "MAGEA4"
            ]
          },
          {
            "name": "cf_apc_partner_contamination_gene",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "HLA-DRA",
              "CD68",
              "LYZ",
              "CD14",
              "C1QA",
              "ITGAX",
              "CD1C"
            ]
          },
          {
            "name": "cf_ligand_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "CD274",
              "PDCD1LG2",
              "CD80",
              "CD86"
            ]
          },
          {
            "name": "cf_attribution_causal_or_baseline",
            "role": "hard_fail",
            "field": "$.difference_attributable_to",
            "kind": "avoid",
            "expected": [
              "apc_engagement",
              "tumor_cell_engagement",
              "clonotype_intrinsic_baseline"
            ]
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "engagement_induces_exhausted_state",
              "pre_exhausted_cells_select_into_apc_conjugates"
            ]
          },
          {
            "name": "cf_causal_fabrication",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "resolved_by_pseudotemporal_ordering",
              "resolved_by_perturbation_experiment"
            ]
          },
          {
            "name": "cf_functional_overclaim",
            "role": "hard_fail",
            "field": "$.functional_interpretation",
            "kind": "avoid",
            "expected": [
              "costimulation_licensed_reinvigoration_competent",
              "costimulation_licensed_checkpoint_responsive",
              "tumor_driven_terminal_exhaustion"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-5",
      "modelName": "claude-opus-5",
      "harness": "pi",
      "task": "10x_5p_melanoma_tcr_matched_apc_vs_tumor_exhaustion",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 2131.15,
      "cost": 4.271839,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "state_discriminating_genes",
            "kind": "list",
            "matchKey": "gene",
            "k": 3,
            "expected": [
              {
                "key": "CTLA4",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TOX",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "ENTPD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "HAVCR2",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TIGIT",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "PDCD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "difference_attributable_to",
            "role": "gate",
            "field": "difference_attributable_to",
            "kind": "value",
            "expected": "covaries_with_apc_conjugate_state"
          },
          {
            "name": "causal_status",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "direction_unresolved_selection_confound"
          },
          {
            "name": "functional_interpretation",
            "role": "gate",
            "field": "functional_interpretation",
            "kind": "value",
            "expected": "costimulation_signature_function_unresolved"
          },
          {
            "name": "cf_contamination_gene_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "MLANA",
              "PMEL",
              "DCT",
              "TYR",
              "TYRP1",
              "SOX10",
              "MITF",
              "MAGEA4"
            ]
          },
          {
            "name": "cf_apc_partner_contamination_gene",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "HLA-DRA",
              "CD68",
              "LYZ",
              "CD14",
              "C1QA",
              "ITGAX",
              "CD1C"
            ]
          },
          {
            "name": "cf_ligand_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "CD274",
              "PDCD1LG2",
              "CD80",
              "CD86"
            ]
          },
          {
            "name": "cf_attribution_causal_or_baseline",
            "role": "hard_fail",
            "field": "$.difference_attributable_to",
            "kind": "avoid",
            "expected": [
              "apc_engagement",
              "tumor_cell_engagement",
              "clonotype_intrinsic_baseline"
            ]
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "engagement_induces_exhausted_state",
              "pre_exhausted_cells_select_into_apc_conjugates"
            ]
          },
          {
            "name": "cf_causal_fabrication",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "resolved_by_pseudotemporal_ordering",
              "resolved_by_perturbation_experiment"
            ]
          },
          {
            "name": "cf_functional_overclaim",
            "role": "hard_fail",
            "field": "$.functional_interpretation",
            "kind": "avoid",
            "expected": [
              "costimulation_licensed_reinvigoration_competent",
              "costimulation_licensed_checkpoint_responsive",
              "tumor_driven_terminal_exhaustion"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-5",
      "modelName": "claude-opus-5",
      "harness": "pi",
      "task": "green_2_tme_context_dependence",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 1911.9,
      "cost": 4.606211,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "program_map",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "sig_includes_itgae",
            "role": "gate",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Itgae"
            ]
          },
          {
            "name": "declared_a_known_confounder",
            "role": "gate",
            "field": "$.excluded_confounders[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "cf_signature_gut_contaminant",
            "role": "hard_fail",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "generic_signature_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd103_program_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "<"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "per_model_regulatory_state",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "poised_unfired"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "late_activation"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "closing_committed"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_tuples",
            "kind": "list",
            "matchKey": "regulator",
            "k": 1,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "role": {
                    "op": "equals",
                    "values": [
                      "repressor"
                    ]
                  },
                  "gating_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  },
                  "evidence_basis": {
                    "op": "equals",
                    "values": [
                      "target_program_behavior"
                    ]
                  },
                  "expr_vs_activity": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "locus_differential_gate",
            "role": "gate",
            "field": "locus_differential",
            "kind": "value",
            "expected": "uniform_across_models"
          },
          {
            "name": "formation_limited_by_gate",
            "role": "gate",
            "field": "formation_limited_by",
            "kind": "value",
            "expected": "absent_induction"
          },
          {
            "name": "locus_competent_in_naive_gate",
            "role": "gate",
            "field": "locus_competent_in_naive",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "ready_silent_fraction_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 3,
            "expected": [
              {
                "key": "B16_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_KPC_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "activator_chromatin_footprint_gate",
            "role": "gate",
            "field": "activator_chromatin_footprint",
            "kind": "value",
            "expected": "inconsistent_across_formers"
          },
          {
            "name": "exhaustion_comparable_gate",
            "role": "gate",
            "field": "exhaustion_comparable_across_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_nondeterminant_gate",
            "role": "gate",
            "field": "exhaustion_explains_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_separable_program_gate",
            "role": "gate",
            "field": "exhaustion_separable_program",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "exhaustion_counterexample_gate",
            "role": "gate",
            "field": "exhaustion_counterexample_model",
            "kind": "value",
            "expected": "KPC_GP"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "exhaustion_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "cf_inconsistent_present_without_activator",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "program",
            "entity": "present",
            "valueField": "activator_state",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_no_block_but_absent",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "none",
            "valueField": "program",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_late_block_without_poised_locus",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "late_activation",
            "valueField": "locus_state",
            "expected": [
              "closing_committed",
              "active"
            ]
          },
          {
            "name": "cf_inconsistent_early_block_with_brake_released",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "early_release",
            "valueField": "brake_state",
            "expected": [
              "released"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-5",
      "modelName": "claude-opus-5",
      "harness": "pi",
      "task": "green_2_tme_context_dependence",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 2140.74,
      "cost": 5.523880749999999,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "program_map",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "sig_includes_itgae",
            "role": "gate",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Itgae"
            ]
          },
          {
            "name": "declared_a_known_confounder",
            "role": "gate",
            "field": "$.excluded_confounders[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "cf_signature_gut_contaminant",
            "role": "hard_fail",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "generic_signature_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd103_program_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "<"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "per_model_regulatory_state",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "poised_unfired"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "late_activation"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "closing_committed"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_tuples",
            "kind": "list",
            "matchKey": "regulator",
            "k": 1,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "role": {
                    "op": "equals",
                    "values": [
                      "repressor"
                    ]
                  },
                  "gating_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  },
                  "evidence_basis": {
                    "op": "equals",
                    "values": [
                      "target_program_behavior"
                    ]
                  },
                  "expr_vs_activity": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "locus_differential_gate",
            "role": "gate",
            "field": "locus_differential",
            "kind": "value",
            "expected": "uniform_across_models"
          },
          {
            "name": "formation_limited_by_gate",
            "role": "gate",
            "field": "formation_limited_by",
            "kind": "value",
            "expected": "absent_induction"
          },
          {
            "name": "locus_competent_in_naive_gate",
            "role": "gate",
            "field": "locus_competent_in_naive",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "ready_silent_fraction_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 3,
            "expected": [
              {
                "key": "B16_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_KPC_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "activator_chromatin_footprint_gate",
            "role": "gate",
            "field": "activator_chromatin_footprint",
            "kind": "value",
            "expected": "inconsistent_across_formers"
          },
          {
            "name": "exhaustion_comparable_gate",
            "role": "gate",
            "field": "exhaustion_comparable_across_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_nondeterminant_gate",
            "role": "gate",
            "field": "exhaustion_explains_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_separable_program_gate",
            "role": "gate",
            "field": "exhaustion_separable_program",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "exhaustion_counterexample_gate",
            "role": "gate",
            "field": "exhaustion_counterexample_model",
            "kind": "value",
            "expected": "KPC_GP"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "exhaustion_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "cf_inconsistent_present_without_activator",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "program",
            "entity": "present",
            "valueField": "activator_state",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_no_block_but_absent",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "none",
            "valueField": "program",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_late_block_without_poised_locus",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "late_activation",
            "valueField": "locus_state",
            "expected": [
              "closing_committed",
              "active"
            ]
          },
          {
            "name": "cf_inconsistent_early_block_with_brake_released",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "early_release",
            "valueField": "brake_state",
            "expected": [
              "released"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-5",
      "modelName": "claude-opus-5",
      "harness": "pi",
      "task": "green_2_tme_context_dependence",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 2157.05,
      "cost": 5.15529075,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "program_map",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "sig_includes_itgae",
            "role": "gate",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Itgae"
            ]
          },
          {
            "name": "declared_a_known_confounder",
            "role": "gate",
            "field": "$.excluded_confounders[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "cf_signature_gut_contaminant",
            "role": "hard_fail",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "generic_signature_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd103_program_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "<"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "per_model_regulatory_state",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "poised_unfired"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "late_activation"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "closing_committed"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_tuples",
            "kind": "list",
            "matchKey": "regulator",
            "k": 1,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "role": {
                    "op": "equals",
                    "values": [
                      "repressor"
                    ]
                  },
                  "gating_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  },
                  "evidence_basis": {
                    "op": "equals",
                    "values": [
                      "target_program_behavior"
                    ]
                  },
                  "expr_vs_activity": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "locus_differential_gate",
            "role": "gate",
            "field": "locus_differential",
            "kind": "value",
            "expected": "uniform_across_models"
          },
          {
            "name": "formation_limited_by_gate",
            "role": "gate",
            "field": "formation_limited_by",
            "kind": "value",
            "expected": "absent_induction"
          },
          {
            "name": "locus_competent_in_naive_gate",
            "role": "gate",
            "field": "locus_competent_in_naive",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "ready_silent_fraction_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 3,
            "expected": [
              {
                "key": "B16_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_KPC_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "activator_chromatin_footprint_gate",
            "role": "gate",
            "field": "activator_chromatin_footprint",
            "kind": "value",
            "expected": "inconsistent_across_formers"
          },
          {
            "name": "exhaustion_comparable_gate",
            "role": "gate",
            "field": "exhaustion_comparable_across_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_nondeterminant_gate",
            "role": "gate",
            "field": "exhaustion_explains_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_separable_program_gate",
            "role": "gate",
            "field": "exhaustion_separable_program",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "exhaustion_counterexample_gate",
            "role": "gate",
            "field": "exhaustion_counterexample_model",
            "kind": "value",
            "expected": "KPC_GP"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "exhaustion_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "cf_inconsistent_present_without_activator",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "program",
            "entity": "present",
            "valueField": "activator_state",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_no_block_but_absent",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "none",
            "valueField": "program",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_late_block_without_poised_locus",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "late_activation",
            "valueField": "locus_state",
            "expected": [
              "closing_committed",
              "active"
            ]
          },
          {
            "name": "cf_inconsistent_early_block_with_brake_released",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "early_release",
            "valueField": "brake_state",
            "expected": [
              "released"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-5",
      "modelName": "claude-opus-5",
      "harness": "pi",
      "task": "green_klf2_hidden_repressor_of_trm_like_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 3916.38,
      "cost": 5.743631749999998,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "regulator_gate",
            "role": "gate",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "KLF2",
              "Klf2"
            ]
          },
          {
            "name": "controlling_node_character_gate",
            "role": "gate",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "high_differential_expression"
          },
          {
            "name": "expression_vs_activity_gate",
            "role": "gate",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "expression_exceeds_activity"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "target_program_partition",
            "kind": "list",
            "matchKey": "gene",
            "k": 7,
            "expected": [
              {
                "key": "S1pr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "S1pr4",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Klrg1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Cx3cr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Itgae",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Hic1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "P2rx7",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              }
            ]
          },
          {
            "name": "causal_status_gate",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "association_only_sufficiency_requires_perturbation"
          },
          {
            "name": "residency_control_logic_gate",
            "role": "gate",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_release_necessary_not_sufficient_multi_regulon"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "trm_like_program_by_model",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_absent"
                    ],
                    "canonical": "niche_absent"
                  }
                }
              }
            ]
          },
          {
            "name": "cf_regulator_activator_or_wrong_node",
            "role": "hard_fail",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "RUNX3",
              "Runx3",
              "PRDM1",
              "Prdm1",
              "BLIMP1",
              "Blimp1",
              "BHLHE40",
              "Bhlhe40",
              "ZNF683",
              "Zfp683",
              "HOBIT",
              "Hobit",
              "AHR",
              "Ahr",
              "BATF",
              "Batf",
              "TOX",
              "Tox",
              "NR4A1",
              "Nr4a1",
              "NR4A2",
              "Nr4a2",
              "NR4A3",
              "Nr4a3",
              "TCF7",
              "Tcf7",
              "TCF1",
              "Tcf1",
              "BACH2",
              "Bach2",
              "TBX21",
              "Tbx21",
              "TBET",
              "Tbet",
              "FLI1",
              "Fli1",
              "NR3C1",
              "Nr3c1",
              "EOMES",
              "Eomes",
              "ZEB2",
              "Zeb2",
              "ID2",
              "Id2",
              "ID3",
              "Id3",
              "IRF4",
              "Irf4"
            ]
          },
          {
            "name": "cf_node_character_hidden_driver_recall",
            "role": "hard_fail",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "hidden_driver_activity_not_expression"
          },
          {
            "name": "cf_effect_relation_hidden_pattern",
            "role": "hard_fail",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "activity_exceeds_expression"
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "causal_status",
            "kind": "value",
            "op": "in",
            "expected": [
              "sufficiency_established",
              "necessity_established",
              "causal_established_in_dataset",
              "sufficiency_via_perturbation_in_dataset"
            ]
          },
          {
            "name": "cf_residency_logic_activator_prior",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "residency_activator_induction_primary"
          },
          {
            "name": "cf_residency_logic_single_regulon",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_controls_entire_residency_program"
          },
          {
            "name": "cf_decoy_called_target",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_distractor_model_reported",
            "role": "hard_fail",
            "field": "$.trm_like_program_by_model[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_chromatin_only_decoy",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-5",
      "modelName": "claude-opus-5",
      "harness": "pi",
      "task": "green_klf2_hidden_repressor_of_trm_like_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 3505.59,
      "cost": 5.173100750000001,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "regulator_gate",
            "role": "gate",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "KLF2",
              "Klf2"
            ]
          },
          {
            "name": "controlling_node_character_gate",
            "role": "gate",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "high_differential_expression"
          },
          {
            "name": "expression_vs_activity_gate",
            "role": "gate",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "expression_exceeds_activity"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "target_program_partition",
            "kind": "list",
            "matchKey": "gene",
            "k": 7,
            "expected": [
              {
                "key": "S1pr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "S1pr4",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Klrg1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Cx3cr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Itgae",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Hic1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "P2rx7",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              }
            ]
          },
          {
            "name": "causal_status_gate",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "association_only_sufficiency_requires_perturbation"
          },
          {
            "name": "residency_control_logic_gate",
            "role": "gate",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_release_necessary_not_sufficient_multi_regulon"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "trm_like_program_by_model",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_absent"
                    ],
                    "canonical": "niche_absent"
                  }
                }
              }
            ]
          },
          {
            "name": "cf_regulator_activator_or_wrong_node",
            "role": "hard_fail",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "RUNX3",
              "Runx3",
              "PRDM1",
              "Prdm1",
              "BLIMP1",
              "Blimp1",
              "BHLHE40",
              "Bhlhe40",
              "ZNF683",
              "Zfp683",
              "HOBIT",
              "Hobit",
              "AHR",
              "Ahr",
              "BATF",
              "Batf",
              "TOX",
              "Tox",
              "NR4A1",
              "Nr4a1",
              "NR4A2",
              "Nr4a2",
              "NR4A3",
              "Nr4a3",
              "TCF7",
              "Tcf7",
              "TCF1",
              "Tcf1",
              "BACH2",
              "Bach2",
              "TBX21",
              "Tbx21",
              "TBET",
              "Tbet",
              "FLI1",
              "Fli1",
              "NR3C1",
              "Nr3c1",
              "EOMES",
              "Eomes",
              "ZEB2",
              "Zeb2",
              "ID2",
              "Id2",
              "ID3",
              "Id3",
              "IRF4",
              "Irf4"
            ]
          },
          {
            "name": "cf_node_character_hidden_driver_recall",
            "role": "hard_fail",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "hidden_driver_activity_not_expression"
          },
          {
            "name": "cf_effect_relation_hidden_pattern",
            "role": "hard_fail",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "activity_exceeds_expression"
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "causal_status",
            "kind": "value",
            "op": "in",
            "expected": [
              "sufficiency_established",
              "necessity_established",
              "causal_established_in_dataset",
              "sufficiency_via_perturbation_in_dataset"
            ]
          },
          {
            "name": "cf_residency_logic_activator_prior",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "residency_activator_induction_primary"
          },
          {
            "name": "cf_residency_logic_single_regulon",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_controls_entire_residency_program"
          },
          {
            "name": "cf_decoy_called_target",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_distractor_model_reported",
            "role": "hard_fail",
            "field": "$.trm_like_program_by_model[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_chromatin_only_decoy",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-5",
      "modelName": "claude-opus-5",
      "harness": "pi",
      "task": "green_klf2_hidden_repressor_of_trm_like_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 814.57,
      "cost": 0.7842710000000002,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "regulator_gate",
            "role": "gate",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "KLF2",
              "Klf2"
            ]
          },
          {
            "name": "controlling_node_character_gate",
            "role": "gate",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "high_differential_expression"
          },
          {
            "name": "expression_vs_activity_gate",
            "role": "gate",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "expression_exceeds_activity"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "target_program_partition",
            "kind": "list",
            "matchKey": "gene",
            "k": 7,
            "expected": [
              {
                "key": "S1pr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "S1pr4",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Klrg1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Cx3cr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Itgae",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Hic1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "P2rx7",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              }
            ]
          },
          {
            "name": "causal_status_gate",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "association_only_sufficiency_requires_perturbation"
          },
          {
            "name": "residency_control_logic_gate",
            "role": "gate",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_release_necessary_not_sufficient_multi_regulon"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "trm_like_program_by_model",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_absent"
                    ],
                    "canonical": "niche_absent"
                  }
                }
              }
            ]
          },
          {
            "name": "cf_regulator_activator_or_wrong_node",
            "role": "hard_fail",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "RUNX3",
              "Runx3",
              "PRDM1",
              "Prdm1",
              "BLIMP1",
              "Blimp1",
              "BHLHE40",
              "Bhlhe40",
              "ZNF683",
              "Zfp683",
              "HOBIT",
              "Hobit",
              "AHR",
              "Ahr",
              "BATF",
              "Batf",
              "TOX",
              "Tox",
              "NR4A1",
              "Nr4a1",
              "NR4A2",
              "Nr4a2",
              "NR4A3",
              "Nr4a3",
              "TCF7",
              "Tcf7",
              "TCF1",
              "Tcf1",
              "BACH2",
              "Bach2",
              "TBX21",
              "Tbx21",
              "TBET",
              "Tbet",
              "FLI1",
              "Fli1",
              "NR3C1",
              "Nr3c1",
              "EOMES",
              "Eomes",
              "ZEB2",
              "Zeb2",
              "ID2",
              "Id2",
              "ID3",
              "Id3",
              "IRF4",
              "Irf4"
            ]
          },
          {
            "name": "cf_node_character_hidden_driver_recall",
            "role": "hard_fail",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "hidden_driver_activity_not_expression"
          },
          {
            "name": "cf_effect_relation_hidden_pattern",
            "role": "hard_fail",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "activity_exceeds_expression"
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "causal_status",
            "kind": "value",
            "op": "in",
            "expected": [
              "sufficiency_established",
              "necessity_established",
              "causal_established_in_dataset",
              "sufficiency_via_perturbation_in_dataset"
            ]
          },
          {
            "name": "cf_residency_logic_activator_prior",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "residency_activator_induction_primary"
          },
          {
            "name": "cf_residency_logic_single_regulon",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_controls_entire_residency_program"
          },
          {
            "name": "cf_decoy_called_target",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_distractor_model_reported",
            "role": "hard_fail",
            "field": "$.trm_like_program_by_model[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_chromatin_only_decoy",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-5",
      "modelName": "claude-opus-5",
      "harness": "pi",
      "task": "green_shared_vs_distinct_regulators_trm_vs_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 1,
      "passed": true,
      "score": null,
      "durationS": 2008.43,
      "cost": 3.990092,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_program_map",
            "kind": "list",
            "matchKey": "regulator",
            "k": 3,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "both"
                    ],
                    "canonical": "both"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "represses"
                    ],
                    "canonical": "represses"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Batf",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "trm_like_til_specific"
                    ],
                    "canonical": "trm_like_til_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "chromatin_over_rna"
                    ],
                    "canonical": "chromatin_over_rna"
                  }
                }
              },
              {
                "key": "Ahr",
                "fields": {
                  "program_specificity": {
                    "op": "in",
                    "values": [
                      "trm_specific",
                      "both"
                    ],
                    "canonical": "trm_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "rna_over_chromatin"
                    ],
                    "canonical": "rna_over_chromatin"
                  }
                }
              }
            ]
          },
          {
            "name": "regulator_set_no_extras_gate",
            "role": "gate",
            "field": "$.regulator_program_map[*]",
            "kind": "other",
            "expected": null
          },
          {
            "name": "program_identity_conclusion_gate",
            "role": "gate",
            "field": "program_identity_conclusion",
            "kind": "value",
            "expected": "distinct_state_shared_brake_divergent_activators"
          },
          {
            "name": "residency_install_logic_gate",
            "role": "gate",
            "field": "residency_install_logic",
            "kind": "value",
            "expected": "brake_release_permits_context_matched_activator"
          },
          {
            "name": "activator_context_dependence_gate",
            "role": "gate",
            "field": "activator_context_dependence",
            "kind": "value",
            "expected": "context_matched_crossover"
          },
          {
            "name": "evidence_ceiling_gate",
            "role": "gate",
            "field": "evidence_ceiling",
            "kind": "value",
            "expected": "specificity_dissociations_require_perturbation"
          },
          {
            "name": "cf_install_logic_flat_or_single_step",
            "role": "hard_fail",
            "field": "residency_install_logic",
            "kind": "value",
            "op": "in",
            "expected": [
              "flat_shared_residency_toolkit",
              "activator_induction_sufficient",
              "brake_release_alone_sufficient"
            ]
          },
          {
            "name": "cf_program_identity_lumped_or_wrong",
            "role": "hard_fail",
            "field": "program_identity_conclusion",
            "kind": "value",
            "op": "in",
            "expected": [
              "trm_like_til_is_tumor_trm",
              "trm_like_til_is_exhausted_subset",
              "trm_like_til_is_effector_state",
              "trm_like_til_is_transitional_intermediate",
              "trm_like_til_is_shared_program_distinct_niche"
            ]
          },
          {
            "name": "cf_ahr_assigned_to_til",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Ahr",
            "valueField": "program_specificity",
            "expected": [
              "trm_like_til_specific"
            ]
          },
          {
            "name": "cf_batf_assigned_to_trm_or_shared",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Batf",
            "valueField": "program_specificity",
            "expected": [
              "trm_specific",
              "both"
            ]
          },
          {
            "name": "cf_klf2_called_activator",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Klf2",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          },
          {
            "name": "cf_evidence_ceiling_overclaim",
            "role": "hard_fail",
            "field": "evidence_ceiling",
            "kind": "value",
            "op": "in",
            "expected": [
              "specificity_established_from_regulon_activity",
              "specificity_established_in_dataset",
              "causation_established_by_cross_sectional_concordance",
              "necessity_and_sufficiency_established"
            ]
          },
          {
            "name": "cf_activator_context_shared_or_single",
            "role": "hard_fail",
            "field": "activator_context_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "shared_activators_both_contexts",
              "single_global_activator",
              "redundant_activators"
            ]
          },
          {
            "name": "cf_single_modality_rule",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_hic1_motif_brake_mislabel",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_runx3_anticanonical_as_controller",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Runx3",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-5",
      "modelName": "claude-opus-5",
      "harness": "pi",
      "task": "green_shared_vs_distinct_regulators_trm_vs_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 3346.35,
      "cost": 4.483111,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_program_map",
            "kind": "list",
            "matchKey": "regulator",
            "k": 3,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "both"
                    ],
                    "canonical": "both"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "represses"
                    ],
                    "canonical": "represses"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Batf",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "trm_like_til_specific"
                    ],
                    "canonical": "trm_like_til_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "chromatin_over_rna"
                    ],
                    "canonical": "chromatin_over_rna"
                  }
                }
              },
              {
                "key": "Ahr",
                "fields": {
                  "program_specificity": {
                    "op": "in",
                    "values": [
                      "trm_specific",
                      "both"
                    ],
                    "canonical": "trm_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "rna_over_chromatin"
                    ],
                    "canonical": "rna_over_chromatin"
                  }
                }
              }
            ]
          },
          {
            "name": "regulator_set_no_extras_gate",
            "role": "gate",
            "field": "$.regulator_program_map[*]",
            "kind": "other",
            "expected": null
          },
          {
            "name": "program_identity_conclusion_gate",
            "role": "gate",
            "field": "program_identity_conclusion",
            "kind": "value",
            "expected": "distinct_state_shared_brake_divergent_activators"
          },
          {
            "name": "residency_install_logic_gate",
            "role": "gate",
            "field": "residency_install_logic",
            "kind": "value",
            "expected": "brake_release_permits_context_matched_activator"
          },
          {
            "name": "activator_context_dependence_gate",
            "role": "gate",
            "field": "activator_context_dependence",
            "kind": "value",
            "expected": "context_matched_crossover"
          },
          {
            "name": "evidence_ceiling_gate",
            "role": "gate",
            "field": "evidence_ceiling",
            "kind": "value",
            "expected": "specificity_dissociations_require_perturbation"
          },
          {
            "name": "cf_install_logic_flat_or_single_step",
            "role": "hard_fail",
            "field": "residency_install_logic",
            "kind": "value",
            "op": "in",
            "expected": [
              "flat_shared_residency_toolkit",
              "activator_induction_sufficient",
              "brake_release_alone_sufficient"
            ]
          },
          {
            "name": "cf_program_identity_lumped_or_wrong",
            "role": "hard_fail",
            "field": "program_identity_conclusion",
            "kind": "value",
            "op": "in",
            "expected": [
              "trm_like_til_is_tumor_trm",
              "trm_like_til_is_exhausted_subset",
              "trm_like_til_is_effector_state",
              "trm_like_til_is_transitional_intermediate",
              "trm_like_til_is_shared_program_distinct_niche"
            ]
          },
          {
            "name": "cf_ahr_assigned_to_til",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Ahr",
            "valueField": "program_specificity",
            "expected": [
              "trm_like_til_specific"
            ]
          },
          {
            "name": "cf_batf_assigned_to_trm_or_shared",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Batf",
            "valueField": "program_specificity",
            "expected": [
              "trm_specific",
              "both"
            ]
          },
          {
            "name": "cf_klf2_called_activator",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Klf2",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          },
          {
            "name": "cf_evidence_ceiling_overclaim",
            "role": "hard_fail",
            "field": "evidence_ceiling",
            "kind": "value",
            "op": "in",
            "expected": [
              "specificity_established_from_regulon_activity",
              "specificity_established_in_dataset",
              "causation_established_by_cross_sectional_concordance",
              "necessity_and_sufficiency_established"
            ]
          },
          {
            "name": "cf_activator_context_shared_or_single",
            "role": "hard_fail",
            "field": "activator_context_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "shared_activators_both_contexts",
              "single_global_activator",
              "redundant_activators"
            ]
          },
          {
            "name": "cf_single_modality_rule",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_hic1_motif_brake_mislabel",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_runx3_anticanonical_as_controller",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Runx3",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-5",
      "modelName": "claude-opus-5",
      "harness": "pi",
      "task": "green_shared_vs_distinct_regulators_trm_vs_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1272.65,
      "cost": 2.274657,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_program_map",
            "kind": "list",
            "matchKey": "regulator",
            "k": 3,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "both"
                    ],
                    "canonical": "both"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "represses"
                    ],
                    "canonical": "represses"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Batf",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "trm_like_til_specific"
                    ],
                    "canonical": "trm_like_til_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "chromatin_over_rna"
                    ],
                    "canonical": "chromatin_over_rna"
                  }
                }
              },
              {
                "key": "Ahr",
                "fields": {
                  "program_specificity": {
                    "op": "in",
                    "values": [
                      "trm_specific",
                      "both"
                    ],
                    "canonical": "trm_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "rna_over_chromatin"
                    ],
                    "canonical": "rna_over_chromatin"
                  }
                }
              }
            ]
          },
          {
            "name": "regulator_set_no_extras_gate",
            "role": "gate",
            "field": "$.regulator_program_map[*]",
            "kind": "other",
            "expected": null
          },
          {
            "name": "program_identity_conclusion_gate",
            "role": "gate",
            "field": "program_identity_conclusion",
            "kind": "value",
            "expected": "distinct_state_shared_brake_divergent_activators"
          },
          {
            "name": "residency_install_logic_gate",
            "role": "gate",
            "field": "residency_install_logic",
            "kind": "value",
            "expected": "brake_release_permits_context_matched_activator"
          },
          {
            "name": "activator_context_dependence_gate",
            "role": "gate",
            "field": "activator_context_dependence",
            "kind": "value",
            "expected": "context_matched_crossover"
          },
          {
            "name": "evidence_ceiling_gate",
            "role": "gate",
            "field": "evidence_ceiling",
            "kind": "value",
            "expected": "specificity_dissociations_require_perturbation"
          },
          {
            "name": "cf_install_logic_flat_or_single_step",
            "role": "hard_fail",
            "field": "residency_install_logic",
            "kind": "value",
            "op": "in",
            "expected": [
              "flat_shared_residency_toolkit",
              "activator_induction_sufficient",
              "brake_release_alone_sufficient"
            ]
          },
          {
            "name": "cf_program_identity_lumped_or_wrong",
            "role": "hard_fail",
            "field": "program_identity_conclusion",
            "kind": "value",
            "op": "in",
            "expected": [
              "trm_like_til_is_tumor_trm",
              "trm_like_til_is_exhausted_subset",
              "trm_like_til_is_effector_state",
              "trm_like_til_is_transitional_intermediate",
              "trm_like_til_is_shared_program_distinct_niche"
            ]
          },
          {
            "name": "cf_ahr_assigned_to_til",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Ahr",
            "valueField": "program_specificity",
            "expected": [
              "trm_like_til_specific"
            ]
          },
          {
            "name": "cf_batf_assigned_to_trm_or_shared",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Batf",
            "valueField": "program_specificity",
            "expected": [
              "trm_specific",
              "both"
            ]
          },
          {
            "name": "cf_klf2_called_activator",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Klf2",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          },
          {
            "name": "cf_evidence_ceiling_overclaim",
            "role": "hard_fail",
            "field": "evidence_ceiling",
            "kind": "value",
            "op": "in",
            "expected": [
              "specificity_established_from_regulon_activity",
              "specificity_established_in_dataset",
              "causation_established_by_cross_sectional_concordance",
              "necessity_and_sufficiency_established"
            ]
          },
          {
            "name": "cf_activator_context_shared_or_single",
            "role": "hard_fail",
            "field": "activator_context_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "shared_activators_both_contexts",
              "single_global_activator",
              "redundant_activators"
            ]
          },
          {
            "name": "cf_single_modality_rule",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_hic1_motif_brake_mislabel",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_runx3_anticanonical_as_controller",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Runx3",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-5",
      "modelName": "claude-opus-5",
      "harness": "pi",
      "task": "gse297023_q2_senescence_not_mechanism",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 1,
      "passed": true,
      "score": null,
      "durationS": 1671.68,
      "cost": 1.7116395000000002,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "candidate_is_senescence",
            "role": "gate",
            "field": "$.candidate_mechanism",
            "kind": "value",
            "op": "in",
            "expected": [
              "cellular_senescence",
              "oncogene_induced_senescence",
              "replicative_senescence"
            ]
          },
          {
            "name": "arrest_core_not_engaged",
            "role": "gate",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_not_engaged"
            ]
          },
          {
            "name": "secretory_arm_is_inflammaging",
            "role": "gate",
            "field": "$.secretory_arm_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "acute_phase_inflammaging_response"
            ]
          },
          {
            "name": "secretory_arm_broadly_elevated",
            "role": "gate",
            "field": "$.secretory_arm_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "broadly_elevated"
            ]
          },
          {
            "name": "carries_aging_program",
            "role": "gate",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "carries_bona_fide_aging_program_distinct_from_nominated"
            ]
          },
          {
            "name": "conclusion_senescence_unlikely",
            "role": "gate",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "unlikely_major_contributor"
            ]
          },
          {
            "name": "cf_arrest_core_engaged_recall",
            "role": "hard_fail",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_engaged"
            ]
          },
          {
            "name": "cf_aging_program_conflation_or_explain_away",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "aging_program_indistinguishable_from_nominated_program",
              "aging_program_is_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_no_aging_program_sensitivity_failure",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_detectable_aging_program"
            ]
          },
          {
            "name": "cf_senescence_driver_conclusion",
            "role": "hard_fail",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "likely_primary_driver",
              "plausible_supported_contributor"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-5",
      "modelName": "claude-opus-5",
      "harness": "pi",
      "task": "gse297023_q2_senescence_not_mechanism",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 2102.58,
      "cost": 2.40163925,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "candidate_is_senescence",
            "role": "gate",
            "field": "$.candidate_mechanism",
            "kind": "value",
            "op": "in",
            "expected": [
              "cellular_senescence",
              "oncogene_induced_senescence",
              "replicative_senescence"
            ]
          },
          {
            "name": "arrest_core_not_engaged",
            "role": "gate",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_not_engaged"
            ]
          },
          {
            "name": "secretory_arm_is_inflammaging",
            "role": "gate",
            "field": "$.secretory_arm_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "acute_phase_inflammaging_response"
            ]
          },
          {
            "name": "secretory_arm_broadly_elevated",
            "role": "gate",
            "field": "$.secretory_arm_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "broadly_elevated"
            ]
          },
          {
            "name": "carries_aging_program",
            "role": "gate",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "carries_bona_fide_aging_program_distinct_from_nominated"
            ]
          },
          {
            "name": "conclusion_senescence_unlikely",
            "role": "gate",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "unlikely_major_contributor"
            ]
          },
          {
            "name": "cf_arrest_core_engaged_recall",
            "role": "hard_fail",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_engaged"
            ]
          },
          {
            "name": "cf_aging_program_conflation_or_explain_away",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "aging_program_indistinguishable_from_nominated_program",
              "aging_program_is_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_no_aging_program_sensitivity_failure",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_detectable_aging_program"
            ]
          },
          {
            "name": "cf_senescence_driver_conclusion",
            "role": "hard_fail",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "likely_primary_driver",
              "plausible_supported_contributor"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-5",
      "modelName": "claude-opus-5",
      "harness": "pi",
      "task": "gse297023_q2_senescence_not_mechanism",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1171.16,
      "cost": 2.0979537500000003,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "candidate_is_senescence",
            "role": "gate",
            "field": "$.candidate_mechanism",
            "kind": "value",
            "op": "in",
            "expected": [
              "cellular_senescence",
              "oncogene_induced_senescence",
              "replicative_senescence"
            ]
          },
          {
            "name": "arrest_core_not_engaged",
            "role": "gate",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_not_engaged"
            ]
          },
          {
            "name": "secretory_arm_is_inflammaging",
            "role": "gate",
            "field": "$.secretory_arm_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "acute_phase_inflammaging_response"
            ]
          },
          {
            "name": "secretory_arm_broadly_elevated",
            "role": "gate",
            "field": "$.secretory_arm_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "broadly_elevated"
            ]
          },
          {
            "name": "carries_aging_program",
            "role": "gate",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "carries_bona_fide_aging_program_distinct_from_nominated"
            ]
          },
          {
            "name": "conclusion_senescence_unlikely",
            "role": "gate",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "unlikely_major_contributor"
            ]
          },
          {
            "name": "cf_arrest_core_engaged_recall",
            "role": "hard_fail",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_engaged"
            ]
          },
          {
            "name": "cf_aging_program_conflation_or_explain_away",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "aging_program_indistinguishable_from_nominated_program",
              "aging_program_is_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_no_aging_program_sensitivity_failure",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_detectable_aging_program"
            ]
          },
          {
            "name": "cf_senescence_driver_conclusion",
            "role": "hard_fail",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "likely_primary_driver",
              "plausible_supported_contributor"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-5",
      "modelName": "claude-opus-5",
      "harness": "pi",
      "task": "gse297023_q3_age_retained_mapk_reduced",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 1,
      "passed": true,
      "score": null,
      "durationS": 5283.59,
      "cost": 3.678446,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "age_retained",
            "role": "gate",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_retained"
            ]
          },
          {
            "name": "identity_is_aging",
            "role": "gate",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_aging_signatures"
            ]
          },
          {
            "name": "nature_state_shift",
            "role": "gate",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_state_expression_shift"
            ]
          },
          {
            "name": "pathway_is_mapk",
            "role": "gate",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "mapk"
            ]
          },
          {
            "name": "pathway_reduced_in_aged",
            "role": "gate",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "reduced_in_aged"
            ]
          },
          {
            "name": "cf_mapk_increased_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_in_aged"
            ]
          },
          {
            "name": "cf_age_erased",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_erased"
            ]
          },
          {
            "name": "cf_pervasive_overcall",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_pervasive_artifactual"
            ]
          },
          {
            "name": "cf_explain_away_aging",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_compositional_or_identity",
            "role": "hard_fail",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_proportion_shift",
              "cell_identity_shift",
              "composition_dominant_state_secondary",
              "both_state_and_composition_equally"
            ]
          },
          {
            "name": "cf_biggest_inflammatory_signal_not_mapk",
            "role": "hard_fail",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "jak_stat",
              "nfkb",
              "tnfa"
            ]
          },
          {
            "name": "cf_senescence_misattribution",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_senescence_signature_specifically"
            ]
          },
          {
            "name": "cf_dismiss_as_batch_artifact",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_batch_or_technical_artifact"
            ]
          },
          {
            "name": "cf_mapk_comparable_constitutive_kras_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "comparable_across_ages"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-5",
      "modelName": "claude-opus-5",
      "harness": "pi",
      "task": "gse297023_q3_age_retained_mapk_reduced",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 2,
      "passed": true,
      "score": null,
      "durationS": 3030.49,
      "cost": 3.199489,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "age_retained",
            "role": "gate",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_retained"
            ]
          },
          {
            "name": "identity_is_aging",
            "role": "gate",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_aging_signatures"
            ]
          },
          {
            "name": "nature_state_shift",
            "role": "gate",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_state_expression_shift"
            ]
          },
          {
            "name": "pathway_is_mapk",
            "role": "gate",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "mapk"
            ]
          },
          {
            "name": "pathway_reduced_in_aged",
            "role": "gate",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "reduced_in_aged"
            ]
          },
          {
            "name": "cf_mapk_increased_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_in_aged"
            ]
          },
          {
            "name": "cf_age_erased",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_erased"
            ]
          },
          {
            "name": "cf_pervasive_overcall",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_pervasive_artifactual"
            ]
          },
          {
            "name": "cf_explain_away_aging",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_compositional_or_identity",
            "role": "hard_fail",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_proportion_shift",
              "cell_identity_shift",
              "composition_dominant_state_secondary",
              "both_state_and_composition_equally"
            ]
          },
          {
            "name": "cf_biggest_inflammatory_signal_not_mapk",
            "role": "hard_fail",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "jak_stat",
              "nfkb",
              "tnfa"
            ]
          },
          {
            "name": "cf_senescence_misattribution",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_senescence_signature_specifically"
            ]
          },
          {
            "name": "cf_dismiss_as_batch_artifact",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_batch_or_technical_artifact"
            ]
          },
          {
            "name": "cf_mapk_comparable_constitutive_kras_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "comparable_across_ages"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-5",
      "modelName": "claude-opus-5",
      "harness": "pi",
      "task": "gse297023_q3_age_retained_mapk_reduced",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 4084.82,
      "cost": 2.486812749999999,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "age_retained",
            "role": "gate",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_retained"
            ]
          },
          {
            "name": "identity_is_aging",
            "role": "gate",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_aging_signatures"
            ]
          },
          {
            "name": "nature_state_shift",
            "role": "gate",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_state_expression_shift"
            ]
          },
          {
            "name": "pathway_is_mapk",
            "role": "gate",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "mapk"
            ]
          },
          {
            "name": "pathway_reduced_in_aged",
            "role": "gate",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "reduced_in_aged"
            ]
          },
          {
            "name": "cf_mapk_increased_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_in_aged"
            ]
          },
          {
            "name": "cf_age_erased",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_erased"
            ]
          },
          {
            "name": "cf_pervasive_overcall",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_pervasive_artifactual"
            ]
          },
          {
            "name": "cf_explain_away_aging",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_compositional_or_identity",
            "role": "hard_fail",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_proportion_shift",
              "cell_identity_shift",
              "composition_dominant_state_secondary",
              "both_state_and_composition_equally"
            ]
          },
          {
            "name": "cf_biggest_inflammatory_signal_not_mapk",
            "role": "hard_fail",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "jak_stat",
              "nfkb",
              "tnfa"
            ]
          },
          {
            "name": "cf_senescence_misattribution",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_senescence_signature_specifically"
            ]
          },
          {
            "name": "cf_dismiss_as_batch_artifact",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_batch_or_technical_artifact"
            ]
          },
          {
            "name": "cf_mapk_comparable_constitutive_kras_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "comparable_across_ages"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-5",
      "modelName": "claude-opus-5",
      "harness": "pi",
      "task": "gse297023_q5_tme_age_dependent_remodeling",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 11945.93,
      "cost": 6.837086750000001,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "perturbation_is_pten",
            "role": "gate",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "pten"
            ]
          },
          {
            "name": "tme_non_cell_autonomous",
            "role": "gate",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "substantial_non_cell_autonomous_remodeling"
            ]
          },
          {
            "name": "tme_state_not_composition",
            "role": "gate",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "tme_age_dependent",
            "role": "gate",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_dependent"
            ]
          },
          {
            "name": "tme_response_broader_in_aged",
            "role": "gate",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_aged"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "most_responsive_noncancer_cell_types",
            "kind": "list",
            "matchKey": "cell_type",
            "k": 2,
            "expected": [
              {
                "key": "alveolar_macrophage",
                "fields": {}
              },
              {
                "key": "endothelial",
                "fields": {}
              }
            ]
          },
          {
            "name": "cf_cell_intrinsic_only",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "negligible_cell_intrinsic_only_effect",
              "remodeling_unresolved",
              "modest_remodeling_few_cell_types"
            ]
          },
          {
            "name": "cf_remodeling_is_contamination",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "apparent_remodeling_reflects_cancer_contamination"
            ]
          },
          {
            "name": "cf_perturbation_design_misread",
            "role": "hard_fail",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "multiple_tumor_suppressors_pooled",
              "no_additional_gene_difference_kras_only",
              "rb1_cdkn2a_double",
              "pcna_essential_gene_control",
              "difference_is_oncogene_not_suppressor"
            ]
          },
          {
            "name": "cf_age_independent_tme",
            "role": "hard_fail",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_independent"
            ]
          },
          {
            "name": "cf_compositional_shift",
            "role": "hard_fail",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_frequency_shift",
              "compositional_immune_infiltration",
              "composition_dominant_state_secondary",
              "both_state_and_composition"
            ]
          },
          {
            "name": "cf_epithelial_or_cancer_substitution",
            "role": "hard_fail",
            "field": "$.most_responsive_noncancer_cell_types[*].cell_type",
            "kind": "avoid",
            "expected": [
              "AT2_cell",
              "AT1_cell",
              "cancer_cell",
              "ciliated",
              "secretory",
              "club_cell",
              "basal_cell",
              "goblet_cell"
            ]
          },
          {
            "name": "cf_tme_breadth_broader_in_young",
            "role": "hard_fail",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_young"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-5",
      "modelName": "claude-opus-5",
      "harness": "pi",
      "task": "gse297023_q5_tme_age_dependent_remodeling",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 10632.11,
      "cost": 6.3003205000000015,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "perturbation_is_pten",
            "role": "gate",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "pten"
            ]
          },
          {
            "name": "tme_non_cell_autonomous",
            "role": "gate",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "substantial_non_cell_autonomous_remodeling"
            ]
          },
          {
            "name": "tme_state_not_composition",
            "role": "gate",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "tme_age_dependent",
            "role": "gate",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_dependent"
            ]
          },
          {
            "name": "tme_response_broader_in_aged",
            "role": "gate",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_aged"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "most_responsive_noncancer_cell_types",
            "kind": "list",
            "matchKey": "cell_type",
            "k": 2,
            "expected": [
              {
                "key": "alveolar_macrophage",
                "fields": {}
              },
              {
                "key": "endothelial",
                "fields": {}
              }
            ]
          },
          {
            "name": "cf_cell_intrinsic_only",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "negligible_cell_intrinsic_only_effect",
              "remodeling_unresolved",
              "modest_remodeling_few_cell_types"
            ]
          },
          {
            "name": "cf_remodeling_is_contamination",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "apparent_remodeling_reflects_cancer_contamination"
            ]
          },
          {
            "name": "cf_perturbation_design_misread",
            "role": "hard_fail",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "multiple_tumor_suppressors_pooled",
              "no_additional_gene_difference_kras_only",
              "rb1_cdkn2a_double",
              "pcna_essential_gene_control",
              "difference_is_oncogene_not_suppressor"
            ]
          },
          {
            "name": "cf_age_independent_tme",
            "role": "hard_fail",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_independent"
            ]
          },
          {
            "name": "cf_compositional_shift",
            "role": "hard_fail",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_frequency_shift",
              "compositional_immune_infiltration",
              "composition_dominant_state_secondary",
              "both_state_and_composition"
            ]
          },
          {
            "name": "cf_epithelial_or_cancer_substitution",
            "role": "hard_fail",
            "field": "$.most_responsive_noncancer_cell_types[*].cell_type",
            "kind": "avoid",
            "expected": [
              "AT2_cell",
              "AT1_cell",
              "cancer_cell",
              "ciliated",
              "secretory",
              "club_cell",
              "basal_cell",
              "goblet_cell"
            ]
          },
          {
            "name": "cf_tme_breadth_broader_in_young",
            "role": "hard_fail",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_young"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-5",
      "modelName": "claude-opus-5",
      "harness": "pi",
      "task": "gse297023_q5_tme_age_dependent_remodeling",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 11813.52,
      "cost": 8.365137000000002,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "perturbation_is_pten",
            "role": "gate",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "pten"
            ]
          },
          {
            "name": "tme_non_cell_autonomous",
            "role": "gate",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "substantial_non_cell_autonomous_remodeling"
            ]
          },
          {
            "name": "tme_state_not_composition",
            "role": "gate",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "tme_age_dependent",
            "role": "gate",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_dependent"
            ]
          },
          {
            "name": "tme_response_broader_in_aged",
            "role": "gate",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_aged"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "most_responsive_noncancer_cell_types",
            "kind": "list",
            "matchKey": "cell_type",
            "k": 2,
            "expected": [
              {
                "key": "alveolar_macrophage",
                "fields": {}
              },
              {
                "key": "endothelial",
                "fields": {}
              }
            ]
          },
          {
            "name": "cf_cell_intrinsic_only",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "negligible_cell_intrinsic_only_effect",
              "remodeling_unresolved",
              "modest_remodeling_few_cell_types"
            ]
          },
          {
            "name": "cf_remodeling_is_contamination",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "apparent_remodeling_reflects_cancer_contamination"
            ]
          },
          {
            "name": "cf_perturbation_design_misread",
            "role": "hard_fail",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "multiple_tumor_suppressors_pooled",
              "no_additional_gene_difference_kras_only",
              "rb1_cdkn2a_double",
              "pcna_essential_gene_control",
              "difference_is_oncogene_not_suppressor"
            ]
          },
          {
            "name": "cf_age_independent_tme",
            "role": "hard_fail",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_independent"
            ]
          },
          {
            "name": "cf_compositional_shift",
            "role": "hard_fail",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_frequency_shift",
              "compositional_immune_infiltration",
              "composition_dominant_state_secondary",
              "both_state_and_composition"
            ]
          },
          {
            "name": "cf_epithelial_or_cancer_substitution",
            "role": "hard_fail",
            "field": "$.most_responsive_noncancer_cell_types[*].cell_type",
            "kind": "avoid",
            "expected": [
              "AT2_cell",
              "AT1_cell",
              "cancer_cell",
              "ciliated",
              "secretory",
              "club_cell",
              "basal_cell",
              "goblet_cell"
            ]
          },
          {
            "name": "cf_tme_breadth_broader_in_young",
            "role": "hard_fail",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_young"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-5",
      "modelName": "claude-opus-5",
      "harness": "pi",
      "task": "gse297023_q6_age_immune_infiltration",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 1,
      "passed": true,
      "score": null,
      "durationS": 2834.1,
      "cost": 1.9459547500000005,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "composition_stable_with_age",
            "role": "gate",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "frequencies_largely_stable"
            ]
          },
          {
            "name": "age_signal_is_state_not_composition",
            "role": "gate",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "frequencies_uninformative_for_infiltration",
            "role": "gate",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_cannot_establish_infiltration"
            ]
          },
          {
            "name": "infiltration_increases_with_age",
            "role": "gate",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_with_age"
            ]
          },
          {
            "name": "infiltration_strength_calibrated",
            "role": "gate",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "significant_increase_strongest_markers_only",
              "consistent_increase_not_statistically_robust"
            ]
          },
          {
            "name": "genotype_marker_fraction_lower_in_pten",
            "role": "gate",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "lower_in_pten_deficient"
            ]
          },
          {
            "name": "cf_composition_shift_reading",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "major_compositional_shift_with_age",
              "increased_immune_fraction_with_age"
            ]
          },
          {
            "name": "cf_lineage_specific_composition_claim",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "myeloid_expansion_with_age",
              "lymphoid_expansion_with_age",
              "minor_shifts_in_some_cell_types"
            ]
          },
          {
            "name": "cf_immunosenescence_depletion_prior",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "immune_depletion_with_age"
            ]
          },
          {
            "name": "cf_age_modality_compositional",
            "role": "hard_fail",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_type_composition_shift",
              "both_equally",
              "composition_dominant_state_secondary"
            ]
          },
          {
            "name": "cf_overread_frequencies_as_infiltration",
            "role": "hard_fail",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_shows_no_infiltration_change",
              "single_cell_quantifies_infiltration",
              "frequencies_show_increased_infiltration",
              "frequencies_show_decreased_infiltration"
            ]
          },
          {
            "name": "cf_infiltration_decreases_immunosenescence",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "decreased_with_age"
            ]
          },
          {
            "name": "cf_infiltration_unchanged_scrna_overread",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "unchanged_with_age"
            ]
          },
          {
            "name": "cf_infiltration_indeterminate_unused_node",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "direction_indeterminate_from_data"
            ]
          },
          {
            "name": "cf_infiltration_mixed_direction",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "mixed_by_immune_cell_type"
            ]
          },
          {
            "name": "cf_infiltration_genotype_confusion",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_only_in_pten_deficient",
              "decreased_only_in_pten_deficient"
            ]
          },
          {
            "name": "cf_pten_infiltration_paper_text_recall",
            "role": "hard_fail",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "higher_in_pten_deficient",
              "higher_in_pten_deficient_only_in_young",
              "higher_in_pten_deficient_only_in_aged"
            ]
          },
          {
            "name": "cf_pseudoreplication_overclaim",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_increase_across_markers"
            ]
          },
          {
            "name": "cf_infiltration_strength_decrease_direction",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_decrease_across_markers",
              "significant_decrease_strongest_markers_only",
              "consistent_decrease_not_statistically_robust"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-5",
      "modelName": "claude-opus-5",
      "harness": "pi",
      "task": "gse297023_q6_age_immune_infiltration",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 2,
      "passed": true,
      "score": null,
      "durationS": 3725.25,
      "cost": 2.6243535,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "composition_stable_with_age",
            "role": "gate",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "frequencies_largely_stable"
            ]
          },
          {
            "name": "age_signal_is_state_not_composition",
            "role": "gate",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "frequencies_uninformative_for_infiltration",
            "role": "gate",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_cannot_establish_infiltration"
            ]
          },
          {
            "name": "infiltration_increases_with_age",
            "role": "gate",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_with_age"
            ]
          },
          {
            "name": "infiltration_strength_calibrated",
            "role": "gate",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "significant_increase_strongest_markers_only",
              "consistent_increase_not_statistically_robust"
            ]
          },
          {
            "name": "genotype_marker_fraction_lower_in_pten",
            "role": "gate",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "lower_in_pten_deficient"
            ]
          },
          {
            "name": "cf_composition_shift_reading",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "major_compositional_shift_with_age",
              "increased_immune_fraction_with_age"
            ]
          },
          {
            "name": "cf_lineage_specific_composition_claim",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "myeloid_expansion_with_age",
              "lymphoid_expansion_with_age",
              "minor_shifts_in_some_cell_types"
            ]
          },
          {
            "name": "cf_immunosenescence_depletion_prior",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "immune_depletion_with_age"
            ]
          },
          {
            "name": "cf_age_modality_compositional",
            "role": "hard_fail",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_type_composition_shift",
              "both_equally",
              "composition_dominant_state_secondary"
            ]
          },
          {
            "name": "cf_overread_frequencies_as_infiltration",
            "role": "hard_fail",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_shows_no_infiltration_change",
              "single_cell_quantifies_infiltration",
              "frequencies_show_increased_infiltration",
              "frequencies_show_decreased_infiltration"
            ]
          },
          {
            "name": "cf_infiltration_decreases_immunosenescence",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "decreased_with_age"
            ]
          },
          {
            "name": "cf_infiltration_unchanged_scrna_overread",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "unchanged_with_age"
            ]
          },
          {
            "name": "cf_infiltration_indeterminate_unused_node",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "direction_indeterminate_from_data"
            ]
          },
          {
            "name": "cf_infiltration_mixed_direction",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "mixed_by_immune_cell_type"
            ]
          },
          {
            "name": "cf_infiltration_genotype_confusion",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_only_in_pten_deficient",
              "decreased_only_in_pten_deficient"
            ]
          },
          {
            "name": "cf_pten_infiltration_paper_text_recall",
            "role": "hard_fail",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "higher_in_pten_deficient",
              "higher_in_pten_deficient_only_in_young",
              "higher_in_pten_deficient_only_in_aged"
            ]
          },
          {
            "name": "cf_pseudoreplication_overclaim",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_increase_across_markers"
            ]
          },
          {
            "name": "cf_infiltration_strength_decrease_direction",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_decrease_across_markers",
              "significant_decrease_strongest_markers_only",
              "consistent_decrease_not_statistically_robust"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-opus-5",
      "modelName": "claude-opus-5",
      "harness": "pi",
      "task": "gse297023_q6_age_immune_infiltration",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 3735,
      "cost": 2.3956419999999996,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "composition_stable_with_age",
            "role": "gate",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "frequencies_largely_stable"
            ]
          },
          {
            "name": "age_signal_is_state_not_composition",
            "role": "gate",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "frequencies_uninformative_for_infiltration",
            "role": "gate",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_cannot_establish_infiltration"
            ]
          },
          {
            "name": "infiltration_increases_with_age",
            "role": "gate",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_with_age"
            ]
          },
          {
            "name": "infiltration_strength_calibrated",
            "role": "gate",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "significant_increase_strongest_markers_only",
              "consistent_increase_not_statistically_robust"
            ]
          },
          {
            "name": "genotype_marker_fraction_lower_in_pten",
            "role": "gate",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "lower_in_pten_deficient"
            ]
          },
          {
            "name": "cf_composition_shift_reading",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "major_compositional_shift_with_age",
              "increased_immune_fraction_with_age"
            ]
          },
          {
            "name": "cf_lineage_specific_composition_claim",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "myeloid_expansion_with_age",
              "lymphoid_expansion_with_age",
              "minor_shifts_in_some_cell_types"
            ]
          },
          {
            "name": "cf_immunosenescence_depletion_prior",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "immune_depletion_with_age"
            ]
          },
          {
            "name": "cf_age_modality_compositional",
            "role": "hard_fail",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_type_composition_shift",
              "both_equally",
              "composition_dominant_state_secondary"
            ]
          },
          {
            "name": "cf_overread_frequencies_as_infiltration",
            "role": "hard_fail",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_shows_no_infiltration_change",
              "single_cell_quantifies_infiltration",
              "frequencies_show_increased_infiltration",
              "frequencies_show_decreased_infiltration"
            ]
          },
          {
            "name": "cf_infiltration_decreases_immunosenescence",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "decreased_with_age"
            ]
          },
          {
            "name": "cf_infiltration_unchanged_scrna_overread",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "unchanged_with_age"
            ]
          },
          {
            "name": "cf_infiltration_indeterminate_unused_node",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "direction_indeterminate_from_data"
            ]
          },
          {
            "name": "cf_infiltration_mixed_direction",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "mixed_by_immune_cell_type"
            ]
          },
          {
            "name": "cf_infiltration_genotype_confusion",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_only_in_pten_deficient",
              "decreased_only_in_pten_deficient"
            ]
          },
          {
            "name": "cf_pten_infiltration_paper_text_recall",
            "role": "hard_fail",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "higher_in_pten_deficient",
              "higher_in_pten_deficient_only_in_young",
              "higher_in_pten_deficient_only_in_aged"
            ]
          },
          {
            "name": "cf_pseudoreplication_overclaim",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_increase_across_markers"
            ]
          },
          {
            "name": "cf_infiltration_strength_decrease_direction",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_decrease_across_markers",
              "significant_decrease_strongest_markers_only",
              "consistent_decrease_not_statistically_robust"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-kimi-k3-pi",
      "modelName": "kimi-k3",
      "harness": "pi",
      "task": "gse155381_q4_pseudotime_delay",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 2747,
      "cost": 2.1966461999999995,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "pseudotime_direction_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "E17_chronological_match_weighted",
            "role": "additive",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pluripotency_exit_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pseudotime_direction_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_synchronous_with_monkey_EPI",
              "chimera_human_EPI_developmentally_advanced",
              "chimera_human_EPI_delayed_at_pre_implantation_only_resolved_post_implantation"
            ]
          },
          {
            "name": "E17_chronological_match_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "resembles_monkey_E17_Gast_EPI",
              "resembles_chimera_monkey_E17_EPI_specifically",
              "chimera_human_E17_EPI_more_advanced_than_monkey_E17_Gast"
            ]
          },
          {
            "name": "pluripotency_exit_paper_inverted_or_prior_driven",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_advances_pluripotency_exit_prematurely",
              "naive_pluripotency_program_specifically_engaged",
              "pluripotency_core_retained_via_KLF17_naive_marker_specifically"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-kimi-k3-pi",
      "modelName": "kimi-k3",
      "harness": "pi",
      "task": "gse155381_q4_pseudotime_delay",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 2144,
      "cost": 1.7041763999999997,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "pseudotime_direction_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "E17_chronological_match_weighted",
            "role": "additive",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pluripotency_exit_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pseudotime_direction_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_synchronous_with_monkey_EPI",
              "chimera_human_EPI_developmentally_advanced",
              "chimera_human_EPI_delayed_at_pre_implantation_only_resolved_post_implantation"
            ]
          },
          {
            "name": "E17_chronological_match_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "resembles_monkey_E17_Gast_EPI",
              "resembles_chimera_monkey_E17_EPI_specifically",
              "chimera_human_E17_EPI_more_advanced_than_monkey_E17_Gast"
            ]
          },
          {
            "name": "pluripotency_exit_paper_inverted_or_prior_driven",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_advances_pluripotency_exit_prematurely",
              "naive_pluripotency_program_specifically_engaged",
              "pluripotency_core_retained_via_KLF17_naive_marker_specifically"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-kimi-k3-pi",
      "modelName": "kimi-k3",
      "harness": "pi",
      "task": "gse155381_q4_pseudotime_delay",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 2833,
      "cost": 2.2369908,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "pseudotime_direction_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "E17_chronological_match_weighted",
            "role": "additive",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pluripotency_exit_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pseudotime_direction_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_synchronous_with_monkey_EPI",
              "chimera_human_EPI_developmentally_advanced",
              "chimera_human_EPI_delayed_at_pre_implantation_only_resolved_post_implantation"
            ]
          },
          {
            "name": "E17_chronological_match_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "resembles_monkey_E17_Gast_EPI",
              "resembles_chimera_monkey_E17_EPI_specifically",
              "chimera_human_E17_EPI_more_advanced_than_monkey_E17_Gast"
            ]
          },
          {
            "name": "pluripotency_exit_paper_inverted_or_prior_driven",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_advances_pluripotency_exit_prematurely",
              "naive_pluripotency_program_specifically_engaged",
              "pluripotency_core_retained_via_KLF17_naive_marker_specifically"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-kimi-k3-pi",
      "modelName": "kimi-k3",
      "harness": "pi",
      "task": "gse155381_q5_chimera_monkey_perturbation",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 1458,
      "cost": 0.8967215999999999,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "deg_direction_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "comparison_subset_choice_weighted",
            "role": "additive",
            "field": "$.comparison_subset_choice",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "panel_resolved_strong_target_ccn1_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN1",
              "CYR61"
            ]
          },
          {
            "name": "panel_resolved_strong_target_ccn2_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN2",
              "CTGF"
            ]
          },
          {
            "name": "hippo_pathway_weighted",
            "role": "additive",
            "field": "$.hippo_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "tgf_beta_pathway_weighted",
            "role": "additive",
            "field": "$.tgf_beta_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "deg_direction_batch_effect_uncorrected",
            "role": "hard_fail",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_dominated_global",
              "up_dominated_in_chosen_subset"
            ]
          },
          {
            "name": "silent_panel_dropout_paired_with_directional_pathway_claim",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "hippo_pathway_direction": {
                "op": "in",
                "values": [
                  "hippo_pathway_upregulated",
                  "hippo_pathway_unchanged",
                  "wnt_pathway_specifically_perturbed",
                  "stress_response_pathway_dominantly_engaged",
                  "no_pathway_signal_above_confound",
                  "yap_tead_direct_target_panel_downregulated",
                  "yap_tead_panel_dependent_direction",
                  "hippo_pathway_downregulated"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-kimi-k3-pi",
      "modelName": "kimi-k3",
      "harness": "pi",
      "task": "gse155381_q5_chimera_monkey_perturbation",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 1902,
      "cost": 1.4567747999999998,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "deg_direction_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "comparison_subset_choice_weighted",
            "role": "additive",
            "field": "$.comparison_subset_choice",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "panel_resolved_strong_target_ccn1_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN1",
              "CYR61"
            ]
          },
          {
            "name": "panel_resolved_strong_target_ccn2_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN2",
              "CTGF"
            ]
          },
          {
            "name": "hippo_pathway_weighted",
            "role": "additive",
            "field": "$.hippo_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "tgf_beta_pathway_weighted",
            "role": "additive",
            "field": "$.tgf_beta_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "deg_direction_batch_effect_uncorrected",
            "role": "hard_fail",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_dominated_global",
              "up_dominated_in_chosen_subset"
            ]
          },
          {
            "name": "silent_panel_dropout_paired_with_directional_pathway_claim",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "hippo_pathway_direction": {
                "op": "in",
                "values": [
                  "hippo_pathway_upregulated",
                  "hippo_pathway_unchanged",
                  "wnt_pathway_specifically_perturbed",
                  "stress_response_pathway_dominantly_engaged",
                  "no_pathway_signal_above_confound",
                  "yap_tead_direct_target_panel_downregulated",
                  "yap_tead_panel_dependent_direction",
                  "hippo_pathway_downregulated"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-kimi-k3-pi",
      "modelName": "kimi-k3",
      "harness": "pi",
      "task": "gse155381_q5_chimera_monkey_perturbation",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1756,
      "cost": 1.1653578000000002,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "deg_direction_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "comparison_subset_choice_weighted",
            "role": "additive",
            "field": "$.comparison_subset_choice",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "panel_resolved_strong_target_ccn1_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN1",
              "CYR61"
            ]
          },
          {
            "name": "panel_resolved_strong_target_ccn2_or_alias",
            "role": "additive",
            "field": "$.yap_target_panel_resolved_symbols",
            "kind": "avoid",
            "expected": [
              "CCN2",
              "CTGF"
            ]
          },
          {
            "name": "hippo_pathway_weighted",
            "role": "additive",
            "field": "$.hippo_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "tgf_beta_pathway_weighted",
            "role": "additive",
            "field": "$.tgf_beta_pathway_direction",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "deg_direction_batch_effect_uncorrected",
            "role": "hard_fail",
            "field": "$.chimera_monkey_DEG_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_dominated_global",
              "up_dominated_in_chosen_subset"
            ]
          },
          {
            "name": "silent_panel_dropout_paired_with_directional_pathway_claim",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "hippo_pathway_direction": {
                "op": "in",
                "values": [
                  "hippo_pathway_upregulated",
                  "hippo_pathway_unchanged",
                  "wnt_pathway_specifically_perturbed",
                  "stress_response_pathway_dominantly_engaged",
                  "no_pathway_signal_above_confound",
                  "yap_tead_direct_target_panel_downregulated",
                  "yap_tead_panel_dependent_direction",
                  "hippo_pathway_downregulated"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-kimi-k3-pi",
      "modelName": "kimi-k3",
      "harness": "pi",
      "task": "gse155381_q7_xspecies_epi_communication",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 1448,
      "cost": 0.9646968,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "richness_weighted",
            "role": "additive",
            "field": "$.xspecies_signaling_richness",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_weighted",
            "role": "additive",
            "field": "$.dominant_signaling_axis",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_wnt_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "WNT_FZD_LRP_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_fgf_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "FGF_FGFR_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_notch_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "NOTCH_DLL_JAG_family"
                ]
              }
            }
          },
          {
            "name": "signaling_directionality_weighted",
            "role": "additive",
            "field": "$.signaling_directionality",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "top_signal_interaction_class_weighted",
            "role": "additive",
            "field": "$.top_signal_interaction_class",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          }
        ]
      }
    },
    {
      "modelId": "sclong-kimi-k3-pi",
      "modelName": "kimi-k3",
      "harness": "pi",
      "task": "gse155381_q7_xspecies_epi_communication",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 1463,
      "cost": 1.303416,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "richness_weighted",
            "role": "additive",
            "field": "$.xspecies_signaling_richness",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_weighted",
            "role": "additive",
            "field": "$.dominant_signaling_axis",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_wnt_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "WNT_FZD_LRP_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_fgf_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "FGF_FGFR_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_notch_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "NOTCH_DLL_JAG_family"
                ]
              }
            }
          },
          {
            "name": "signaling_directionality_weighted",
            "role": "additive",
            "field": "$.signaling_directionality",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "top_signal_interaction_class_weighted",
            "role": "additive",
            "field": "$.top_signal_interaction_class",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          }
        ]
      }
    },
    {
      "modelId": "sclong-kimi-k3-pi",
      "modelName": "kimi-k3",
      "harness": "pi",
      "task": "gse155381_q7_xspecies_epi_communication",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1618,
      "cost": 1.2633012,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "richness_weighted",
            "role": "additive",
            "field": "$.xspecies_signaling_richness",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_weighted",
            "role": "additive",
            "field": "$.dominant_signaling_axis",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_wnt_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "WNT_FZD_LRP_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_fgf_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "FGF_FGFR_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_notch_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "NOTCH_DLL_JAG_family"
                ]
              }
            }
          },
          {
            "name": "signaling_directionality_weighted",
            "role": "additive",
            "field": "$.signaling_directionality",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "top_signal_interaction_class_weighted",
            "role": "additive",
            "field": "$.top_signal_interaction_class",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          }
        ]
      }
    },
    {
      "modelId": "sclong-kimi-k3-pi",
      "modelName": "kimi-k3",
      "harness": "pi",
      "task": "gse155381_q8_bidirectional_convergence",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 4136,
      "cost": 4.411250400000001,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "chimera_human_identity_shift_target_weighted",
            "role": "additive",
            "field": "$.chimera_human_identity_shift_target",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_in_vivo_state_assignment_weighted",
            "role": "additive",
            "field": "$.chimera_human_in_vivo_state_assignment",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_shift_magnitude_weighted",
            "role": "additive",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_monkey_identity_shift_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "both_arms_null_contradicted_by_calibration",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "chimera_human_identity_shift_target": {
                "op": "equals",
                "values": [
                  "chimera_human_maintains_origin_identity_no_measurable_shift"
                ]
              },
              "chimera_monkey_identity_shift": {
                "op": "equals",
                "values": [
                  "chimera_monkey_unchanged_relative_to_control"
                ]
              }
            }
          },
          {
            "name": "chimera_monkey_paper_claim_on_confounded_input",
            "role": "hard_fail",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "value",
            "expected": "chimera_monkey_shifts_toward_chimera_human_identity"
          },
          {
            "name": "chimera_human_magnitude_wrong_attribution_signature",
            "role": "hard_fail",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "value",
            "op": "in",
            "expected": [
              "dramatic_complete_replacement_of_origin_identity",
              "magnitude_dominated_by_culture_or_metabolic_axis",
              "magnitude_dominated_by_developmental_stage_axis",
              "magnitude_dominated_by_cell_cycle_axis",
              "magnitude_dominated_by_batch_effect_axis"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-kimi-k3-pi",
      "modelName": "kimi-k3",
      "harness": "pi",
      "task": "gse155381_q8_bidirectional_convergence",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 4261,
      "cost": 4.391225400000001,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "chimera_human_identity_shift_target_weighted",
            "role": "additive",
            "field": "$.chimera_human_identity_shift_target",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_in_vivo_state_assignment_weighted",
            "role": "additive",
            "field": "$.chimera_human_in_vivo_state_assignment",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_shift_magnitude_weighted",
            "role": "additive",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_monkey_identity_shift_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "both_arms_null_contradicted_by_calibration",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "chimera_human_identity_shift_target": {
                "op": "equals",
                "values": [
                  "chimera_human_maintains_origin_identity_no_measurable_shift"
                ]
              },
              "chimera_monkey_identity_shift": {
                "op": "equals",
                "values": [
                  "chimera_monkey_unchanged_relative_to_control"
                ]
              }
            }
          },
          {
            "name": "chimera_monkey_paper_claim_on_confounded_input",
            "role": "hard_fail",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "value",
            "expected": "chimera_monkey_shifts_toward_chimera_human_identity"
          },
          {
            "name": "chimera_human_magnitude_wrong_attribution_signature",
            "role": "hard_fail",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "value",
            "op": "in",
            "expected": [
              "dramatic_complete_replacement_of_origin_identity",
              "magnitude_dominated_by_culture_or_metabolic_axis",
              "magnitude_dominated_by_developmental_stage_axis",
              "magnitude_dominated_by_cell_cycle_axis",
              "magnitude_dominated_by_batch_effect_axis"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-kimi-k3-pi",
      "modelName": "kimi-k3",
      "harness": "pi",
      "task": "gse155381_q8_bidirectional_convergence",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 3608,
      "cost": 4.3423061999999994,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "chimera_human_identity_shift_target_weighted",
            "role": "additive",
            "field": "$.chimera_human_identity_shift_target",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_in_vivo_state_assignment_weighted",
            "role": "additive",
            "field": "$.chimera_human_in_vivo_state_assignment",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_human_shift_magnitude_weighted",
            "role": "additive",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "chimera_monkey_identity_shift_weighted",
            "role": "additive",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "both_arms_null_contradicted_by_calibration",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "chimera_human_identity_shift_target": {
                "op": "equals",
                "values": [
                  "chimera_human_maintains_origin_identity_no_measurable_shift"
                ]
              },
              "chimera_monkey_identity_shift": {
                "op": "equals",
                "values": [
                  "chimera_monkey_unchanged_relative_to_control"
                ]
              }
            }
          },
          {
            "name": "chimera_monkey_paper_claim_on_confounded_input",
            "role": "hard_fail",
            "field": "$.chimera_monkey_identity_shift",
            "kind": "value",
            "expected": "chimera_monkey_shifts_toward_chimera_human_identity"
          },
          {
            "name": "chimera_human_magnitude_wrong_attribution_signature",
            "role": "hard_fail",
            "field": "$.chimera_human_shift_magnitude",
            "kind": "value",
            "op": "in",
            "expected": [
              "dramatic_complete_replacement_of_origin_identity",
              "magnitude_dominated_by_culture_or_metabolic_axis",
              "magnitude_dominated_by_developmental_stage_axis",
              "magnitude_dominated_by_cell_cycle_axis",
              "magnitude_dominated_by_batch_effect_axis"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-kimi-k3-pi",
      "modelName": "kimi-k3",
      "harness": "pi",
      "task": "gse171524_ectopic_airway_rare_cell",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 2088,
      "cost": 1.5938214,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "focus_population_identity_gate",
            "role": "gate",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "chemosensory_marker_expressing_population_alpha"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "focus_population_marker_calls",
            "kind": "list",
            "matchKey": "marker",
            "k": 3,
            "expected": [
              {
                "key": "POU2F3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "CHAT",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "TRPM5",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "LRMP",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNAT3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "AVIL",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNG13",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              }
            ]
          },
          {
            "name": "focus_population_condition_distribution_gate",
            "role": "gate",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_enriched_with_substantial_control_presence",
              "covid_predominant",
              "covid_only",
              "covid_enriched_trend_not_significant"
            ]
          },
          {
            "name": "focus_population_location_gate",
            "role": "gate",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "ectopic_parenchyma_in_covid",
              "parenchyma_only",
              "bronchiolized_or_peribronchial_parenchyma",
              "airway_and_parenchyma"
            ]
          },
          {
            "name": "focus_population_recovery_mode_gate",
            "role": "gate",
            "field": "$.focus_population_recovery_mode",
            "kind": "value",
            "op": "in",
            "expected": [
              "continuum_not_discrete_cluster"
            ]
          },
          {
            "name": "focus_population_identity_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "ionocyte_like_beta",
              "neuroendocrine_like_gamma",
              "basal_stem_progenitor_like_delta",
              "aberrant_basaloid_transitional_like_epsilon",
              "neuronal_or_neural_gene_minor_population_zeta",
              "deuterosomal_ciliated_progenitor_like_eta",
              "hillock_transitional_like_theta",
              "low_quality_doublet_or_debris",
              "no_rare_cell_recovered",
              "not_assessed"
            ]
          },
          {
            "name": "cf_wrong_lineage_marker_called_positive",
            "role": "hard_fail",
            "field": "$.focus_population_marker_calls[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "focus_population_condition_distribution_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "equally_distributed",
              "control_predominant",
              "not_assessed"
            ]
          },
          {
            "name": "focus_population_location_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "airway_only",
              "not_assessed"
            ]
          },
          {
            "name": "cross_axis_canonical_identity_with_airway_only_location_incoherence",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "focus_population_identity": {
                "op": "in",
                "values": [
                  "chemosensory_marker_expressing_population_alpha"
                ]
              },
              "focus_population_location": {
                "op": "in",
                "values": [
                  "airway_only"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-kimi-k3-pi",
      "modelName": "kimi-k3",
      "harness": "pi",
      "task": "gse171524_ectopic_airway_rare_cell",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 2588,
      "cost": 0.9055644,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "focus_population_identity_gate",
            "role": "gate",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "chemosensory_marker_expressing_population_alpha"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "focus_population_marker_calls",
            "kind": "list",
            "matchKey": "marker",
            "k": 3,
            "expected": [
              {
                "key": "POU2F3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "CHAT",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "TRPM5",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "LRMP",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNAT3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "AVIL",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNG13",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              }
            ]
          },
          {
            "name": "focus_population_condition_distribution_gate",
            "role": "gate",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_enriched_with_substantial_control_presence",
              "covid_predominant",
              "covid_only",
              "covid_enriched_trend_not_significant"
            ]
          },
          {
            "name": "focus_population_location_gate",
            "role": "gate",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "ectopic_parenchyma_in_covid",
              "parenchyma_only",
              "bronchiolized_or_peribronchial_parenchyma",
              "airway_and_parenchyma"
            ]
          },
          {
            "name": "focus_population_recovery_mode_gate",
            "role": "gate",
            "field": "$.focus_population_recovery_mode",
            "kind": "value",
            "op": "in",
            "expected": [
              "continuum_not_discrete_cluster"
            ]
          },
          {
            "name": "focus_population_identity_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "ionocyte_like_beta",
              "neuroendocrine_like_gamma",
              "basal_stem_progenitor_like_delta",
              "aberrant_basaloid_transitional_like_epsilon",
              "neuronal_or_neural_gene_minor_population_zeta",
              "deuterosomal_ciliated_progenitor_like_eta",
              "hillock_transitional_like_theta",
              "low_quality_doublet_or_debris",
              "no_rare_cell_recovered",
              "not_assessed"
            ]
          },
          {
            "name": "cf_wrong_lineage_marker_called_positive",
            "role": "hard_fail",
            "field": "$.focus_population_marker_calls[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "focus_population_condition_distribution_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "equally_distributed",
              "control_predominant",
              "not_assessed"
            ]
          },
          {
            "name": "focus_population_location_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "airway_only",
              "not_assessed"
            ]
          },
          {
            "name": "cross_axis_canonical_identity_with_airway_only_location_incoherence",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "focus_population_identity": {
                "op": "in",
                "values": [
                  "chemosensory_marker_expressing_population_alpha"
                ]
              },
              "focus_population_location": {
                "op": "in",
                "values": [
                  "airway_only"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-kimi-k3-pi",
      "modelName": "kimi-k3",
      "harness": "pi",
      "task": "gse171524_ectopic_airway_rare_cell",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 2690,
      "cost": 0.8205042,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "focus_population_identity_gate",
            "role": "gate",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "chemosensory_marker_expressing_population_alpha"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "focus_population_marker_calls",
            "kind": "list",
            "matchKey": "marker",
            "k": 3,
            "expected": [
              {
                "key": "POU2F3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "CHAT",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "TRPM5",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "LRMP",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNAT3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "AVIL",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNG13",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              }
            ]
          },
          {
            "name": "focus_population_condition_distribution_gate",
            "role": "gate",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_enriched_with_substantial_control_presence",
              "covid_predominant",
              "covid_only",
              "covid_enriched_trend_not_significant"
            ]
          },
          {
            "name": "focus_population_location_gate",
            "role": "gate",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "ectopic_parenchyma_in_covid",
              "parenchyma_only",
              "bronchiolized_or_peribronchial_parenchyma",
              "airway_and_parenchyma"
            ]
          },
          {
            "name": "focus_population_recovery_mode_gate",
            "role": "gate",
            "field": "$.focus_population_recovery_mode",
            "kind": "value",
            "op": "in",
            "expected": [
              "continuum_not_discrete_cluster"
            ]
          },
          {
            "name": "focus_population_identity_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "ionocyte_like_beta",
              "neuroendocrine_like_gamma",
              "basal_stem_progenitor_like_delta",
              "aberrant_basaloid_transitional_like_epsilon",
              "neuronal_or_neural_gene_minor_population_zeta",
              "deuterosomal_ciliated_progenitor_like_eta",
              "hillock_transitional_like_theta",
              "low_quality_doublet_or_debris",
              "no_rare_cell_recovered",
              "not_assessed"
            ]
          },
          {
            "name": "cf_wrong_lineage_marker_called_positive",
            "role": "hard_fail",
            "field": "$.focus_population_marker_calls[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "focus_population_condition_distribution_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "equally_distributed",
              "control_predominant",
              "not_assessed"
            ]
          },
          {
            "name": "focus_population_location_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "airway_only",
              "not_assessed"
            ]
          },
          {
            "name": "cross_axis_canonical_identity_with_airway_only_location_incoherence",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "focus_population_identity": {
                "op": "in",
                "values": [
                  "chemosensory_marker_expressing_population_alpha"
                ]
              },
              "focus_population_location": {
                "op": "in",
                "values": [
                  "airway_only"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-kimi-k3-pi",
      "modelName": "kimi-k3",
      "harness": "pi",
      "task": "gse171524_pathological_fibroblast_master_regulator_druggable_targets",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 1,
      "passed": true,
      "score": null,
      "durationS": 11500,
      "cost": 4.7148144000000025,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "disease_associated_fibroblast_subtype_weighted",
            "role": "additive",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "mr_tf_junb_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_alpha"
            ]
          },
          {
            "name": "mr_tf_jund_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_beta"
            ]
          },
          {
            "name": "drug_mmp14_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_alpha"
            ]
          },
          {
            "name": "drug_stat3_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_beta"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_weighted",
            "role": "additive",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "disease_associated_fibroblast_subtype_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "value",
            "op": "in",
            "expected": [
              "subtype_lambda",
              "subtype_epsilon",
              "subtype_iota",
              "subtype_beta",
              "subtype_eta",
              "subtype_zeta",
              "subtype_not_assessed"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "simple_de_ranking_by_log_fc",
              "marker_based_de_only",
              "not_computed"
            ]
          },
          {
            "name": "top_master_regulator_transcription_factors_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_not_computed"
            ]
          },
          {
            "name": "top_druggable_proteins_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_not_assessed"
            ]
          },
          {
            "name": "cross_axis_circular_marker_in_druggable_proteins",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_gamma",
              "drug_target_delta",
              "drug_target_eta"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-kimi-k3-pi",
      "modelName": "kimi-k3",
      "harness": "pi",
      "task": "gse171524_pathological_fibroblast_master_regulator_druggable_targets",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 3984,
      "cost": 1.5327684,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "disease_associated_fibroblast_subtype_weighted",
            "role": "additive",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "mr_tf_junb_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_alpha"
            ]
          },
          {
            "name": "mr_tf_jund_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_beta"
            ]
          },
          {
            "name": "drug_mmp14_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_alpha"
            ]
          },
          {
            "name": "drug_stat3_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_beta"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_weighted",
            "role": "additive",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "disease_associated_fibroblast_subtype_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "value",
            "op": "in",
            "expected": [
              "subtype_lambda",
              "subtype_epsilon",
              "subtype_iota",
              "subtype_beta",
              "subtype_eta",
              "subtype_zeta",
              "subtype_not_assessed"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "simple_de_ranking_by_log_fc",
              "marker_based_de_only",
              "not_computed"
            ]
          },
          {
            "name": "top_master_regulator_transcription_factors_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_not_computed"
            ]
          },
          {
            "name": "top_druggable_proteins_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_not_assessed"
            ]
          },
          {
            "name": "cross_axis_circular_marker_in_druggable_proteins",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_gamma",
              "drug_target_delta",
              "drug_target_eta"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-kimi-k3-pi",
      "modelName": "kimi-k3",
      "harness": "pi",
      "task": "gse171524_pathological_fibroblast_master_regulator_druggable_targets",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 4859,
      "cost": 2.5952969999999995,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "disease_associated_fibroblast_subtype_weighted",
            "role": "additive",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "mr_tf_junb_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_alpha"
            ]
          },
          {
            "name": "mr_tf_jund_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_beta"
            ]
          },
          {
            "name": "drug_mmp14_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_alpha"
            ]
          },
          {
            "name": "drug_stat3_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_beta"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_weighted",
            "role": "additive",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "disease_associated_fibroblast_subtype_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "value",
            "op": "in",
            "expected": [
              "subtype_lambda",
              "subtype_epsilon",
              "subtype_iota",
              "subtype_beta",
              "subtype_eta",
              "subtype_zeta",
              "subtype_not_assessed"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "simple_de_ranking_by_log_fc",
              "marker_based_de_only",
              "not_computed"
            ]
          },
          {
            "name": "top_master_regulator_transcription_factors_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_not_computed"
            ]
          },
          {
            "name": "top_druggable_proteins_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_not_assessed"
            ]
          },
          {
            "name": "cross_axis_circular_marker_in_druggable_proteins",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_gamma",
              "drug_target_delta",
              "drug_target_eta"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-kimi-k3-pi",
      "modelName": "kimi-k3",
      "harness": "pi",
      "task": "gse171524_plasma_bcr_neutralizing_clonotype",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 1206,
      "cost": 0.9396426,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dominant_pairing_heavy_gate",
            "role": "gate",
            "field": "$.dominant_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV4-34"
            ]
          },
          {
            "name": "dominant_pairing_light_gate",
            "role": "gate",
            "field": "$.dominant_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGLV3-19"
            ]
          },
          {
            "name": "dominant_neutralizer_match_gate",
            "role": "gate",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_known_neutralizer_match"
            ]
          },
          {
            "name": "neutralizer_pairing_heavy_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV1-18"
            ]
          },
          {
            "name": "neutralizer_pairing_light_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGKV3-20"
            ]
          },
          {
            "name": "neutralizer_recurrence_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "observed_in_2_to_5_donors"
            ]
          },
          {
            "name": "neutralizer_rank_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "top_5_pairing",
              "top_6_to_20_pairing"
            ]
          },
          {
            "name": "dominant_clone_isotype_gate",
            "role": "gate",
            "field": "$.dominant_clone_isotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "IgG"
            ]
          },
          {
            "name": "blineage_class_switch_contrast_gate",
            "role": "gate",
            "field": "$.blineage_class_switch_contrast",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_class_switched_control_higher_unswitched",
              "class_switched_in_both_cohorts"
            ]
          },
          {
            "name": "dominant_match_narrative_trap_or_null",
            "role": "hard_fail",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_rbd_neutralizer",
              "matches_known_autoreactive_clone",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_heavy",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_light",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_recurrence_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_rank_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "cross_axis_found_dominant_but_failed_to_scan_for_embedded_neutralizer",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_pairing_neutralizer_match": {
                "op": "in",
                "values": [
                  "no_known_neutralizer_match"
                ]
              },
              "neutralizer_pairing_heavy": {
                "op": "in",
                "values": [
                  "none_detected"
                ]
              }
            }
          },
          {
            "name": "neutralizer_pairing_not_exactly_the_s309_pair",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "op": "not",
            "expected": {
              "op": "and",
              "args": [
                {
                  "op": "field",
                  "name": "neutralizer_pairing_heavy",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGHV1-18"
                    ]
                  }
                },
                {
                  "op": "field",
                  "name": "neutralizer_pairing_light",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGKV3-20"
                    ]
                  }
                }
              ]
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-kimi-k3-pi",
      "modelName": "kimi-k3",
      "harness": "pi",
      "task": "gse171524_plasma_bcr_neutralizing_clonotype",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 849,
      "cost": 0.6324924,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dominant_pairing_heavy_gate",
            "role": "gate",
            "field": "$.dominant_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV4-34"
            ]
          },
          {
            "name": "dominant_pairing_light_gate",
            "role": "gate",
            "field": "$.dominant_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGLV3-19"
            ]
          },
          {
            "name": "dominant_neutralizer_match_gate",
            "role": "gate",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_known_neutralizer_match"
            ]
          },
          {
            "name": "neutralizer_pairing_heavy_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV1-18"
            ]
          },
          {
            "name": "neutralizer_pairing_light_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGKV3-20"
            ]
          },
          {
            "name": "neutralizer_recurrence_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "observed_in_2_to_5_donors"
            ]
          },
          {
            "name": "neutralizer_rank_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "top_5_pairing",
              "top_6_to_20_pairing"
            ]
          },
          {
            "name": "dominant_clone_isotype_gate",
            "role": "gate",
            "field": "$.dominant_clone_isotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "IgG"
            ]
          },
          {
            "name": "blineage_class_switch_contrast_gate",
            "role": "gate",
            "field": "$.blineage_class_switch_contrast",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_class_switched_control_higher_unswitched",
              "class_switched_in_both_cohorts"
            ]
          },
          {
            "name": "dominant_match_narrative_trap_or_null",
            "role": "hard_fail",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_rbd_neutralizer",
              "matches_known_autoreactive_clone",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_heavy",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_light",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_recurrence_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_rank_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "cross_axis_found_dominant_but_failed_to_scan_for_embedded_neutralizer",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_pairing_neutralizer_match": {
                "op": "in",
                "values": [
                  "no_known_neutralizer_match"
                ]
              },
              "neutralizer_pairing_heavy": {
                "op": "in",
                "values": [
                  "none_detected"
                ]
              }
            }
          },
          {
            "name": "neutralizer_pairing_not_exactly_the_s309_pair",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "op": "not",
            "expected": {
              "op": "and",
              "args": [
                {
                  "op": "field",
                  "name": "neutralizer_pairing_heavy",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGHV1-18"
                    ]
                  }
                },
                {
                  "op": "field",
                  "name": "neutralizer_pairing_light",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGKV3-20"
                    ]
                  }
                }
              ]
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-kimi-k3-pi",
      "modelName": "kimi-k3",
      "harness": "pi",
      "task": "gse171524_plasma_bcr_neutralizing_clonotype",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 778,
      "cost": 0.6055398000000002,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dominant_pairing_heavy_gate",
            "role": "gate",
            "field": "$.dominant_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV4-34"
            ]
          },
          {
            "name": "dominant_pairing_light_gate",
            "role": "gate",
            "field": "$.dominant_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGLV3-19"
            ]
          },
          {
            "name": "dominant_neutralizer_match_gate",
            "role": "gate",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_known_neutralizer_match"
            ]
          },
          {
            "name": "neutralizer_pairing_heavy_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGHV1-18"
            ]
          },
          {
            "name": "neutralizer_pairing_light_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "IGKV3-20"
            ]
          },
          {
            "name": "neutralizer_recurrence_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "observed_in_2_to_5_donors"
            ]
          },
          {
            "name": "neutralizer_rank_gate",
            "role": "gate",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "top_5_pairing",
              "top_6_to_20_pairing"
            ]
          },
          {
            "name": "dominant_clone_isotype_gate",
            "role": "gate",
            "field": "$.dominant_clone_isotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "IgG"
            ]
          },
          {
            "name": "blineage_class_switch_contrast_gate",
            "role": "gate",
            "field": "$.blineage_class_switch_contrast",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_class_switched_control_higher_unswitched",
              "class_switched_in_both_cohorts"
            ]
          },
          {
            "name": "dominant_match_narrative_trap_or_null",
            "role": "hard_fail",
            "field": "$.dominant_pairing_neutralizer_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_rbd_neutralizer",
              "matches_known_autoreactive_clone",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_heavy",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_heavy",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_pairing_missed_light",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_light",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected",
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_recurrence_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_donor_recurrence",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "neutralizer_rank_not_resolved",
            "role": "hard_fail",
            "field": "$.neutralizer_pairing_rank",
            "kind": "value",
            "op": "in",
            "expected": [
              "not_resolved"
            ]
          },
          {
            "name": "cross_axis_found_dominant_but_failed_to_scan_for_embedded_neutralizer",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_pairing_neutralizer_match": {
                "op": "in",
                "values": [
                  "no_known_neutralizer_match"
                ]
              },
              "neutralizer_pairing_heavy": {
                "op": "in",
                "values": [
                  "none_detected"
                ]
              }
            }
          },
          {
            "name": "neutralizer_pairing_not_exactly_the_s309_pair",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "op": "not",
            "expected": {
              "op": "and",
              "args": [
                {
                  "op": "field",
                  "name": "neutralizer_pairing_heavy",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGHV1-18"
                    ]
                  }
                },
                {
                  "op": "field",
                  "name": "neutralizer_pairing_light",
                  "body": {
                    "op": "in",
                    "args": [
                      "IGKV3-20"
                    ]
                  }
                }
              ]
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-kimi-k3-pi",
      "modelName": "kimi-k3",
      "harness": "pi",
      "task": "q6_myeloid_subset_restricted_de",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 1,
      "passed": true,
      "score": null,
      "durationS": 3834,
      "cost": 1.3501614000000002,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "myeloid_subset_with_strongest_change_gate",
            "role": "gate",
            "field": "$.myeloid_subset_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "subset_gamma"
            ]
          },
          {
            "name": "gene_with_strongest_change_gate",
            "role": "gate",
            "field": "$.gene_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "AXL",
              "PROS1"
            ]
          },
          {
            "name": "direction_of_change_gate",
            "role": "gate",
            "field": "$.direction_of_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "down_in_covid"
            ]
          },
          {
            "name": "biological_function_interpretation_gate",
            "role": "gate",
            "field": "$.biological_function_interpretation",
            "kind": "value",
            "op": "in",
            "expected": [
              "efferocytosis",
              "phagocytosis"
            ]
          },
          {
            "name": "myeloid_compartment_abundance_gate",
            "role": "gate",
            "field": "$.myeloid_compartment_abundance",
            "kind": "value",
            "op": "in",
            "expected": [
              "expands_in_covid"
            ]
          },
          {
            "name": "myeloid_expansion_localization_gate",
            "role": "gate",
            "field": "$.myeloid_expansion_localization",
            "kind": "value",
            "op": "in",
            "expected": [
              "recruited_driven_resident_dysfunction"
            ]
          },
          {
            "name": "dominant_signature_gate",
            "role": "gate",
            "field": "$.dominant_signature",
            "kind": "value",
            "op": "in",
            "expected": [
              "technical_or_agonal_artifact"
            ]
          },
          {
            "name": "dominant_signature_direction_gate",
            "role": "gate",
            "field": "$.dominant_signature_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_in_covid"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-kimi-k3-pi",
      "modelName": "kimi-k3",
      "harness": "pi",
      "task": "q6_myeloid_subset_restricted_de",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 2869,
      "cost": 1.4706690000000002,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "myeloid_subset_with_strongest_change_gate",
            "role": "gate",
            "field": "$.myeloid_subset_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "subset_gamma"
            ]
          },
          {
            "name": "gene_with_strongest_change_gate",
            "role": "gate",
            "field": "$.gene_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "AXL",
              "PROS1"
            ]
          },
          {
            "name": "direction_of_change_gate",
            "role": "gate",
            "field": "$.direction_of_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "down_in_covid"
            ]
          },
          {
            "name": "biological_function_interpretation_gate",
            "role": "gate",
            "field": "$.biological_function_interpretation",
            "kind": "value",
            "op": "in",
            "expected": [
              "efferocytosis",
              "phagocytosis"
            ]
          },
          {
            "name": "myeloid_compartment_abundance_gate",
            "role": "gate",
            "field": "$.myeloid_compartment_abundance",
            "kind": "value",
            "op": "in",
            "expected": [
              "expands_in_covid"
            ]
          },
          {
            "name": "myeloid_expansion_localization_gate",
            "role": "gate",
            "field": "$.myeloid_expansion_localization",
            "kind": "value",
            "op": "in",
            "expected": [
              "recruited_driven_resident_dysfunction"
            ]
          },
          {
            "name": "dominant_signature_gate",
            "role": "gate",
            "field": "$.dominant_signature",
            "kind": "value",
            "op": "in",
            "expected": [
              "technical_or_agonal_artifact"
            ]
          },
          {
            "name": "dominant_signature_direction_gate",
            "role": "gate",
            "field": "$.dominant_signature_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_in_covid"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-kimi-k3-pi",
      "modelName": "kimi-k3",
      "harness": "pi",
      "task": "q6_myeloid_subset_restricted_de",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 3446,
      "cost": 1.5953255999999996,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "myeloid_subset_with_strongest_change_gate",
            "role": "gate",
            "field": "$.myeloid_subset_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "subset_gamma"
            ]
          },
          {
            "name": "gene_with_strongest_change_gate",
            "role": "gate",
            "field": "$.gene_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "AXL",
              "PROS1"
            ]
          },
          {
            "name": "direction_of_change_gate",
            "role": "gate",
            "field": "$.direction_of_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "down_in_covid"
            ]
          },
          {
            "name": "biological_function_interpretation_gate",
            "role": "gate",
            "field": "$.biological_function_interpretation",
            "kind": "value",
            "op": "in",
            "expected": [
              "efferocytosis",
              "phagocytosis"
            ]
          },
          {
            "name": "myeloid_compartment_abundance_gate",
            "role": "gate",
            "field": "$.myeloid_compartment_abundance",
            "kind": "value",
            "op": "in",
            "expected": [
              "expands_in_covid"
            ]
          },
          {
            "name": "myeloid_expansion_localization_gate",
            "role": "gate",
            "field": "$.myeloid_expansion_localization",
            "kind": "value",
            "op": "in",
            "expected": [
              "recruited_driven_resident_dysfunction"
            ]
          },
          {
            "name": "dominant_signature_gate",
            "role": "gate",
            "field": "$.dominant_signature",
            "kind": "value",
            "op": "in",
            "expected": [
              "technical_or_agonal_artifact"
            ]
          },
          {
            "name": "dominant_signature_direction_gate",
            "role": "gate",
            "field": "$.dominant_signature_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_in_covid"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-kimi-k3-pi",
      "modelName": "kimi-k3",
      "harness": "pi",
      "task": "snrnaseq_per_donor_composition_shifts_lethal_covid_lung",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 3284,
      "cost": 0.5651892000000001,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dict_match",
            "role": "gate",
            "field": "cell_type_direction_map",
            "kind": "dict",
            "expected": {
              "Myeloid": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "Epithelia": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Fibroblasts": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "T_NK": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "B_plasma": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Endothelial": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Mast": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "APC": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Neuronal": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              }
            }
          },
          {
            "name": "over_merge_immune_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Immune",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_leukocyte_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Leukocyte",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_lymphoid_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Lymphoid",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_stromal_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Stromal",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-kimi-k3-pi",
      "modelName": "kimi-k3",
      "harness": "pi",
      "task": "snrnaseq_per_donor_composition_shifts_lethal_covid_lung",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 2051,
      "cost": 0.4537607999999999,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dict_match",
            "role": "gate",
            "field": "cell_type_direction_map",
            "kind": "dict",
            "expected": {
              "Myeloid": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "Epithelia": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Fibroblasts": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "T_NK": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "B_plasma": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Endothelial": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Mast": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "APC": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Neuronal": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              }
            }
          },
          {
            "name": "over_merge_immune_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Immune",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_leukocyte_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Leukocyte",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_lymphoid_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Lymphoid",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_stromal_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Stromal",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-kimi-k3-pi",
      "modelName": "kimi-k3",
      "harness": "pi",
      "task": "snrnaseq_per_donor_composition_shifts_lethal_covid_lung",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 2488,
      "cost": 0.5446326,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dict_match",
            "role": "gate",
            "field": "cell_type_direction_map",
            "kind": "dict",
            "expected": {
              "Myeloid": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "Epithelia": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Fibroblasts": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "T_NK": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "B_plasma": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Endothelial": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Mast": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "APC": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Neuronal": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              }
            }
          },
          {
            "name": "over_merge_immune_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Immune",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_leukocyte_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Leukocyte",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_lymphoid_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Lymphoid",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_stromal_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Stromal",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-kimi-k3-pi",
      "modelName": "kimi-k3",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_enriched_apc_states_v2",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 2424,
      "cost": 1.5229914000000002,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "top_3_cluster_enriched_apc_states",
            "kind": "list",
            "matchKey": "apc_state",
            "k": 3,
            "expected": [
              {
                "key": "plasmacytoid_DC",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "T_cell_stimulatory_role_supported",
                      "role_direction_unresolved"
                    ],
                    "canonical": "T_cell_stimulatory_role_supported"
                  }
                }
              },
              {
                "key": "C1qhigh_lipid_associated_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "C1qhigh_inflammatory_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "plasma_cell",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "cohort_significant_per_patient_inconsistent"
                    ],
                    "canonical": "cohort_significant_per_patient_inconsistent"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CD80_CD86": {
                "op": "in",
                "value": [
                  "mreg_DC",
                  "shared_across_APC_states"
                ]
              },
              "PD_L1_in_APC_clusters": {
                "op": "in",
                "value": [
                  "C1qhigh_macrophage_either_subtype",
                  "shared_across_APC_states",
                  "mreg_DC"
                ]
              },
              "type_I_IFN": {
                "op": "in",
                "value": [
                  "plasmacytoid_DC",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "cf_mreg_dc_claimed_enriched",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "mreg_DC",
              "mature_DC_enriched_in_immunoregulatory_molecules"
            ]
          },
          {
            "name": "cf_framework_offcompartment_or_generic_substitution",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "cDC1",
              "Maier_mregDC",
              "Cheng_C1QC_TAM",
              "Cheng_SPP1_TAM",
              "Cheng_FOLR2_TAM",
              "Villani_pDC",
              "Zhang_inflammatory_monocyte",
              "LAMP3_pos_DC",
              "myeloid_derived_suppressor_cell",
              "MDSC",
              "tumor_associated_neutrophil",
              "mast_cell",
              "APC",
              "antigen_presenting_cell",
              "macrophage",
              "dendritic_cell",
              "tumor_associated_macrophage",
              "TAM"
            ]
          },
          {
            "name": "cf_direction_commit_on_unresolvable_slot",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_not_resolvable_hedge",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].functional_attribution_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_type_I_IFN_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.type_I_IFN",
            "kind": "value",
            "op": "in",
            "expected": [
              "mreg_DC",
              "C1qhigh_macrophage_either_subtype"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-kimi-k3-pi",
      "modelName": "kimi-k3",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_enriched_apc_states_v2",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 2252,
      "cost": 1.6100691,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "top_3_cluster_enriched_apc_states",
            "kind": "list",
            "matchKey": "apc_state",
            "k": 3,
            "expected": [
              {
                "key": "plasmacytoid_DC",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "T_cell_stimulatory_role_supported",
                      "role_direction_unresolved"
                    ],
                    "canonical": "T_cell_stimulatory_role_supported"
                  }
                }
              },
              {
                "key": "C1qhigh_lipid_associated_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "C1qhigh_inflammatory_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "plasma_cell",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "cohort_significant_per_patient_inconsistent"
                    ],
                    "canonical": "cohort_significant_per_patient_inconsistent"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CD80_CD86": {
                "op": "in",
                "value": [
                  "mreg_DC",
                  "shared_across_APC_states"
                ]
              },
              "PD_L1_in_APC_clusters": {
                "op": "in",
                "value": [
                  "C1qhigh_macrophage_either_subtype",
                  "shared_across_APC_states",
                  "mreg_DC"
                ]
              },
              "type_I_IFN": {
                "op": "in",
                "value": [
                  "plasmacytoid_DC",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "cf_mreg_dc_claimed_enriched",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "mreg_DC",
              "mature_DC_enriched_in_immunoregulatory_molecules"
            ]
          },
          {
            "name": "cf_framework_offcompartment_or_generic_substitution",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "cDC1",
              "Maier_mregDC",
              "Cheng_C1QC_TAM",
              "Cheng_SPP1_TAM",
              "Cheng_FOLR2_TAM",
              "Villani_pDC",
              "Zhang_inflammatory_monocyte",
              "LAMP3_pos_DC",
              "myeloid_derived_suppressor_cell",
              "MDSC",
              "tumor_associated_neutrophil",
              "mast_cell",
              "APC",
              "antigen_presenting_cell",
              "macrophage",
              "dendritic_cell",
              "tumor_associated_macrophage",
              "TAM"
            ]
          },
          {
            "name": "cf_direction_commit_on_unresolvable_slot",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_not_resolvable_hedge",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].functional_attribution_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_type_I_IFN_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.type_I_IFN",
            "kind": "value",
            "op": "in",
            "expected": [
              "mreg_DC",
              "C1qhigh_macrophage_either_subtype"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-kimi-k3-pi",
      "modelName": "kimi-k3",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_enriched_apc_states_v2",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 3934,
      "cost": 3.6385800000000006,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "top_3_cluster_enriched_apc_states",
            "kind": "list",
            "matchKey": "apc_state",
            "k": 3,
            "expected": [
              {
                "key": "plasmacytoid_DC",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "T_cell_stimulatory_role_supported",
                      "role_direction_unresolved"
                    ],
                    "canonical": "T_cell_stimulatory_role_supported"
                  }
                }
              },
              {
                "key": "C1qhigh_lipid_associated_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "C1qhigh_inflammatory_macrophage",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "role_direction_unresolved",
                      "immunosuppressive_prior_direction_unresolved"
                    ],
                    "canonical": "role_direction_unresolved"
                  }
                }
              },
              {
                "key": "plasma_cell",
                "fields": {
                  "functional_attribution_status": {
                    "op": "in",
                    "values": [
                      "cohort_significant_per_patient_inconsistent"
                    ],
                    "canonical": "cohort_significant_per_patient_inconsistent"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CD80_CD86": {
                "op": "in",
                "value": [
                  "mreg_DC",
                  "shared_across_APC_states"
                ]
              },
              "PD_L1_in_APC_clusters": {
                "op": "in",
                "value": [
                  "C1qhigh_macrophage_either_subtype",
                  "shared_across_APC_states",
                  "mreg_DC"
                ]
              },
              "type_I_IFN": {
                "op": "in",
                "value": [
                  "plasmacytoid_DC",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "cf_mreg_dc_claimed_enriched",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "mreg_DC",
              "mature_DC_enriched_in_immunoregulatory_molecules"
            ]
          },
          {
            "name": "cf_framework_offcompartment_or_generic_substitution",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].apc_state",
            "kind": "avoid",
            "expected": [
              "cDC1",
              "Maier_mregDC",
              "Cheng_C1QC_TAM",
              "Cheng_SPP1_TAM",
              "Cheng_FOLR2_TAM",
              "Villani_pDC",
              "Zhang_inflammatory_monocyte",
              "LAMP3_pos_DC",
              "myeloid_derived_suppressor_cell",
              "MDSC",
              "tumor_associated_neutrophil",
              "mast_cell",
              "APC",
              "antigen_presenting_cell",
              "macrophage",
              "dendritic_cell",
              "tumor_associated_macrophage",
              "TAM"
            ]
          },
          {
            "name": "cf_direction_commit_on_unresolvable_slot",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_not_resolvable_hedge",
            "role": "hard_fail",
            "field": "$.top_3_cluster_enriched_apc_states[*].functional_attribution_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_type_I_IFN_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.type_I_IFN",
            "kind": "value",
            "op": "in",
            "expected": [
              "mreg_DC",
              "C1qhigh_macrophage_either_subtype"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-kimi-k3-pi",
      "modelName": "kimi-k3",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_tumor_state_causation_v5",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 5867,
      "cost": 3.378582,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "enriched_tumor_states",
            "kind": "list",
            "matchKey": "tumor_state",
            "k": 3,
            "expected": [
              {
                "key": "immune_response",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "direction_unresolved_cross_sectional"
                    ],
                    "canonical": "direction_unresolved_cross_sectional"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "IFN_response_axis",
                      "broad_antigen_presentation_MHC"
                    ],
                    "canonical": "IFN_response_axis"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "enrichment_diminished_by_decontamination",
                      "threshold_sensitive"
                    ],
                    "canonical": "enrichment_diminished_by_decontamination"
                  }
                }
              },
              {
                "key": "stress_hypoxia",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "data_supports_cell_intrinsic_cause"
                    ],
                    "canonical": "data_supports_cell_intrinsic_cause"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "HIF_hypoxia_program_specific"
                    ],
                    "canonical": "HIF_hypoxia_program_specific"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "robust_across_methods",
                      "robust_after_decontamination"
                    ],
                    "canonical": "robust_across_methods"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CXCL9_CXCL10": {
                "op": "in",
                "value": [
                  "immune_response_tumor_state"
                ]
              },
              "CCL5": {
                "op": "in",
                "value": [
                  "shared_across_tumor_states",
                  "partner_lymphocyte_derived",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "gate_conjugate_gate_confounder",
            "role": "gate",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "partner_lymphocyte_capture"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "antigen_presentation_decomposition",
            "kind": "dict",
            "expected": {
              "MHC_class_I": {
                "op": "in",
                "value": [
                  "co_regulated_with_dominant_program",
                  "tumor_intrinsic_enriched_independent"
                ]
              },
              "MHC_class_II": {
                "op": "in",
                "value": [
                  "partner_cell_attributable",
                  "flat_not_enriched",
                  "method_sensitive_no_robust_call",
                  "co_regulated_with_dominant_program"
                ]
              }
            }
          },
          {
            "name": "cf_non_robust_state_claimed_enriched",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "transitory_melanocytic",
              "intermediate_melanocytic",
              "melanocytic",
              "MITF_high",
              "neural_crest_like",
              "mitotic",
              "stress_p53",
              "undifferentiated",
              "dedifferentiated",
              "mesenchymal_tumor",
              "AXL_high_tumor",
              "invasive_tumor",
              "senescence_like",
              "EMT_transition",
              "cancer_stem_cell_like",
              "oxidative_stress_tumor"
            ]
          },
          {
            "name": "cf_framework_generic_or_offtissue_substitution",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "Tsoi_undifferentiated",
              "Tsoi_neural_crest_like",
              "Tsoi_transitory",
              "Tsoi_melanocytic",
              "Rambow_NCSC",
              "Wouters_mesenchymal",
              "Pozniak_TCF4_dependent",
              "tumor_cell",
              "melanoma_cell",
              "malignant_cell",
              "tumor_subpopulation",
              "keratinocyte",
              "hepatocyte",
              "lung_adenocarcinoma_cell",
              "colon_adenocarcinoma_cell",
              "Tirosh_cell_cycle_program",
              "Jerby_Arnon_program",
              "glioblastoma_cell",
              "breast_carcinoma_cell"
            ]
          },
          {
            "name": "cf_immune_response_direction_committed",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_immune_robustness_recall",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_ambiguous_hedge",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].causation_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_stress_generic_stress_response_evidence",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_conjugate_confounder_contamination_denied",
            "role": "hard_fail",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected"
            ]
          },
          {
            "name": "cf_ccl5_recall_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CCL5",
            "kind": "avoid",
            "expected": [
              "immune_response_tumor_state"
            ]
          },
          {
            "name": "cf_cxcl9_cxcl10_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CXCL9_CXCL10",
            "kind": "avoid",
            "expected": [
              "stress_hypoxia_tumor_state"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-kimi-k3-pi",
      "modelName": "kimi-k3",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_tumor_state_causation_v5",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 3409,
      "cost": 1.7447502000000004,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "enriched_tumor_states",
            "kind": "list",
            "matchKey": "tumor_state",
            "k": 3,
            "expected": [
              {
                "key": "immune_response",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "direction_unresolved_cross_sectional"
                    ],
                    "canonical": "direction_unresolved_cross_sectional"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "IFN_response_axis",
                      "broad_antigen_presentation_MHC"
                    ],
                    "canonical": "IFN_response_axis"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "enrichment_diminished_by_decontamination",
                      "threshold_sensitive"
                    ],
                    "canonical": "enrichment_diminished_by_decontamination"
                  }
                }
              },
              {
                "key": "stress_hypoxia",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "data_supports_cell_intrinsic_cause"
                    ],
                    "canonical": "data_supports_cell_intrinsic_cause"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "HIF_hypoxia_program_specific"
                    ],
                    "canonical": "HIF_hypoxia_program_specific"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "robust_across_methods",
                      "robust_after_decontamination"
                    ],
                    "canonical": "robust_across_methods"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CXCL9_CXCL10": {
                "op": "in",
                "value": [
                  "immune_response_tumor_state"
                ]
              },
              "CCL5": {
                "op": "in",
                "value": [
                  "shared_across_tumor_states",
                  "partner_lymphocyte_derived",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "gate_conjugate_gate_confounder",
            "role": "gate",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "partner_lymphocyte_capture"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "antigen_presentation_decomposition",
            "kind": "dict",
            "expected": {
              "MHC_class_I": {
                "op": "in",
                "value": [
                  "co_regulated_with_dominant_program",
                  "tumor_intrinsic_enriched_independent"
                ]
              },
              "MHC_class_II": {
                "op": "in",
                "value": [
                  "partner_cell_attributable",
                  "flat_not_enriched",
                  "method_sensitive_no_robust_call",
                  "co_regulated_with_dominant_program"
                ]
              }
            }
          },
          {
            "name": "cf_non_robust_state_claimed_enriched",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "transitory_melanocytic",
              "intermediate_melanocytic",
              "melanocytic",
              "MITF_high",
              "neural_crest_like",
              "mitotic",
              "stress_p53",
              "undifferentiated",
              "dedifferentiated",
              "mesenchymal_tumor",
              "AXL_high_tumor",
              "invasive_tumor",
              "senescence_like",
              "EMT_transition",
              "cancer_stem_cell_like",
              "oxidative_stress_tumor"
            ]
          },
          {
            "name": "cf_framework_generic_or_offtissue_substitution",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "Tsoi_undifferentiated",
              "Tsoi_neural_crest_like",
              "Tsoi_transitory",
              "Tsoi_melanocytic",
              "Rambow_NCSC",
              "Wouters_mesenchymal",
              "Pozniak_TCF4_dependent",
              "tumor_cell",
              "melanoma_cell",
              "malignant_cell",
              "tumor_subpopulation",
              "keratinocyte",
              "hepatocyte",
              "lung_adenocarcinoma_cell",
              "colon_adenocarcinoma_cell",
              "Tirosh_cell_cycle_program",
              "Jerby_Arnon_program",
              "glioblastoma_cell",
              "breast_carcinoma_cell"
            ]
          },
          {
            "name": "cf_immune_response_direction_committed",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_immune_robustness_recall",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_ambiguous_hedge",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].causation_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_stress_generic_stress_response_evidence",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_conjugate_confounder_contamination_denied",
            "role": "hard_fail",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected"
            ]
          },
          {
            "name": "cf_ccl5_recall_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CCL5",
            "kind": "avoid",
            "expected": [
              "immune_response_tumor_state"
            ]
          },
          {
            "name": "cf_cxcl9_cxcl10_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CXCL9_CXCL10",
            "kind": "avoid",
            "expected": [
              "stress_hypoxia_tumor_state"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-kimi-k3-pi",
      "modelName": "kimi-k3",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_tumor_state_causation_v5",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 2701,
      "cost": 1.6640304,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "enriched_tumor_states",
            "kind": "list",
            "matchKey": "tumor_state",
            "k": 3,
            "expected": [
              {
                "key": "immune_response",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "direction_unresolved_cross_sectional"
                    ],
                    "canonical": "direction_unresolved_cross_sectional"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "IFN_response_axis",
                      "broad_antigen_presentation_MHC"
                    ],
                    "canonical": "IFN_response_axis"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "enrichment_diminished_by_decontamination",
                      "threshold_sensitive"
                    ],
                    "canonical": "enrichment_diminished_by_decontamination"
                  }
                }
              },
              {
                "key": "stress_hypoxia",
                "fields": {
                  "causation_status": {
                    "op": "in",
                    "values": [
                      "data_supports_cell_intrinsic_cause"
                    ],
                    "canonical": "data_supports_cell_intrinsic_cause"
                  },
                  "enrichment_evidence": {
                    "op": "in",
                    "values": [
                      "HIF_hypoxia_program_specific"
                    ],
                    "canonical": "HIF_hypoxia_program_specific"
                  },
                  "robustness": {
                    "op": "in",
                    "values": [
                      "robust_across_methods",
                      "robust_after_decontamination"
                    ],
                    "canonical": "robust_across_methods"
                  }
                }
              }
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "ligand_attribution",
            "kind": "dict",
            "expected": {
              "CXCL9_CXCL10": {
                "op": "in",
                "value": [
                  "immune_response_tumor_state"
                ]
              },
              "CCL5": {
                "op": "in",
                "value": [
                  "shared_across_tumor_states",
                  "partner_lymphocyte_derived",
                  "below_detection"
                ]
              }
            }
          },
          {
            "name": "gate_conjugate_gate_confounder",
            "role": "gate",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "partner_lymphocyte_capture"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "antigen_presentation_decomposition",
            "kind": "dict",
            "expected": {
              "MHC_class_I": {
                "op": "in",
                "value": [
                  "co_regulated_with_dominant_program",
                  "tumor_intrinsic_enriched_independent"
                ]
              },
              "MHC_class_II": {
                "op": "in",
                "value": [
                  "partner_cell_attributable",
                  "flat_not_enriched",
                  "method_sensitive_no_robust_call",
                  "co_regulated_with_dominant_program"
                ]
              }
            }
          },
          {
            "name": "cf_non_robust_state_claimed_enriched",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "transitory_melanocytic",
              "intermediate_melanocytic",
              "melanocytic",
              "MITF_high",
              "neural_crest_like",
              "mitotic",
              "stress_p53",
              "undifferentiated",
              "dedifferentiated",
              "mesenchymal_tumor",
              "AXL_high_tumor",
              "invasive_tumor",
              "senescence_like",
              "EMT_transition",
              "cancer_stem_cell_like",
              "oxidative_stress_tumor"
            ]
          },
          {
            "name": "cf_framework_generic_or_offtissue_substitution",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].tumor_state",
            "kind": "avoid",
            "expected": [
              "Tsoi_undifferentiated",
              "Tsoi_neural_crest_like",
              "Tsoi_transitory",
              "Tsoi_melanocytic",
              "Rambow_NCSC",
              "Wouters_mesenchymal",
              "Pozniak_TCF4_dependent",
              "tumor_cell",
              "melanoma_cell",
              "malignant_cell",
              "tumor_subpopulation",
              "keratinocyte",
              "hepatocyte",
              "lung_adenocarcinoma_cell",
              "colon_adenocarcinoma_cell",
              "Tirosh_cell_cycle_program",
              "Jerby_Arnon_program",
              "glioblastoma_cell",
              "breast_carcinoma_cell"
            ]
          },
          {
            "name": "cf_immune_response_direction_committed",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_immune_robustness_recall",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_uniform_ambiguous_hedge",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*].causation_status",
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_stress_generic_stress_response_evidence",
            "role": "hard_fail",
            "field": "$.enriched_tumor_states[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_conjugate_confounder_contamination_denied",
            "role": "hard_fail",
            "field": "conjugate_gate_confounder",
            "kind": "value",
            "op": "in",
            "expected": [
              "none_detected"
            ]
          },
          {
            "name": "cf_ccl5_recall_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CCL5",
            "kind": "avoid",
            "expected": [
              "immune_response_tumor_state"
            ]
          },
          {
            "name": "cf_cxcl9_cxcl10_reverse_attribution",
            "role": "hard_fail",
            "field": "$.ligand_attribution.CXCL9_CXCL10",
            "kind": "avoid",
            "expected": [
              "stress_hypoxia_tumor_state"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-kimi-k3-pi",
      "modelName": "kimi-k3",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_vs_cd39_tumor_reactivity_dominance",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 1720,
      "cost": 1.1073377999999998,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "population_findings",
            "kind": "list",
            "matchKey": "capture_gate",
            "k": 4,
            "expected": [
              {
                "key": "tumor_cell_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "apc_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "combined_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "unselected_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "naive",
                      "central_memory",
                      "effector_memory"
                    ],
                    "canonical": "naive"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "low"
                    ],
                    "canonical": "low"
                  },
                  "purity_caveat": {
                    "op": "equals",
                    "values": [
                      "pure_T_cell"
                    ],
                    "canonical": "pure_T_cell"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "no_reactivity_evidence"
                    ],
                    "canonical": "no_reactivity_evidence"
                  }
                }
              },
              {
                "key": "marker_sorted_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "equals",
                    "values": [
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "terminal_exhausted_LAG3_high"
                  },
                  "clonal_expansion_level": {
                    "op": "in",
                    "values": [
                      "high",
                      "intermediate"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "marker_on_non_T_lineages",
                      "single_cell_gating_loses_target"
                    ],
                    "canonical": "marker_on_non_T_lineages"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "marker_or_exhaustion_defined_only"
                    ],
                    "canonical": "marker_or_exhaustion_defined_only"
                  }
                }
              }
            ]
          },
          {
            "name": "best_capture_approach_gate",
            "role": "gate",
            "field": "best_capture_approach",
            "kind": "value",
            "expected": "heterotypic_cell_conjugate"
          },
          {
            "name": "best_selection_marker_gate",
            "role": "gate",
            "field": "best_selection_marker",
            "kind": "value",
            "expected": "none"
          },
          {
            "name": "reactivity_criterion_gate",
            "role": "gate",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "clonal_expansion_and_physical_engagement"
          },
          {
            "name": "cross_patient_robustness_gate",
            "role": "gate",
            "field": "cross_patient_robustness",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_across_patients",
              "marginal_in_some_patients"
            ]
          },
          {
            "name": "marker_selection_consequence_gate",
            "role": "gate",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "discards_clonally_expanded_reactive_cells"
          },
          {
            "name": "superiority_claim_basis_gate",
            "role": "gate",
            "field": "superiority_claim_basis",
            "kind": "value",
            "expected": "requires_functional_reactivity_assay"
          },
          {
            "name": "conjugate_enriched_state_gate",
            "role": "gate",
            "field": "conjugate_enriched_state",
            "kind": "value",
            "expected": "progenitor_exhausted_TCF7_pos_stem_like"
          },
          {
            "name": "cf_circular_exhaustion_criterion",
            "role": "hard_fail",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "exhaustion_signature_intensity"
          },
          {
            "name": "cf_marker_sort_prior",
            "role": "hard_fail",
            "field": "best_selection_marker",
            "kind": "value",
            "op": "in",
            "expected": [
              "CD39",
              "PD1",
              "PD1_high",
              "TIM3",
              "PD1_CD39",
              "CD39_CD103"
            ]
          },
          {
            "name": "cf_single_cell_capture_prior",
            "role": "hard_fail",
            "field": "best_capture_approach",
            "kind": "value",
            "op": "in",
            "expected": [
              "dissociated_single_cell",
              "dissociated_bulk_population"
            ]
          },
          {
            "name": "cf_off_data_capture_gate",
            "role": "hard_fail",
            "field": "$.population_findings[*].capture_gate",
            "kind": "avoid",
            "expected": [
              "dissociated_bulk",
              "matched_peripheral_blood",
              "matched_lymph_node",
              "post_hoc_expanded_clonotype_group"
            ]
          },
          {
            "name": "cf_conjugate_mischaracterized",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_marker_singlet_called_engaged",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "contains",
            "matchKey": "capture_gate",
            "entity": "marker_sorted_singlet",
            "valueField": "reactivity_evidence",
            "expected": [
              "engagement_and_clonal_expansion"
            ]
          },
          {
            "name": "cf_no_engaged_reactive_population",
            "role": "hard_fail",
            "field": "$.population_findings[*].reactivity_evidence",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_marker_captures_all_reactive",
            "role": "hard_fail",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "captures_full_reactive_repertoire"
          },
          {
            "name": "cf_superiority_overclaimed_from_phenotype",
            "role": "hard_fail",
            "field": "superiority_claim_basis",
            "kind": "value",
            "op": "in",
            "expected": [
              "established_by_scrna_phenotype",
              "established_by_reactivity_proxy_scores"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-kimi-k3-pi",
      "modelName": "kimi-k3",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_vs_cd39_tumor_reactivity_dominance",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 1575,
      "cost": 1.023975,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "population_findings",
            "kind": "list",
            "matchKey": "capture_gate",
            "k": 4,
            "expected": [
              {
                "key": "tumor_cell_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "apc_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "combined_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "unselected_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "naive",
                      "central_memory",
                      "effector_memory"
                    ],
                    "canonical": "naive"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "low"
                    ],
                    "canonical": "low"
                  },
                  "purity_caveat": {
                    "op": "equals",
                    "values": [
                      "pure_T_cell"
                    ],
                    "canonical": "pure_T_cell"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "no_reactivity_evidence"
                    ],
                    "canonical": "no_reactivity_evidence"
                  }
                }
              },
              {
                "key": "marker_sorted_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "equals",
                    "values": [
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "terminal_exhausted_LAG3_high"
                  },
                  "clonal_expansion_level": {
                    "op": "in",
                    "values": [
                      "high",
                      "intermediate"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "marker_on_non_T_lineages",
                      "single_cell_gating_loses_target"
                    ],
                    "canonical": "marker_on_non_T_lineages"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "marker_or_exhaustion_defined_only"
                    ],
                    "canonical": "marker_or_exhaustion_defined_only"
                  }
                }
              }
            ]
          },
          {
            "name": "best_capture_approach_gate",
            "role": "gate",
            "field": "best_capture_approach",
            "kind": "value",
            "expected": "heterotypic_cell_conjugate"
          },
          {
            "name": "best_selection_marker_gate",
            "role": "gate",
            "field": "best_selection_marker",
            "kind": "value",
            "expected": "none"
          },
          {
            "name": "reactivity_criterion_gate",
            "role": "gate",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "clonal_expansion_and_physical_engagement"
          },
          {
            "name": "cross_patient_robustness_gate",
            "role": "gate",
            "field": "cross_patient_robustness",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_across_patients",
              "marginal_in_some_patients"
            ]
          },
          {
            "name": "marker_selection_consequence_gate",
            "role": "gate",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "discards_clonally_expanded_reactive_cells"
          },
          {
            "name": "superiority_claim_basis_gate",
            "role": "gate",
            "field": "superiority_claim_basis",
            "kind": "value",
            "expected": "requires_functional_reactivity_assay"
          },
          {
            "name": "conjugate_enriched_state_gate",
            "role": "gate",
            "field": "conjugate_enriched_state",
            "kind": "value",
            "expected": "progenitor_exhausted_TCF7_pos_stem_like"
          },
          {
            "name": "cf_circular_exhaustion_criterion",
            "role": "hard_fail",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "exhaustion_signature_intensity"
          },
          {
            "name": "cf_marker_sort_prior",
            "role": "hard_fail",
            "field": "best_selection_marker",
            "kind": "value",
            "op": "in",
            "expected": [
              "CD39",
              "PD1",
              "PD1_high",
              "TIM3",
              "PD1_CD39",
              "CD39_CD103"
            ]
          },
          {
            "name": "cf_single_cell_capture_prior",
            "role": "hard_fail",
            "field": "best_capture_approach",
            "kind": "value",
            "op": "in",
            "expected": [
              "dissociated_single_cell",
              "dissociated_bulk_population"
            ]
          },
          {
            "name": "cf_off_data_capture_gate",
            "role": "hard_fail",
            "field": "$.population_findings[*].capture_gate",
            "kind": "avoid",
            "expected": [
              "dissociated_bulk",
              "matched_peripheral_blood",
              "matched_lymph_node",
              "post_hoc_expanded_clonotype_group"
            ]
          },
          {
            "name": "cf_conjugate_mischaracterized",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_marker_singlet_called_engaged",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "contains",
            "matchKey": "capture_gate",
            "entity": "marker_sorted_singlet",
            "valueField": "reactivity_evidence",
            "expected": [
              "engagement_and_clonal_expansion"
            ]
          },
          {
            "name": "cf_no_engaged_reactive_population",
            "role": "hard_fail",
            "field": "$.population_findings[*].reactivity_evidence",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_marker_captures_all_reactive",
            "role": "hard_fail",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "captures_full_reactive_repertoire"
          },
          {
            "name": "cf_superiority_overclaimed_from_phenotype",
            "role": "hard_fail",
            "field": "superiority_claim_basis",
            "kind": "value",
            "op": "in",
            "expected": [
              "established_by_scrna_phenotype",
              "established_by_reactivity_proxy_scores"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-kimi-k3-pi",
      "modelName": "kimi-k3",
      "harness": "pi",
      "task": "10x_5p_melanoma_cluster_vs_cd39_tumor_reactivity_dominance",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 1809,
      "cost": 1.027143,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "population_findings",
            "kind": "list",
            "matchKey": "capture_gate",
            "k": 4,
            "expected": [
              {
                "key": "tumor_cell_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "apc_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "combined_conjugate",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "progenitor_exhausted_TCF7_pos_stem_like",
                      "proliferating_cycling",
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "progenitor_exhausted_TCF7_pos_stem_like"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "high"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "includes_conjugate_partner_rna"
                    ],
                    "canonical": "includes_conjugate_partner_rna"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "engagement_and_clonal_expansion"
                    ],
                    "canonical": "engagement_and_clonal_expansion"
                  }
                }
              },
              {
                "key": "unselected_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "in",
                    "values": [
                      "naive",
                      "central_memory",
                      "effector_memory"
                    ],
                    "canonical": "naive"
                  },
                  "clonal_expansion_level": {
                    "op": "equals",
                    "values": [
                      "low"
                    ],
                    "canonical": "low"
                  },
                  "purity_caveat": {
                    "op": "equals",
                    "values": [
                      "pure_T_cell"
                    ],
                    "canonical": "pure_T_cell"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "no_reactivity_evidence"
                    ],
                    "canonical": "no_reactivity_evidence"
                  }
                }
              },
              {
                "key": "marker_sorted_singlet",
                "fields": {
                  "dominant_state": {
                    "op": "equals",
                    "values": [
                      "terminal_exhausted_LAG3_high"
                    ],
                    "canonical": "terminal_exhausted_LAG3_high"
                  },
                  "clonal_expansion_level": {
                    "op": "in",
                    "values": [
                      "high",
                      "intermediate"
                    ],
                    "canonical": "high"
                  },
                  "purity_caveat": {
                    "op": "in",
                    "values": [
                      "pure_T_cell",
                      "marker_on_non_T_lineages",
                      "single_cell_gating_loses_target"
                    ],
                    "canonical": "marker_on_non_T_lineages"
                  },
                  "reactivity_evidence": {
                    "op": "equals",
                    "values": [
                      "marker_or_exhaustion_defined_only"
                    ],
                    "canonical": "marker_or_exhaustion_defined_only"
                  }
                }
              }
            ]
          },
          {
            "name": "best_capture_approach_gate",
            "role": "gate",
            "field": "best_capture_approach",
            "kind": "value",
            "expected": "heterotypic_cell_conjugate"
          },
          {
            "name": "best_selection_marker_gate",
            "role": "gate",
            "field": "best_selection_marker",
            "kind": "value",
            "expected": "none"
          },
          {
            "name": "reactivity_criterion_gate",
            "role": "gate",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "clonal_expansion_and_physical_engagement"
          },
          {
            "name": "cross_patient_robustness_gate",
            "role": "gate",
            "field": "cross_patient_robustness",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_across_patients",
              "marginal_in_some_patients"
            ]
          },
          {
            "name": "marker_selection_consequence_gate",
            "role": "gate",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "discards_clonally_expanded_reactive_cells"
          },
          {
            "name": "superiority_claim_basis_gate",
            "role": "gate",
            "field": "superiority_claim_basis",
            "kind": "value",
            "expected": "requires_functional_reactivity_assay"
          },
          {
            "name": "conjugate_enriched_state_gate",
            "role": "gate",
            "field": "conjugate_enriched_state",
            "kind": "value",
            "expected": "progenitor_exhausted_TCF7_pos_stem_like"
          },
          {
            "name": "cf_circular_exhaustion_criterion",
            "role": "hard_fail",
            "field": "reactivity_criterion",
            "kind": "value",
            "expected": "exhaustion_signature_intensity"
          },
          {
            "name": "cf_marker_sort_prior",
            "role": "hard_fail",
            "field": "best_selection_marker",
            "kind": "value",
            "op": "in",
            "expected": [
              "CD39",
              "PD1",
              "PD1_high",
              "TIM3",
              "PD1_CD39",
              "CD39_CD103"
            ]
          },
          {
            "name": "cf_single_cell_capture_prior",
            "role": "hard_fail",
            "field": "best_capture_approach",
            "kind": "value",
            "op": "in",
            "expected": [
              "dissociated_single_cell",
              "dissociated_bulk_population"
            ]
          },
          {
            "name": "cf_off_data_capture_gate",
            "role": "hard_fail",
            "field": "$.population_findings[*].capture_gate",
            "kind": "avoid",
            "expected": [
              "dissociated_bulk",
              "matched_peripheral_blood",
              "matched_lymph_node",
              "post_hoc_expanded_clonotype_group"
            ]
          },
          {
            "name": "cf_conjugate_mischaracterized",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_marker_singlet_called_engaged",
            "role": "hard_fail",
            "field": "$.population_findings[*]",
            "kind": "contains",
            "matchKey": "capture_gate",
            "entity": "marker_sorted_singlet",
            "valueField": "reactivity_evidence",
            "expected": [
              "engagement_and_clonal_expansion"
            ]
          },
          {
            "name": "cf_no_engaged_reactive_population",
            "role": "hard_fail",
            "field": "$.population_findings[*].reactivity_evidence",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_marker_captures_all_reactive",
            "role": "hard_fail",
            "field": "marker_selection_consequence",
            "kind": "value",
            "expected": "captures_full_reactive_repertoire"
          },
          {
            "name": "cf_superiority_overclaimed_from_phenotype",
            "role": "hard_fail",
            "field": "superiority_claim_basis",
            "kind": "value",
            "op": "in",
            "expected": [
              "established_by_scrna_phenotype",
              "established_by_reactivity_proxy_scores"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-kimi-k3-pi",
      "modelName": "kimi-k3",
      "harness": "pi",
      "task": "10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 1,
      "passed": true,
      "score": null,
      "durationS": 3453,
      "cost": 1.9062288,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_singlet_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "TOXhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "LAG3high_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "GZMKhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "intermediate_exhausted_gzmk"
                    ],
                    "canonical": "intermediate_exhausted_gzmk"
                  }
                }
              },
              {
                "key": "TCF7_stem_like_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "stem_like_exhausted"
                    ],
                    "canonical": "stem_like_exhausted"
                  }
                }
              },
              {
                "key": "MKI67_pos_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              },
              {
                "key": "MKI67high_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_tumor_cluster_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "Tn",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "Tn_Tmem",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "early_Tem_GZMK",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              },
              {
                "key": "Tem_GZMH",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              }
            ]
          },
          {
            "name": "clonotype_expansion_direction_data_true",
            "role": "gate",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_substantially_more_expanded_than_cluster",
              "singlet_modestly_more_expanded_than_cluster"
            ]
          },
          {
            "name": "biological_inference_data_true",
            "role": "gate",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_antigen_experienced_cluster_bystander"
            ]
          },
          {
            "name": "singlet_top3_contains_naive_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Tn",
              "Tn_Tmem",
              "early_Tem_GZMK",
              "Tem_GZMH"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "TOXhigh_Tex",
              "LAG3high_Tex",
              "GZMKhigh_Tex",
              "TCF7_stem_like_Tex",
              "MKI67_pos_Tex_Tprol",
              "MKI67high_Tex_Tprol"
            ]
          },
          {
            "name": "singlet_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "TOXhigh_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "TCF7_stem_like_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "MKI67_state_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "Tn_or_Tn_Tmem_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_stemlike_precursor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_reported_canonical_expansion_direction",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_substantially_more_expanded_than_singlet",
              "cluster_modestly_more_expanded_than_singlet"
            ]
          },
          {
            "name": "cf_expansion_lazy_defer",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "expansion_pattern_not_detectable"
            ]
          },
          {
            "name": "cf_reported_canonical_expected_biology",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_antigen_experienced_singlet_bystander"
            ]
          },
          {
            "name": "cf_biological_inference_null_artifact_or_lineage",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "differences_reflect_dissociation_or_sorting_artifact",
              "cluster_and_singlet_capture_same_biology",
              "cluster_and_singlet_capture_distinct_cd8_lineages"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "biological_relations",
            "kind": "dict",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_greater_than",
                  "greater_than"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "less_than",
                  "much_less_than"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than",
                  "approximately_equal"
                ]
              },
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal"
                ]
              },
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal",
                  "greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_exhaustion_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_naive_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_proliferation_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_expansion_richness_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_multichain_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_intermediate_gzmk_overgeneralized",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_melanocytic_conjugate_overcall",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "values": [
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_relations_mass_hedge",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-kimi-k3-pi",
      "modelName": "kimi-k3",
      "harness": "pi",
      "task": "10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 3056,
      "cost": 1.6292052,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_singlet_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "TOXhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "LAG3high_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "GZMKhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "intermediate_exhausted_gzmk"
                    ],
                    "canonical": "intermediate_exhausted_gzmk"
                  }
                }
              },
              {
                "key": "TCF7_stem_like_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "stem_like_exhausted"
                    ],
                    "canonical": "stem_like_exhausted"
                  }
                }
              },
              {
                "key": "MKI67_pos_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              },
              {
                "key": "MKI67high_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_tumor_cluster_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "Tn",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "Tn_Tmem",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "early_Tem_GZMK",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              },
              {
                "key": "Tem_GZMH",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              }
            ]
          },
          {
            "name": "clonotype_expansion_direction_data_true",
            "role": "gate",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_substantially_more_expanded_than_cluster",
              "singlet_modestly_more_expanded_than_cluster"
            ]
          },
          {
            "name": "biological_inference_data_true",
            "role": "gate",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_antigen_experienced_cluster_bystander"
            ]
          },
          {
            "name": "singlet_top3_contains_naive_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Tn",
              "Tn_Tmem",
              "early_Tem_GZMK",
              "Tem_GZMH"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "TOXhigh_Tex",
              "LAG3high_Tex",
              "GZMKhigh_Tex",
              "TCF7_stem_like_Tex",
              "MKI67_pos_Tex_Tprol",
              "MKI67high_Tex_Tprol"
            ]
          },
          {
            "name": "singlet_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "TOXhigh_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "TCF7_stem_like_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "MKI67_state_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "Tn_or_Tn_Tmem_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_stemlike_precursor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_reported_canonical_expansion_direction",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_substantially_more_expanded_than_singlet",
              "cluster_modestly_more_expanded_than_singlet"
            ]
          },
          {
            "name": "cf_expansion_lazy_defer",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "expansion_pattern_not_detectable"
            ]
          },
          {
            "name": "cf_reported_canonical_expected_biology",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_antigen_experienced_singlet_bystander"
            ]
          },
          {
            "name": "cf_biological_inference_null_artifact_or_lineage",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "differences_reflect_dissociation_or_sorting_artifact",
              "cluster_and_singlet_capture_same_biology",
              "cluster_and_singlet_capture_distinct_cd8_lineages"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "biological_relations",
            "kind": "dict",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_greater_than",
                  "greater_than"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "less_than",
                  "much_less_than"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than",
                  "approximately_equal"
                ]
              },
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal"
                ]
              },
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal",
                  "greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_exhaustion_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_naive_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_proliferation_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_expansion_richness_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_multichain_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_intermediate_gzmk_overgeneralized",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_melanocytic_conjugate_overcall",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "values": [
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_relations_mass_hedge",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-kimi-k3-pi",
      "modelName": "kimi-k3",
      "harness": "pi",
      "task": "10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 2138,
      "cost": 1.5223776000000002,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_singlet_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "TOXhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "LAG3high_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "GZMKhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "intermediate_exhausted_gzmk"
                    ],
                    "canonical": "intermediate_exhausted_gzmk"
                  }
                }
              },
              {
                "key": "TCF7_stem_like_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "stem_like_exhausted"
                    ],
                    "canonical": "stem_like_exhausted"
                  }
                }
              },
              {
                "key": "MKI67_pos_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              },
              {
                "key": "MKI67high_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_tumor_cluster_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "Tn",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "Tn_Tmem",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "early_Tem_GZMK",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              },
              {
                "key": "Tem_GZMH",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              }
            ]
          },
          {
            "name": "clonotype_expansion_direction_data_true",
            "role": "gate",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_substantially_more_expanded_than_cluster",
              "singlet_modestly_more_expanded_than_cluster"
            ]
          },
          {
            "name": "biological_inference_data_true",
            "role": "gate",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_antigen_experienced_cluster_bystander"
            ]
          },
          {
            "name": "singlet_top3_contains_naive_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Tn",
              "Tn_Tmem",
              "early_Tem_GZMK",
              "Tem_GZMH"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "TOXhigh_Tex",
              "LAG3high_Tex",
              "GZMKhigh_Tex",
              "TCF7_stem_like_Tex",
              "MKI67_pos_Tex_Tprol",
              "MKI67high_Tex_Tprol"
            ]
          },
          {
            "name": "singlet_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "TOXhigh_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "TCF7_stem_like_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "MKI67_state_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "Tn_or_Tn_Tmem_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_stemlike_precursor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_reported_canonical_expansion_direction",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_substantially_more_expanded_than_singlet",
              "cluster_modestly_more_expanded_than_singlet"
            ]
          },
          {
            "name": "cf_expansion_lazy_defer",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "expansion_pattern_not_detectable"
            ]
          },
          {
            "name": "cf_reported_canonical_expected_biology",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_antigen_experienced_singlet_bystander"
            ]
          },
          {
            "name": "cf_biological_inference_null_artifact_or_lineage",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "differences_reflect_dissociation_or_sorting_artifact",
              "cluster_and_singlet_capture_same_biology",
              "cluster_and_singlet_capture_distinct_cd8_lineages"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "biological_relations",
            "kind": "dict",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_greater_than",
                  "greater_than"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "less_than",
                  "much_less_than"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than",
                  "approximately_equal"
                ]
              },
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal"
                ]
              },
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal",
                  "greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_exhaustion_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_naive_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_proliferation_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_expansion_richness_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_multichain_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_intermediate_gzmk_overgeneralized",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_melanocytic_conjugate_overcall",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "values": [
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_relations_mass_hedge",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              }
            }
          }
        ]
      }
    },
    {
      "modelId": "sclong-kimi-k3-pi",
      "modelName": "kimi-k3",
      "harness": "pi",
      "task": "10x_5p_melanoma_tcr_matched_apc_vs_tumor_exhaustion",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 1,
      "passed": true,
      "score": null,
      "durationS": 1392,
      "cost": 0.4811268,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "state_discriminating_genes",
            "kind": "list",
            "matchKey": "gene",
            "k": 3,
            "expected": [
              {
                "key": "CTLA4",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TOX",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "ENTPD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "HAVCR2",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TIGIT",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "PDCD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "difference_attributable_to",
            "role": "gate",
            "field": "difference_attributable_to",
            "kind": "value",
            "expected": "covaries_with_apc_conjugate_state"
          },
          {
            "name": "causal_status",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "direction_unresolved_selection_confound"
          },
          {
            "name": "functional_interpretation",
            "role": "gate",
            "field": "functional_interpretation",
            "kind": "value",
            "expected": "costimulation_signature_function_unresolved"
          },
          {
            "name": "cf_contamination_gene_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "MLANA",
              "PMEL",
              "DCT",
              "TYR",
              "TYRP1",
              "SOX10",
              "MITF",
              "MAGEA4"
            ]
          },
          {
            "name": "cf_apc_partner_contamination_gene",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "HLA-DRA",
              "CD68",
              "LYZ",
              "CD14",
              "C1QA",
              "ITGAX",
              "CD1C"
            ]
          },
          {
            "name": "cf_ligand_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "CD274",
              "PDCD1LG2",
              "CD80",
              "CD86"
            ]
          },
          {
            "name": "cf_attribution_causal_or_baseline",
            "role": "hard_fail",
            "field": "$.difference_attributable_to",
            "kind": "avoid",
            "expected": [
              "apc_engagement",
              "tumor_cell_engagement",
              "clonotype_intrinsic_baseline"
            ]
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "engagement_induces_exhausted_state",
              "pre_exhausted_cells_select_into_apc_conjugates"
            ]
          },
          {
            "name": "cf_causal_fabrication",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "resolved_by_pseudotemporal_ordering",
              "resolved_by_perturbation_experiment"
            ]
          },
          {
            "name": "cf_functional_overclaim",
            "role": "hard_fail",
            "field": "$.functional_interpretation",
            "kind": "avoid",
            "expected": [
              "costimulation_licensed_reinvigoration_competent",
              "costimulation_licensed_checkpoint_responsive",
              "tumor_driven_terminal_exhaustion"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-kimi-k3-pi",
      "modelName": "kimi-k3",
      "harness": "pi",
      "task": "10x_5p_melanoma_tcr_matched_apc_vs_tumor_exhaustion",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 2,
      "passed": true,
      "score": null,
      "durationS": 1062,
      "cost": 0.8560356000000001,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "state_discriminating_genes",
            "kind": "list",
            "matchKey": "gene",
            "k": 3,
            "expected": [
              {
                "key": "CTLA4",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TOX",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "ENTPD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "HAVCR2",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TIGIT",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "PDCD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "difference_attributable_to",
            "role": "gate",
            "field": "difference_attributable_to",
            "kind": "value",
            "expected": "covaries_with_apc_conjugate_state"
          },
          {
            "name": "causal_status",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "direction_unresolved_selection_confound"
          },
          {
            "name": "functional_interpretation",
            "role": "gate",
            "field": "functional_interpretation",
            "kind": "value",
            "expected": "costimulation_signature_function_unresolved"
          },
          {
            "name": "cf_contamination_gene_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "MLANA",
              "PMEL",
              "DCT",
              "TYR",
              "TYRP1",
              "SOX10",
              "MITF",
              "MAGEA4"
            ]
          },
          {
            "name": "cf_apc_partner_contamination_gene",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "HLA-DRA",
              "CD68",
              "LYZ",
              "CD14",
              "C1QA",
              "ITGAX",
              "CD1C"
            ]
          },
          {
            "name": "cf_ligand_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "CD274",
              "PDCD1LG2",
              "CD80",
              "CD86"
            ]
          },
          {
            "name": "cf_attribution_causal_or_baseline",
            "role": "hard_fail",
            "field": "$.difference_attributable_to",
            "kind": "avoid",
            "expected": [
              "apc_engagement",
              "tumor_cell_engagement",
              "clonotype_intrinsic_baseline"
            ]
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "engagement_induces_exhausted_state",
              "pre_exhausted_cells_select_into_apc_conjugates"
            ]
          },
          {
            "name": "cf_causal_fabrication",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "resolved_by_pseudotemporal_ordering",
              "resolved_by_perturbation_experiment"
            ]
          },
          {
            "name": "cf_functional_overclaim",
            "role": "hard_fail",
            "field": "$.functional_interpretation",
            "kind": "avoid",
            "expected": [
              "costimulation_licensed_reinvigoration_competent",
              "costimulation_licensed_checkpoint_responsive",
              "tumor_driven_terminal_exhaustion"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-kimi-k3-pi",
      "modelName": "kimi-k3",
      "harness": "pi",
      "task": "10x_5p_melanoma_tcr_matched_apc_vs_tumor_exhaustion",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 716,
      "cost": 0.5237064000000001,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "state_discriminating_genes",
            "kind": "list",
            "matchKey": "gene",
            "k": 3,
            "expected": [
              {
                "key": "CTLA4",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TOX",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "ENTPD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "HAVCR2",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TIGIT",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "PDCD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "difference_attributable_to",
            "role": "gate",
            "field": "difference_attributable_to",
            "kind": "value",
            "expected": "covaries_with_apc_conjugate_state"
          },
          {
            "name": "causal_status",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "direction_unresolved_selection_confound"
          },
          {
            "name": "functional_interpretation",
            "role": "gate",
            "field": "functional_interpretation",
            "kind": "value",
            "expected": "costimulation_signature_function_unresolved"
          },
          {
            "name": "cf_contamination_gene_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "MLANA",
              "PMEL",
              "DCT",
              "TYR",
              "TYRP1",
              "SOX10",
              "MITF",
              "MAGEA4"
            ]
          },
          {
            "name": "cf_apc_partner_contamination_gene",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "HLA-DRA",
              "CD68",
              "LYZ",
              "CD14",
              "C1QA",
              "ITGAX",
              "CD1C"
            ]
          },
          {
            "name": "cf_ligand_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "CD274",
              "PDCD1LG2",
              "CD80",
              "CD86"
            ]
          },
          {
            "name": "cf_attribution_causal_or_baseline",
            "role": "hard_fail",
            "field": "$.difference_attributable_to",
            "kind": "avoid",
            "expected": [
              "apc_engagement",
              "tumor_cell_engagement",
              "clonotype_intrinsic_baseline"
            ]
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "engagement_induces_exhausted_state",
              "pre_exhausted_cells_select_into_apc_conjugates"
            ]
          },
          {
            "name": "cf_causal_fabrication",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "resolved_by_pseudotemporal_ordering",
              "resolved_by_perturbation_experiment"
            ]
          },
          {
            "name": "cf_functional_overclaim",
            "role": "hard_fail",
            "field": "$.functional_interpretation",
            "kind": "avoid",
            "expected": [
              "costimulation_licensed_reinvigoration_competent",
              "costimulation_licensed_checkpoint_responsive",
              "tumor_driven_terminal_exhaustion"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-kimi-k3-pi",
      "modelName": "kimi-k3",
      "harness": "pi",
      "task": "green_2_tme_context_dependence",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 2497,
      "cost": 1.8527814000000005,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "program_map",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "sig_includes_itgae",
            "role": "gate",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Itgae"
            ]
          },
          {
            "name": "declared_a_known_confounder",
            "role": "gate",
            "field": "$.excluded_confounders[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "cf_signature_gut_contaminant",
            "role": "hard_fail",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "generic_signature_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd103_program_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "<"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "per_model_regulatory_state",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "poised_unfired"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "late_activation"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "closing_committed"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_tuples",
            "kind": "list",
            "matchKey": "regulator",
            "k": 1,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "role": {
                    "op": "equals",
                    "values": [
                      "repressor"
                    ]
                  },
                  "gating_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  },
                  "evidence_basis": {
                    "op": "equals",
                    "values": [
                      "target_program_behavior"
                    ]
                  },
                  "expr_vs_activity": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "locus_differential_gate",
            "role": "gate",
            "field": "locus_differential",
            "kind": "value",
            "expected": "uniform_across_models"
          },
          {
            "name": "formation_limited_by_gate",
            "role": "gate",
            "field": "formation_limited_by",
            "kind": "value",
            "expected": "absent_induction"
          },
          {
            "name": "locus_competent_in_naive_gate",
            "role": "gate",
            "field": "locus_competent_in_naive",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "ready_silent_fraction_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 3,
            "expected": [
              {
                "key": "B16_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_KPC_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "activator_chromatin_footprint_gate",
            "role": "gate",
            "field": "activator_chromatin_footprint",
            "kind": "value",
            "expected": "inconsistent_across_formers"
          },
          {
            "name": "exhaustion_comparable_gate",
            "role": "gate",
            "field": "exhaustion_comparable_across_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_nondeterminant_gate",
            "role": "gate",
            "field": "exhaustion_explains_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_separable_program_gate",
            "role": "gate",
            "field": "exhaustion_separable_program",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "exhaustion_counterexample_gate",
            "role": "gate",
            "field": "exhaustion_counterexample_model",
            "kind": "value",
            "expected": "KPC_GP"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "exhaustion_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "cf_inconsistent_present_without_activator",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "program",
            "entity": "present",
            "valueField": "activator_state",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_no_block_but_absent",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "none",
            "valueField": "program",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_late_block_without_poised_locus",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "late_activation",
            "valueField": "locus_state",
            "expected": [
              "closing_committed",
              "active"
            ]
          },
          {
            "name": "cf_inconsistent_early_block_with_brake_released",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "early_release",
            "valueField": "brake_state",
            "expected": [
              "released"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-kimi-k3-pi",
      "modelName": "kimi-k3",
      "harness": "pi",
      "task": "green_2_tme_context_dependence",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 3557,
      "cost": 2.6783235,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "program_map",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "sig_includes_itgae",
            "role": "gate",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Itgae"
            ]
          },
          {
            "name": "declared_a_known_confounder",
            "role": "gate",
            "field": "$.excluded_confounders[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "cf_signature_gut_contaminant",
            "role": "hard_fail",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "generic_signature_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd103_program_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "<"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "per_model_regulatory_state",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "poised_unfired"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "late_activation"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "closing_committed"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_tuples",
            "kind": "list",
            "matchKey": "regulator",
            "k": 1,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "role": {
                    "op": "equals",
                    "values": [
                      "repressor"
                    ]
                  },
                  "gating_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  },
                  "evidence_basis": {
                    "op": "equals",
                    "values": [
                      "target_program_behavior"
                    ]
                  },
                  "expr_vs_activity": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "locus_differential_gate",
            "role": "gate",
            "field": "locus_differential",
            "kind": "value",
            "expected": "uniform_across_models"
          },
          {
            "name": "formation_limited_by_gate",
            "role": "gate",
            "field": "formation_limited_by",
            "kind": "value",
            "expected": "absent_induction"
          },
          {
            "name": "locus_competent_in_naive_gate",
            "role": "gate",
            "field": "locus_competent_in_naive",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "ready_silent_fraction_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 3,
            "expected": [
              {
                "key": "B16_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_KPC_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "activator_chromatin_footprint_gate",
            "role": "gate",
            "field": "activator_chromatin_footprint",
            "kind": "value",
            "expected": "inconsistent_across_formers"
          },
          {
            "name": "exhaustion_comparable_gate",
            "role": "gate",
            "field": "exhaustion_comparable_across_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_nondeterminant_gate",
            "role": "gate",
            "field": "exhaustion_explains_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_separable_program_gate",
            "role": "gate",
            "field": "exhaustion_separable_program",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "exhaustion_counterexample_gate",
            "role": "gate",
            "field": "exhaustion_counterexample_model",
            "kind": "value",
            "expected": "KPC_GP"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "exhaustion_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "cf_inconsistent_present_without_activator",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "program",
            "entity": "present",
            "valueField": "activator_state",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_no_block_but_absent",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "none",
            "valueField": "program",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_late_block_without_poised_locus",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "late_activation",
            "valueField": "locus_state",
            "expected": [
              "closing_committed",
              "active"
            ]
          },
          {
            "name": "cf_inconsistent_early_block_with_brake_released",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "early_release",
            "valueField": "brake_state",
            "expected": [
              "released"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-kimi-k3-pi",
      "modelName": "kimi-k3",
      "harness": "pi",
      "task": "green_2_tme_context_dependence",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 3454,
      "cost": 2.8892052,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "program_map",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "sig_includes_itgae",
            "role": "gate",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Itgae"
            ]
          },
          {
            "name": "declared_a_known_confounder",
            "role": "gate",
            "field": "$.excluded_confounders[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "cf_signature_gut_contaminant",
            "role": "hard_fail",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "generic_signature_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd103_program_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "<"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "per_model_regulatory_state",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "poised_unfired"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "late_activation"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "closing_committed"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_tuples",
            "kind": "list",
            "matchKey": "regulator",
            "k": 1,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "role": {
                    "op": "equals",
                    "values": [
                      "repressor"
                    ]
                  },
                  "gating_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  },
                  "evidence_basis": {
                    "op": "equals",
                    "values": [
                      "target_program_behavior"
                    ]
                  },
                  "expr_vs_activity": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "locus_differential_gate",
            "role": "gate",
            "field": "locus_differential",
            "kind": "value",
            "expected": "uniform_across_models"
          },
          {
            "name": "formation_limited_by_gate",
            "role": "gate",
            "field": "formation_limited_by",
            "kind": "value",
            "expected": "absent_induction"
          },
          {
            "name": "locus_competent_in_naive_gate",
            "role": "gate",
            "field": "locus_competent_in_naive",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "ready_silent_fraction_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 3,
            "expected": [
              {
                "key": "B16_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_KPC_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "activator_chromatin_footprint_gate",
            "role": "gate",
            "field": "activator_chromatin_footprint",
            "kind": "value",
            "expected": "inconsistent_across_formers"
          },
          {
            "name": "exhaustion_comparable_gate",
            "role": "gate",
            "field": "exhaustion_comparable_across_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_nondeterminant_gate",
            "role": "gate",
            "field": "exhaustion_explains_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_separable_program_gate",
            "role": "gate",
            "field": "exhaustion_separable_program",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "exhaustion_counterexample_gate",
            "role": "gate",
            "field": "exhaustion_counterexample_model",
            "kind": "value",
            "expected": "KPC_GP"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "exhaustion_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "cf_inconsistent_present_without_activator",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "program",
            "entity": "present",
            "valueField": "activator_state",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_no_block_but_absent",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "none",
            "valueField": "program",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_late_block_without_poised_locus",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "late_activation",
            "valueField": "locus_state",
            "expected": [
              "closing_committed",
              "active"
            ]
          },
          {
            "name": "cf_inconsistent_early_block_with_brake_released",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "early_release",
            "valueField": "brake_state",
            "expected": [
              "released"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-kimi-k3-pi",
      "modelName": "kimi-k3",
      "harness": "pi",
      "task": "green_klf2_hidden_repressor_of_trm_like_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 2983,
      "cost": 1.9511891999999995,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "regulator_gate",
            "role": "gate",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "KLF2",
              "Klf2"
            ]
          },
          {
            "name": "controlling_node_character_gate",
            "role": "gate",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "high_differential_expression"
          },
          {
            "name": "expression_vs_activity_gate",
            "role": "gate",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "expression_exceeds_activity"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "target_program_partition",
            "kind": "list",
            "matchKey": "gene",
            "k": 7,
            "expected": [
              {
                "key": "S1pr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "S1pr4",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Klrg1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Cx3cr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Itgae",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Hic1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "P2rx7",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              }
            ]
          },
          {
            "name": "causal_status_gate",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "association_only_sufficiency_requires_perturbation"
          },
          {
            "name": "residency_control_logic_gate",
            "role": "gate",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_release_necessary_not_sufficient_multi_regulon"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "trm_like_program_by_model",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_absent"
                    ],
                    "canonical": "niche_absent"
                  }
                }
              }
            ]
          },
          {
            "name": "cf_regulator_activator_or_wrong_node",
            "role": "hard_fail",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "RUNX3",
              "Runx3",
              "PRDM1",
              "Prdm1",
              "BLIMP1",
              "Blimp1",
              "BHLHE40",
              "Bhlhe40",
              "ZNF683",
              "Zfp683",
              "HOBIT",
              "Hobit",
              "AHR",
              "Ahr",
              "BATF",
              "Batf",
              "TOX",
              "Tox",
              "NR4A1",
              "Nr4a1",
              "NR4A2",
              "Nr4a2",
              "NR4A3",
              "Nr4a3",
              "TCF7",
              "Tcf7",
              "TCF1",
              "Tcf1",
              "BACH2",
              "Bach2",
              "TBX21",
              "Tbx21",
              "TBET",
              "Tbet",
              "FLI1",
              "Fli1",
              "NR3C1",
              "Nr3c1",
              "EOMES",
              "Eomes",
              "ZEB2",
              "Zeb2",
              "ID2",
              "Id2",
              "ID3",
              "Id3",
              "IRF4",
              "Irf4"
            ]
          },
          {
            "name": "cf_node_character_hidden_driver_recall",
            "role": "hard_fail",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "hidden_driver_activity_not_expression"
          },
          {
            "name": "cf_effect_relation_hidden_pattern",
            "role": "hard_fail",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "activity_exceeds_expression"
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "causal_status",
            "kind": "value",
            "op": "in",
            "expected": [
              "sufficiency_established",
              "necessity_established",
              "causal_established_in_dataset",
              "sufficiency_via_perturbation_in_dataset"
            ]
          },
          {
            "name": "cf_residency_logic_activator_prior",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "residency_activator_induction_primary"
          },
          {
            "name": "cf_residency_logic_single_regulon",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_controls_entire_residency_program"
          },
          {
            "name": "cf_decoy_called_target",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_distractor_model_reported",
            "role": "hard_fail",
            "field": "$.trm_like_program_by_model[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_chromatin_only_decoy",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          }
        ]
      }
    },
    {
      "modelId": "sclong-kimi-k3-pi",
      "modelName": "kimi-k3",
      "harness": "pi",
      "task": "green_klf2_hidden_repressor_of_trm_like_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 6409,
      "cost": 4.957282200000001,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "regulator_gate",
            "role": "gate",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "KLF2",
              "Klf2"
            ]
          },
          {
            "name": "controlling_node_character_gate",
            "role": "gate",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "high_differential_expression"
          },
          {
            "name": "expression_vs_activity_gate",
            "role": "gate",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "expression_exceeds_activity"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "target_program_partition",
            "kind": "list",
            "matchKey": "gene",
            "k": 7,
            "expected": [
              {
                "key": "S1pr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "S1pr4",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Klrg1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Cx3cr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Itgae",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Hic1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "P2rx7",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              }
            ]
          },
          {
            "name": "causal_status_gate",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "association_only_sufficiency_requires_perturbation"
          },
          {
            "name": "residency_control_logic_gate",
            "role": "gate",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_release_necessary_not_sufficient_multi_regulon"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "trm_like_program_by_model",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_absent"
                    ],
                    "canonical": "niche_absent"
                  }
                }
              }
            ]
          },
          {
            "name": "cf_regulator_activator_or_wrong_node",
            "role": "hard_fail",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "RUNX3",
              "Runx3",
              "PRDM1",
              "Prdm1",
              "BLIMP1",
              "Blimp1",
              "BHLHE40",
              "Bhlhe40",
              "ZNF683",
              "Zfp683",
              "HOBIT",
              "Hobit",
              "AHR",
              "Ahr",
              "BATF",
              "Batf",
              "TOX",
              "Tox",
              "NR4A1",
              "Nr4a1",
              "NR4A2",
              "Nr4a2",
              "NR4A3",
              "Nr4a3",
              "TCF7",
              "Tcf7",
              "TCF1",
              "Tcf1",
              "BACH2",
              "Bach2",
              "TBX21",
              "Tbx21",
              "TBET",
              "Tbet",
              "FLI1",
              "Fli1",
              "NR3C1",
              "Nr3c1",
              "EOMES",
              "Eomes",
              "ZEB2",
              "Zeb2",
              "ID2",
              "Id2",
              "ID3",
              "Id3",
              "IRF4",
              "Irf4"
            ]
          },
          {
            "name": "cf_node_character_hidden_driver_recall",
            "role": "hard_fail",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "hidden_driver_activity_not_expression"
          },
          {
            "name": "cf_effect_relation_hidden_pattern",
            "role": "hard_fail",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "activity_exceeds_expression"
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "causal_status",
            "kind": "value",
            "op": "in",
            "expected": [
              "sufficiency_established",
              "necessity_established",
              "causal_established_in_dataset",
              "sufficiency_via_perturbation_in_dataset"
            ]
          },
          {
            "name": "cf_residency_logic_activator_prior",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "residency_activator_induction_primary"
          },
          {
            "name": "cf_residency_logic_single_regulon",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_controls_entire_residency_program"
          },
          {
            "name": "cf_decoy_called_target",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_distractor_model_reported",
            "role": "hard_fail",
            "field": "$.trm_like_program_by_model[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_chromatin_only_decoy",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          }
        ]
      }
    },
    {
      "modelId": "sclong-kimi-k3-pi",
      "modelName": "kimi-k3",
      "harness": "pi",
      "task": "green_klf2_hidden_repressor_of_trm_like_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 2907,
      "cost": 2.2970322,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "regulator_gate",
            "role": "gate",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "KLF2",
              "Klf2"
            ]
          },
          {
            "name": "controlling_node_character_gate",
            "role": "gate",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "high_differential_expression"
          },
          {
            "name": "expression_vs_activity_gate",
            "role": "gate",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "expression_exceeds_activity"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "target_program_partition",
            "kind": "list",
            "matchKey": "gene",
            "k": 7,
            "expected": [
              {
                "key": "S1pr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "S1pr4",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Klrg1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Cx3cr1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "raised"
                    ],
                    "canonical": "raised"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "egress"
                    ],
                    "canonical": "egress"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Itgae",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Hic1",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "P2rx7",
                "fields": {
                  "direction": {
                    "op": "equals",
                    "values": [
                      "repressed"
                    ],
                    "canonical": "repressed"
                  },
                  "program": {
                    "op": "equals",
                    "values": [
                      "residency"
                    ],
                    "canonical": "residency"
                  },
                  "cross_modal": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              }
            ]
          },
          {
            "name": "causal_status_gate",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "association_only_sufficiency_requires_perturbation"
          },
          {
            "name": "residency_control_logic_gate",
            "role": "gate",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_release_necessary_not_sufficient_multi_regulon"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "trm_like_program_by_model",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "carries"
                    ],
                    "canonical": "carries"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_permissive"
                    ],
                    "canonical": "niche_permissive"
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "trm_like_status": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ],
                    "canonical": "absent"
                  },
                  "niche_status": {
                    "op": "equals",
                    "values": [
                      "niche_absent"
                    ],
                    "canonical": "niche_absent"
                  }
                }
              }
            ]
          },
          {
            "name": "cf_regulator_activator_or_wrong_node",
            "role": "hard_fail",
            "field": "regulator",
            "kind": "value",
            "op": "in",
            "expected": [
              "RUNX3",
              "Runx3",
              "PRDM1",
              "Prdm1",
              "BLIMP1",
              "Blimp1",
              "BHLHE40",
              "Bhlhe40",
              "ZNF683",
              "Zfp683",
              "HOBIT",
              "Hobit",
              "AHR",
              "Ahr",
              "BATF",
              "Batf",
              "TOX",
              "Tox",
              "NR4A1",
              "Nr4a1",
              "NR4A2",
              "Nr4a2",
              "NR4A3",
              "Nr4a3",
              "TCF7",
              "Tcf7",
              "TCF1",
              "Tcf1",
              "BACH2",
              "Bach2",
              "TBX21",
              "Tbx21",
              "TBET",
              "Tbet",
              "FLI1",
              "Fli1",
              "NR3C1",
              "Nr3c1",
              "EOMES",
              "Eomes",
              "ZEB2",
              "Zeb2",
              "ID2",
              "Id2",
              "ID3",
              "Id3",
              "IRF4",
              "Irf4"
            ]
          },
          {
            "name": "cf_node_character_hidden_driver_recall",
            "role": "hard_fail",
            "field": "controlling_node_character",
            "kind": "value",
            "expected": "hidden_driver_activity_not_expression"
          },
          {
            "name": "cf_effect_relation_hidden_pattern",
            "role": "hard_fail",
            "field": "expression_vs_activity_effect",
            "kind": "value",
            "expected": "activity_exceeds_expression"
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "causal_status",
            "kind": "value",
            "op": "in",
            "expected": [
              "sufficiency_established",
              "necessity_established",
              "causal_established_in_dataset",
              "sufficiency_via_perturbation_in_dataset"
            ]
          },
          {
            "name": "cf_residency_logic_activator_prior",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "residency_activator_induction_primary"
          },
          {
            "name": "cf_residency_logic_single_regulon",
            "role": "hard_fail",
            "field": "residency_control_logic",
            "kind": "value",
            "expected": "brake_controls_entire_residency_program"
          },
          {
            "name": "cf_decoy_called_target",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_distractor_model_reported",
            "role": "hard_fail",
            "field": "$.trm_like_program_by_model[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_chromatin_only_decoy",
            "role": "hard_fail",
            "field": "$.target_program_partition[*]",
            "kind": "avoid",
            "expected": []
          }
        ]
      }
    },
    {
      "modelId": "sclong-kimi-k3-pi",
      "modelName": "kimi-k3",
      "harness": "pi",
      "task": "green_shared_vs_distinct_regulators_trm_vs_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 1,
      "passed": true,
      "score": null,
      "durationS": null,
      "cost": null,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_program_map",
            "kind": "list",
            "matchKey": "regulator",
            "k": 3,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "both"
                    ],
                    "canonical": "both"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "represses"
                    ],
                    "canonical": "represses"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Batf",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "trm_like_til_specific"
                    ],
                    "canonical": "trm_like_til_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "chromatin_over_rna"
                    ],
                    "canonical": "chromatin_over_rna"
                  }
                }
              },
              {
                "key": "Ahr",
                "fields": {
                  "program_specificity": {
                    "op": "in",
                    "values": [
                      "trm_specific",
                      "both"
                    ],
                    "canonical": "trm_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "rna_over_chromatin"
                    ],
                    "canonical": "rna_over_chromatin"
                  }
                }
              }
            ]
          },
          {
            "name": "regulator_set_no_extras_gate",
            "role": "gate",
            "field": "$.regulator_program_map[*]",
            "kind": "other",
            "expected": null
          },
          {
            "name": "program_identity_conclusion_gate",
            "role": "gate",
            "field": "program_identity_conclusion",
            "kind": "value",
            "expected": "distinct_state_shared_brake_divergent_activators"
          },
          {
            "name": "residency_install_logic_gate",
            "role": "gate",
            "field": "residency_install_logic",
            "kind": "value",
            "expected": "brake_release_permits_context_matched_activator"
          },
          {
            "name": "activator_context_dependence_gate",
            "role": "gate",
            "field": "activator_context_dependence",
            "kind": "value",
            "expected": "context_matched_crossover"
          },
          {
            "name": "evidence_ceiling_gate",
            "role": "gate",
            "field": "evidence_ceiling",
            "kind": "value",
            "expected": "specificity_dissociations_require_perturbation"
          },
          {
            "name": "cf_install_logic_flat_or_single_step",
            "role": "hard_fail",
            "field": "residency_install_logic",
            "kind": "value",
            "op": "in",
            "expected": [
              "flat_shared_residency_toolkit",
              "activator_induction_sufficient",
              "brake_release_alone_sufficient"
            ]
          },
          {
            "name": "cf_program_identity_lumped_or_wrong",
            "role": "hard_fail",
            "field": "program_identity_conclusion",
            "kind": "value",
            "op": "in",
            "expected": [
              "trm_like_til_is_tumor_trm",
              "trm_like_til_is_exhausted_subset",
              "trm_like_til_is_effector_state",
              "trm_like_til_is_transitional_intermediate",
              "trm_like_til_is_shared_program_distinct_niche"
            ]
          },
          {
            "name": "cf_ahr_assigned_to_til",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Ahr",
            "valueField": "program_specificity",
            "expected": [
              "trm_like_til_specific"
            ]
          },
          {
            "name": "cf_batf_assigned_to_trm_or_shared",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Batf",
            "valueField": "program_specificity",
            "expected": [
              "trm_specific",
              "both"
            ]
          },
          {
            "name": "cf_klf2_called_activator",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Klf2",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          },
          {
            "name": "cf_evidence_ceiling_overclaim",
            "role": "hard_fail",
            "field": "evidence_ceiling",
            "kind": "value",
            "op": "in",
            "expected": [
              "specificity_established_from_regulon_activity",
              "specificity_established_in_dataset",
              "causation_established_by_cross_sectional_concordance",
              "necessity_and_sufficiency_established"
            ]
          },
          {
            "name": "cf_activator_context_shared_or_single",
            "role": "hard_fail",
            "field": "activator_context_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "shared_activators_both_contexts",
              "single_global_activator",
              "redundant_activators"
            ]
          },
          {
            "name": "cf_single_modality_rule",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_hic1_motif_brake_mislabel",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_runx3_anticanonical_as_controller",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Runx3",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-kimi-k3-pi",
      "modelName": "kimi-k3",
      "harness": "pi",
      "task": "green_shared_vs_distinct_regulators_trm_vs_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 2139,
      "cost": 1.7241546000000003,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_program_map",
            "kind": "list",
            "matchKey": "regulator",
            "k": 3,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "both"
                    ],
                    "canonical": "both"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "represses"
                    ],
                    "canonical": "represses"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Batf",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "trm_like_til_specific"
                    ],
                    "canonical": "trm_like_til_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "chromatin_over_rna"
                    ],
                    "canonical": "chromatin_over_rna"
                  }
                }
              },
              {
                "key": "Ahr",
                "fields": {
                  "program_specificity": {
                    "op": "in",
                    "values": [
                      "trm_specific",
                      "both"
                    ],
                    "canonical": "trm_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "rna_over_chromatin"
                    ],
                    "canonical": "rna_over_chromatin"
                  }
                }
              }
            ]
          },
          {
            "name": "regulator_set_no_extras_gate",
            "role": "gate",
            "field": "$.regulator_program_map[*]",
            "kind": "other",
            "expected": null
          },
          {
            "name": "program_identity_conclusion_gate",
            "role": "gate",
            "field": "program_identity_conclusion",
            "kind": "value",
            "expected": "distinct_state_shared_brake_divergent_activators"
          },
          {
            "name": "residency_install_logic_gate",
            "role": "gate",
            "field": "residency_install_logic",
            "kind": "value",
            "expected": "brake_release_permits_context_matched_activator"
          },
          {
            "name": "activator_context_dependence_gate",
            "role": "gate",
            "field": "activator_context_dependence",
            "kind": "value",
            "expected": "context_matched_crossover"
          },
          {
            "name": "evidence_ceiling_gate",
            "role": "gate",
            "field": "evidence_ceiling",
            "kind": "value",
            "expected": "specificity_dissociations_require_perturbation"
          },
          {
            "name": "cf_install_logic_flat_or_single_step",
            "role": "hard_fail",
            "field": "residency_install_logic",
            "kind": "value",
            "op": "in",
            "expected": [
              "flat_shared_residency_toolkit",
              "activator_induction_sufficient",
              "brake_release_alone_sufficient"
            ]
          },
          {
            "name": "cf_program_identity_lumped_or_wrong",
            "role": "hard_fail",
            "field": "program_identity_conclusion",
            "kind": "value",
            "op": "in",
            "expected": [
              "trm_like_til_is_tumor_trm",
              "trm_like_til_is_exhausted_subset",
              "trm_like_til_is_effector_state",
              "trm_like_til_is_transitional_intermediate",
              "trm_like_til_is_shared_program_distinct_niche"
            ]
          },
          {
            "name": "cf_ahr_assigned_to_til",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Ahr",
            "valueField": "program_specificity",
            "expected": [
              "trm_like_til_specific"
            ]
          },
          {
            "name": "cf_batf_assigned_to_trm_or_shared",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Batf",
            "valueField": "program_specificity",
            "expected": [
              "trm_specific",
              "both"
            ]
          },
          {
            "name": "cf_klf2_called_activator",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Klf2",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          },
          {
            "name": "cf_evidence_ceiling_overclaim",
            "role": "hard_fail",
            "field": "evidence_ceiling",
            "kind": "value",
            "op": "in",
            "expected": [
              "specificity_established_from_regulon_activity",
              "specificity_established_in_dataset",
              "causation_established_by_cross_sectional_concordance",
              "necessity_and_sufficiency_established"
            ]
          },
          {
            "name": "cf_activator_context_shared_or_single",
            "role": "hard_fail",
            "field": "activator_context_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "shared_activators_both_contexts",
              "single_global_activator",
              "redundant_activators"
            ]
          },
          {
            "name": "cf_single_modality_rule",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_hic1_motif_brake_mislabel",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_runx3_anticanonical_as_controller",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Runx3",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-kimi-k3-pi",
      "modelName": "kimi-k3",
      "harness": "pi",
      "task": "green_shared_vs_distinct_regulators_trm_vs_til",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 12463,
      "cost": 5.783143800000001,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_program_map",
            "kind": "list",
            "matchKey": "regulator",
            "k": 3,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "both"
                    ],
                    "canonical": "both"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "represses"
                    ],
                    "canonical": "represses"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ],
                    "canonical": "concordant"
                  }
                }
              },
              {
                "key": "Batf",
                "fields": {
                  "program_specificity": {
                    "op": "equals",
                    "values": [
                      "trm_like_til_specific"
                    ],
                    "canonical": "trm_like_til_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "chromatin_over_rna"
                    ],
                    "canonical": "chromatin_over_rna"
                  }
                }
              },
              {
                "key": "Ahr",
                "fields": {
                  "program_specificity": {
                    "op": "in",
                    "values": [
                      "trm_specific",
                      "both"
                    ],
                    "canonical": "trm_specific"
                  },
                  "direction": {
                    "op": "equals",
                    "values": [
                      "augments"
                    ],
                    "canonical": "augments"
                  },
                  "decisive_modality": {
                    "op": "equals",
                    "values": [
                      "rna_over_chromatin"
                    ],
                    "canonical": "rna_over_chromatin"
                  }
                }
              }
            ]
          },
          {
            "name": "regulator_set_no_extras_gate",
            "role": "gate",
            "field": "$.regulator_program_map[*]",
            "kind": "other",
            "expected": null
          },
          {
            "name": "program_identity_conclusion_gate",
            "role": "gate",
            "field": "program_identity_conclusion",
            "kind": "value",
            "expected": "distinct_state_shared_brake_divergent_activators"
          },
          {
            "name": "residency_install_logic_gate",
            "role": "gate",
            "field": "residency_install_logic",
            "kind": "value",
            "expected": "brake_release_permits_context_matched_activator"
          },
          {
            "name": "activator_context_dependence_gate",
            "role": "gate",
            "field": "activator_context_dependence",
            "kind": "value",
            "expected": "context_matched_crossover"
          },
          {
            "name": "evidence_ceiling_gate",
            "role": "gate",
            "field": "evidence_ceiling",
            "kind": "value",
            "expected": "specificity_dissociations_require_perturbation"
          },
          {
            "name": "cf_install_logic_flat_or_single_step",
            "role": "hard_fail",
            "field": "residency_install_logic",
            "kind": "value",
            "op": "in",
            "expected": [
              "flat_shared_residency_toolkit",
              "activator_induction_sufficient",
              "brake_release_alone_sufficient"
            ]
          },
          {
            "name": "cf_program_identity_lumped_or_wrong",
            "role": "hard_fail",
            "field": "program_identity_conclusion",
            "kind": "value",
            "op": "in",
            "expected": [
              "trm_like_til_is_tumor_trm",
              "trm_like_til_is_exhausted_subset",
              "trm_like_til_is_effector_state",
              "trm_like_til_is_transitional_intermediate",
              "trm_like_til_is_shared_program_distinct_niche"
            ]
          },
          {
            "name": "cf_ahr_assigned_to_til",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Ahr",
            "valueField": "program_specificity",
            "expected": [
              "trm_like_til_specific"
            ]
          },
          {
            "name": "cf_batf_assigned_to_trm_or_shared",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Batf",
            "valueField": "program_specificity",
            "expected": [
              "trm_specific",
              "both"
            ]
          },
          {
            "name": "cf_klf2_called_activator",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Klf2",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          },
          {
            "name": "cf_evidence_ceiling_overclaim",
            "role": "hard_fail",
            "field": "evidence_ceiling",
            "kind": "value",
            "op": "in",
            "expected": [
              "specificity_established_from_regulon_activity",
              "specificity_established_in_dataset",
              "causation_established_by_cross_sectional_concordance",
              "necessity_and_sufficiency_established"
            ]
          },
          {
            "name": "cf_activator_context_shared_or_single",
            "role": "hard_fail",
            "field": "activator_context_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "shared_activators_both_contexts",
              "single_global_activator",
              "redundant_activators"
            ]
          },
          {
            "name": "cf_single_modality_rule",
            "role": "hard_fail",
            "field": null,
            "kind": "other",
            "expected": null
          },
          {
            "name": "cf_hic1_motif_brake_mislabel",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "cf_runx3_anticanonical_as_controller",
            "role": "hard_fail",
            "field": "$.regulator_program_map[*]",
            "kind": "contains",
            "matchKey": "regulator",
            "entity": "Runx3",
            "valueField": "direction",
            "expected": [
              "augments"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-kimi-k3-pi",
      "modelName": "kimi-k3",
      "harness": "pi",
      "task": "gse297023_q2_senescence_not_mechanism",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 1,
      "passed": true,
      "score": null,
      "durationS": 1625,
      "cost": 0.5642855999999998,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "candidate_is_senescence",
            "role": "gate",
            "field": "$.candidate_mechanism",
            "kind": "value",
            "op": "in",
            "expected": [
              "cellular_senescence",
              "oncogene_induced_senescence",
              "replicative_senescence"
            ]
          },
          {
            "name": "arrest_core_not_engaged",
            "role": "gate",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_not_engaged"
            ]
          },
          {
            "name": "secretory_arm_is_inflammaging",
            "role": "gate",
            "field": "$.secretory_arm_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "acute_phase_inflammaging_response"
            ]
          },
          {
            "name": "secretory_arm_broadly_elevated",
            "role": "gate",
            "field": "$.secretory_arm_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "broadly_elevated"
            ]
          },
          {
            "name": "carries_aging_program",
            "role": "gate",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "carries_bona_fide_aging_program_distinct_from_nominated"
            ]
          },
          {
            "name": "conclusion_senescence_unlikely",
            "role": "gate",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "unlikely_major_contributor"
            ]
          },
          {
            "name": "cf_arrest_core_engaged_recall",
            "role": "hard_fail",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_engaged"
            ]
          },
          {
            "name": "cf_aging_program_conflation_or_explain_away",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "aging_program_indistinguishable_from_nominated_program",
              "aging_program_is_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_no_aging_program_sensitivity_failure",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_detectable_aging_program"
            ]
          },
          {
            "name": "cf_senescence_driver_conclusion",
            "role": "hard_fail",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "likely_primary_driver",
              "plausible_supported_contributor"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-kimi-k3-pi",
      "modelName": "kimi-k3",
      "harness": "pi",
      "task": "gse297023_q2_senescence_not_mechanism",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 1661,
      "cost": 0.4957374,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "candidate_is_senescence",
            "role": "gate",
            "field": "$.candidate_mechanism",
            "kind": "value",
            "op": "in",
            "expected": [
              "cellular_senescence",
              "oncogene_induced_senescence",
              "replicative_senescence"
            ]
          },
          {
            "name": "arrest_core_not_engaged",
            "role": "gate",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_not_engaged"
            ]
          },
          {
            "name": "secretory_arm_is_inflammaging",
            "role": "gate",
            "field": "$.secretory_arm_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "acute_phase_inflammaging_response"
            ]
          },
          {
            "name": "secretory_arm_broadly_elevated",
            "role": "gate",
            "field": "$.secretory_arm_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "broadly_elevated"
            ]
          },
          {
            "name": "carries_aging_program",
            "role": "gate",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "carries_bona_fide_aging_program_distinct_from_nominated"
            ]
          },
          {
            "name": "conclusion_senescence_unlikely",
            "role": "gate",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "unlikely_major_contributor"
            ]
          },
          {
            "name": "cf_arrest_core_engaged_recall",
            "role": "hard_fail",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_engaged"
            ]
          },
          {
            "name": "cf_aging_program_conflation_or_explain_away",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "aging_program_indistinguishable_from_nominated_program",
              "aging_program_is_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_no_aging_program_sensitivity_failure",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_detectable_aging_program"
            ]
          },
          {
            "name": "cf_senescence_driver_conclusion",
            "role": "hard_fail",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "likely_primary_driver",
              "plausible_supported_contributor"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-kimi-k3-pi",
      "modelName": "kimi-k3",
      "harness": "pi",
      "task": "gse297023_q2_senescence_not_mechanism",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 798,
      "cost": 0.5657483999999999,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "candidate_is_senescence",
            "role": "gate",
            "field": "$.candidate_mechanism",
            "kind": "value",
            "op": "in",
            "expected": [
              "cellular_senescence",
              "oncogene_induced_senescence",
              "replicative_senescence"
            ]
          },
          {
            "name": "arrest_core_not_engaged",
            "role": "gate",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_not_engaged"
            ]
          },
          {
            "name": "secretory_arm_is_inflammaging",
            "role": "gate",
            "field": "$.secretory_arm_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "acute_phase_inflammaging_response"
            ]
          },
          {
            "name": "secretory_arm_broadly_elevated",
            "role": "gate",
            "field": "$.secretory_arm_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "broadly_elevated"
            ]
          },
          {
            "name": "carries_aging_program",
            "role": "gate",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "carries_bona_fide_aging_program_distinct_from_nominated"
            ]
          },
          {
            "name": "conclusion_senescence_unlikely",
            "role": "gate",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "unlikely_major_contributor"
            ]
          },
          {
            "name": "cf_arrest_core_engaged_recall",
            "role": "hard_fail",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_engaged"
            ]
          },
          {
            "name": "cf_aging_program_conflation_or_explain_away",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "aging_program_indistinguishable_from_nominated_program",
              "aging_program_is_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_no_aging_program_sensitivity_failure",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_detectable_aging_program"
            ]
          },
          {
            "name": "cf_senescence_driver_conclusion",
            "role": "hard_fail",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "likely_primary_driver",
              "plausible_supported_contributor"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-kimi-k3-pi",
      "modelName": "kimi-k3",
      "harness": "pi",
      "task": "gse297023_q3_age_retained_mapk_reduced",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": 2680,
      "cost": 1.0020258000000004,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "age_retained",
            "role": "gate",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_retained"
            ]
          },
          {
            "name": "identity_is_aging",
            "role": "gate",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_aging_signatures"
            ]
          },
          {
            "name": "nature_state_shift",
            "role": "gate",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_state_expression_shift"
            ]
          },
          {
            "name": "pathway_is_mapk",
            "role": "gate",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "mapk"
            ]
          },
          {
            "name": "pathway_reduced_in_aged",
            "role": "gate",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "reduced_in_aged"
            ]
          },
          {
            "name": "cf_mapk_increased_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_in_aged"
            ]
          },
          {
            "name": "cf_age_erased",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_erased"
            ]
          },
          {
            "name": "cf_pervasive_overcall",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_pervasive_artifactual"
            ]
          },
          {
            "name": "cf_explain_away_aging",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_compositional_or_identity",
            "role": "hard_fail",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_proportion_shift",
              "cell_identity_shift",
              "composition_dominant_state_secondary",
              "both_state_and_composition_equally"
            ]
          },
          {
            "name": "cf_biggest_inflammatory_signal_not_mapk",
            "role": "hard_fail",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "jak_stat",
              "nfkb",
              "tnfa"
            ]
          },
          {
            "name": "cf_senescence_misattribution",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_senescence_signature_specifically"
            ]
          },
          {
            "name": "cf_dismiss_as_batch_artifact",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_batch_or_technical_artifact"
            ]
          },
          {
            "name": "cf_mapk_comparable_constitutive_kras_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "comparable_across_ages"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-kimi-k3-pi",
      "modelName": "kimi-k3",
      "harness": "pi",
      "task": "gse297023_q3_age_retained_mapk_reduced",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 5246,
      "cost": 2.189898,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "age_retained",
            "role": "gate",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_retained"
            ]
          },
          {
            "name": "identity_is_aging",
            "role": "gate",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_aging_signatures"
            ]
          },
          {
            "name": "nature_state_shift",
            "role": "gate",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_state_expression_shift"
            ]
          },
          {
            "name": "pathway_is_mapk",
            "role": "gate",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "mapk"
            ]
          },
          {
            "name": "pathway_reduced_in_aged",
            "role": "gate",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "reduced_in_aged"
            ]
          },
          {
            "name": "cf_mapk_increased_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_in_aged"
            ]
          },
          {
            "name": "cf_age_erased",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_erased"
            ]
          },
          {
            "name": "cf_pervasive_overcall",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_pervasive_artifactual"
            ]
          },
          {
            "name": "cf_explain_away_aging",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_compositional_or_identity",
            "role": "hard_fail",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_proportion_shift",
              "cell_identity_shift",
              "composition_dominant_state_secondary",
              "both_state_and_composition_equally"
            ]
          },
          {
            "name": "cf_biggest_inflammatory_signal_not_mapk",
            "role": "hard_fail",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "jak_stat",
              "nfkb",
              "tnfa"
            ]
          },
          {
            "name": "cf_senescence_misattribution",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_senescence_signature_specifically"
            ]
          },
          {
            "name": "cf_dismiss_as_batch_artifact",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_batch_or_technical_artifact"
            ]
          },
          {
            "name": "cf_mapk_comparable_constitutive_kras_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "comparable_across_ages"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-kimi-k3-pi",
      "modelName": "kimi-k3",
      "harness": "pi",
      "task": "gse297023_q3_age_retained_mapk_reduced",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 4469,
      "cost": 1.9646405999999996,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "age_retained",
            "role": "gate",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_retained"
            ]
          },
          {
            "name": "identity_is_aging",
            "role": "gate",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_known_aging_signatures"
            ]
          },
          {
            "name": "nature_state_shift",
            "role": "gate",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_state_expression_shift"
            ]
          },
          {
            "name": "pathway_is_mapk",
            "role": "gate",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "mapk"
            ]
          },
          {
            "name": "pathway_reduced_in_aged",
            "role": "gate",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "reduced_in_aged"
            ]
          },
          {
            "name": "cf_mapk_increased_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_in_aged"
            ]
          },
          {
            "name": "cf_age_erased",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_erased"
            ]
          },
          {
            "name": "cf_pervasive_overcall",
            "role": "hard_fail",
            "field": "$.age_signature_persistence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_program_pervasive_artifactual"
            ]
          },
          {
            "name": "cf_explain_away_aging",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_compositional_or_identity",
            "role": "hard_fail",
            "field": "$.age_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_proportion_shift",
              "cell_identity_shift",
              "composition_dominant_state_secondary",
              "both_state_and_composition_equally"
            ]
          },
          {
            "name": "cf_biggest_inflammatory_signal_not_mapk",
            "role": "hard_fail",
            "field": "$.most_altered_oncogenic_pathway",
            "kind": "value",
            "op": "in",
            "expected": [
              "jak_stat",
              "nfkb",
              "tnfa"
            ]
          },
          {
            "name": "cf_senescence_misattribution",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "matches_senescence_signature_specifically"
            ]
          },
          {
            "name": "cf_dismiss_as_batch_artifact",
            "role": "hard_fail",
            "field": "$.age_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "reflects_batch_or_technical_artifact"
            ]
          },
          {
            "name": "cf_mapk_comparable_constitutive_kras_prior",
            "role": "hard_fail",
            "field": "$.pathway_activity_direction_aged_vs_young",
            "kind": "value",
            "op": "in",
            "expected": [
              "comparable_across_ages"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-kimi-k3-pi",
      "modelName": "kimi-k3",
      "harness": "pi",
      "task": "gse297023_q5_tme_age_dependent_remodeling",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 1,
      "passed": false,
      "score": null,
      "durationS": null,
      "cost": null,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "perturbation_is_pten",
            "role": "gate",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "pten"
            ]
          },
          {
            "name": "tme_non_cell_autonomous",
            "role": "gate",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "substantial_non_cell_autonomous_remodeling"
            ]
          },
          {
            "name": "tme_state_not_composition",
            "role": "gate",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "tme_age_dependent",
            "role": "gate",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_dependent"
            ]
          },
          {
            "name": "tme_response_broader_in_aged",
            "role": "gate",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_aged"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "most_responsive_noncancer_cell_types",
            "kind": "list",
            "matchKey": "cell_type",
            "k": 2,
            "expected": [
              {
                "key": "alveolar_macrophage",
                "fields": {}
              },
              {
                "key": "endothelial",
                "fields": {}
              }
            ]
          },
          {
            "name": "cf_cell_intrinsic_only",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "negligible_cell_intrinsic_only_effect",
              "remodeling_unresolved",
              "modest_remodeling_few_cell_types"
            ]
          },
          {
            "name": "cf_remodeling_is_contamination",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "apparent_remodeling_reflects_cancer_contamination"
            ]
          },
          {
            "name": "cf_perturbation_design_misread",
            "role": "hard_fail",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "multiple_tumor_suppressors_pooled",
              "no_additional_gene_difference_kras_only",
              "rb1_cdkn2a_double",
              "pcna_essential_gene_control",
              "difference_is_oncogene_not_suppressor"
            ]
          },
          {
            "name": "cf_age_independent_tme",
            "role": "hard_fail",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_independent"
            ]
          },
          {
            "name": "cf_compositional_shift",
            "role": "hard_fail",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_frequency_shift",
              "compositional_immune_infiltration",
              "composition_dominant_state_secondary",
              "both_state_and_composition"
            ]
          },
          {
            "name": "cf_epithelial_or_cancer_substitution",
            "role": "hard_fail",
            "field": "$.most_responsive_noncancer_cell_types[*].cell_type",
            "kind": "avoid",
            "expected": [
              "AT2_cell",
              "AT1_cell",
              "cancer_cell",
              "ciliated",
              "secretory",
              "club_cell",
              "basal_cell",
              "goblet_cell"
            ]
          },
          {
            "name": "cf_tme_breadth_broader_in_young",
            "role": "hard_fail",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_young"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-kimi-k3-pi",
      "modelName": "kimi-k3",
      "harness": "pi",
      "task": "gse297023_q5_tme_age_dependent_remodeling",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": null,
      "cost": null,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "perturbation_is_pten",
            "role": "gate",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "pten"
            ]
          },
          {
            "name": "tme_non_cell_autonomous",
            "role": "gate",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "substantial_non_cell_autonomous_remodeling"
            ]
          },
          {
            "name": "tme_state_not_composition",
            "role": "gate",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "tme_age_dependent",
            "role": "gate",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_dependent"
            ]
          },
          {
            "name": "tme_response_broader_in_aged",
            "role": "gate",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_aged"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "most_responsive_noncancer_cell_types",
            "kind": "list",
            "matchKey": "cell_type",
            "k": 2,
            "expected": [
              {
                "key": "alveolar_macrophage",
                "fields": {}
              },
              {
                "key": "endothelial",
                "fields": {}
              }
            ]
          },
          {
            "name": "cf_cell_intrinsic_only",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "negligible_cell_intrinsic_only_effect",
              "remodeling_unresolved",
              "modest_remodeling_few_cell_types"
            ]
          },
          {
            "name": "cf_remodeling_is_contamination",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "apparent_remodeling_reflects_cancer_contamination"
            ]
          },
          {
            "name": "cf_perturbation_design_misread",
            "role": "hard_fail",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "multiple_tumor_suppressors_pooled",
              "no_additional_gene_difference_kras_only",
              "rb1_cdkn2a_double",
              "pcna_essential_gene_control",
              "difference_is_oncogene_not_suppressor"
            ]
          },
          {
            "name": "cf_age_independent_tme",
            "role": "hard_fail",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_independent"
            ]
          },
          {
            "name": "cf_compositional_shift",
            "role": "hard_fail",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_frequency_shift",
              "compositional_immune_infiltration",
              "composition_dominant_state_secondary",
              "both_state_and_composition"
            ]
          },
          {
            "name": "cf_epithelial_or_cancer_substitution",
            "role": "hard_fail",
            "field": "$.most_responsive_noncancer_cell_types[*].cell_type",
            "kind": "avoid",
            "expected": [
              "AT2_cell",
              "AT1_cell",
              "cancer_cell",
              "ciliated",
              "secretory",
              "club_cell",
              "basal_cell",
              "goblet_cell"
            ]
          },
          {
            "name": "cf_tme_breadth_broader_in_young",
            "role": "hard_fail",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_young"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-kimi-k3-pi",
      "modelName": "kimi-k3",
      "harness": "pi",
      "task": "gse297023_q5_tme_age_dependent_remodeling",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 7644,
      "cost": 3.652248599999999,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "perturbation_is_pten",
            "role": "gate",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "pten"
            ]
          },
          {
            "name": "tme_non_cell_autonomous",
            "role": "gate",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "substantial_non_cell_autonomous_remodeling"
            ]
          },
          {
            "name": "tme_state_not_composition",
            "role": "gate",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "tme_age_dependent",
            "role": "gate",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_dependent"
            ]
          },
          {
            "name": "tme_response_broader_in_aged",
            "role": "gate",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_aged"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "most_responsive_noncancer_cell_types",
            "kind": "list",
            "matchKey": "cell_type",
            "k": 2,
            "expected": [
              {
                "key": "alveolar_macrophage",
                "fields": {}
              },
              {
                "key": "endothelial",
                "fields": {}
              }
            ]
          },
          {
            "name": "cf_cell_intrinsic_only",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "negligible_cell_intrinsic_only_effect",
              "remodeling_unresolved",
              "modest_remodeling_few_cell_types"
            ]
          },
          {
            "name": "cf_remodeling_is_contamination",
            "role": "hard_fail",
            "field": "$.noncancer_effect_extent",
            "kind": "value",
            "op": "in",
            "expected": [
              "apparent_remodeling_reflects_cancer_contamination"
            ]
          },
          {
            "name": "cf_perturbation_design_misread",
            "role": "hard_fail",
            "field": "$.cancer_genetic_difference_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "multiple_tumor_suppressors_pooled",
              "no_additional_gene_difference_kras_only",
              "rb1_cdkn2a_double",
              "pcna_essential_gene_control",
              "difference_is_oncogene_not_suppressor"
            ]
          },
          {
            "name": "cf_age_independent_tme",
            "role": "hard_fail",
            "field": "$.noncancer_effect_age_dependence",
            "kind": "value",
            "op": "in",
            "expected": [
              "age_independent"
            ]
          },
          {
            "name": "cf_compositional_shift",
            "role": "hard_fail",
            "field": "$.noncancer_effect_nature",
            "kind": "value",
            "op": "in",
            "expected": [
              "compositional_frequency_shift",
              "compositional_immune_infiltration",
              "composition_dominant_state_secondary",
              "both_state_and_composition"
            ]
          },
          {
            "name": "cf_epithelial_or_cancer_substitution",
            "role": "hard_fail",
            "field": "$.most_responsive_noncancer_cell_types[*].cell_type",
            "kind": "avoid",
            "expected": [
              "AT2_cell",
              "AT1_cell",
              "cancer_cell",
              "ciliated",
              "secretory",
              "club_cell",
              "basal_cell",
              "goblet_cell"
            ]
          },
          {
            "name": "cf_tme_breadth_broader_in_young",
            "role": "hard_fail",
            "field": "$.noncancer_effect_breadth_by_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "broader_in_young"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-kimi-k3-pi",
      "modelName": "kimi-k3",
      "harness": "pi",
      "task": "gse297023_q6_age_immune_infiltration",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 1,
      "passed": true,
      "score": null,
      "durationS": 3775,
      "cost": 1.7687616000000002,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "composition_stable_with_age",
            "role": "gate",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "frequencies_largely_stable"
            ]
          },
          {
            "name": "age_signal_is_state_not_composition",
            "role": "gate",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "frequencies_uninformative_for_infiltration",
            "role": "gate",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_cannot_establish_infiltration"
            ]
          },
          {
            "name": "infiltration_increases_with_age",
            "role": "gate",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_with_age"
            ]
          },
          {
            "name": "infiltration_strength_calibrated",
            "role": "gate",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "significant_increase_strongest_markers_only",
              "consistent_increase_not_statistically_robust"
            ]
          },
          {
            "name": "genotype_marker_fraction_lower_in_pten",
            "role": "gate",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "lower_in_pten_deficient"
            ]
          },
          {
            "name": "cf_composition_shift_reading",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "major_compositional_shift_with_age",
              "increased_immune_fraction_with_age"
            ]
          },
          {
            "name": "cf_lineage_specific_composition_claim",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "myeloid_expansion_with_age",
              "lymphoid_expansion_with_age",
              "minor_shifts_in_some_cell_types"
            ]
          },
          {
            "name": "cf_immunosenescence_depletion_prior",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "immune_depletion_with_age"
            ]
          },
          {
            "name": "cf_age_modality_compositional",
            "role": "hard_fail",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_type_composition_shift",
              "both_equally",
              "composition_dominant_state_secondary"
            ]
          },
          {
            "name": "cf_overread_frequencies_as_infiltration",
            "role": "hard_fail",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_shows_no_infiltration_change",
              "single_cell_quantifies_infiltration",
              "frequencies_show_increased_infiltration",
              "frequencies_show_decreased_infiltration"
            ]
          },
          {
            "name": "cf_infiltration_decreases_immunosenescence",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "decreased_with_age"
            ]
          },
          {
            "name": "cf_infiltration_unchanged_scrna_overread",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "unchanged_with_age"
            ]
          },
          {
            "name": "cf_infiltration_indeterminate_unused_node",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "direction_indeterminate_from_data"
            ]
          },
          {
            "name": "cf_infiltration_mixed_direction",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "mixed_by_immune_cell_type"
            ]
          },
          {
            "name": "cf_infiltration_genotype_confusion",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_only_in_pten_deficient",
              "decreased_only_in_pten_deficient"
            ]
          },
          {
            "name": "cf_pten_infiltration_paper_text_recall",
            "role": "hard_fail",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "higher_in_pten_deficient",
              "higher_in_pten_deficient_only_in_young",
              "higher_in_pten_deficient_only_in_aged"
            ]
          },
          {
            "name": "cf_pseudoreplication_overclaim",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_increase_across_markers"
            ]
          },
          {
            "name": "cf_infiltration_strength_decrease_direction",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_decrease_across_markers",
              "significant_decrease_strongest_markers_only",
              "consistent_decrease_not_statistically_robust"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-kimi-k3-pi",
      "modelName": "kimi-k3",
      "harness": "pi",
      "task": "gse297023_q6_age_immune_infiltration",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 2,
      "passed": false,
      "score": null,
      "durationS": 1391,
      "cost": 1.1584853999999998,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "composition_stable_with_age",
            "role": "gate",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "frequencies_largely_stable"
            ]
          },
          {
            "name": "age_signal_is_state_not_composition",
            "role": "gate",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "frequencies_uninformative_for_infiltration",
            "role": "gate",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_cannot_establish_infiltration"
            ]
          },
          {
            "name": "infiltration_increases_with_age",
            "role": "gate",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_with_age"
            ]
          },
          {
            "name": "infiltration_strength_calibrated",
            "role": "gate",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "significant_increase_strongest_markers_only",
              "consistent_increase_not_statistically_robust"
            ]
          },
          {
            "name": "genotype_marker_fraction_lower_in_pten",
            "role": "gate",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "lower_in_pten_deficient"
            ]
          },
          {
            "name": "cf_composition_shift_reading",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "major_compositional_shift_with_age",
              "increased_immune_fraction_with_age"
            ]
          },
          {
            "name": "cf_lineage_specific_composition_claim",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "myeloid_expansion_with_age",
              "lymphoid_expansion_with_age",
              "minor_shifts_in_some_cell_types"
            ]
          },
          {
            "name": "cf_immunosenescence_depletion_prior",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "immune_depletion_with_age"
            ]
          },
          {
            "name": "cf_age_modality_compositional",
            "role": "hard_fail",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_type_composition_shift",
              "both_equally",
              "composition_dominant_state_secondary"
            ]
          },
          {
            "name": "cf_overread_frequencies_as_infiltration",
            "role": "hard_fail",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_shows_no_infiltration_change",
              "single_cell_quantifies_infiltration",
              "frequencies_show_increased_infiltration",
              "frequencies_show_decreased_infiltration"
            ]
          },
          {
            "name": "cf_infiltration_decreases_immunosenescence",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "decreased_with_age"
            ]
          },
          {
            "name": "cf_infiltration_unchanged_scrna_overread",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "unchanged_with_age"
            ]
          },
          {
            "name": "cf_infiltration_indeterminate_unused_node",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "direction_indeterminate_from_data"
            ]
          },
          {
            "name": "cf_infiltration_mixed_direction",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "mixed_by_immune_cell_type"
            ]
          },
          {
            "name": "cf_infiltration_genotype_confusion",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_only_in_pten_deficient",
              "decreased_only_in_pten_deficient"
            ]
          },
          {
            "name": "cf_pten_infiltration_paper_text_recall",
            "role": "hard_fail",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "higher_in_pten_deficient",
              "higher_in_pten_deficient_only_in_young",
              "higher_in_pten_deficient_only_in_aged"
            ]
          },
          {
            "name": "cf_pseudoreplication_overclaim",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_increase_across_markers"
            ]
          },
          {
            "name": "cf_infiltration_strength_decrease_direction",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_decrease_across_markers",
              "significant_decrease_strongest_markers_only",
              "consistent_decrease_not_statistically_robust"
            ]
          }
        ]
      }
    },
    {
      "modelId": "sclong-kimi-k3-pi",
      "modelName": "kimi-k3",
      "harness": "pi",
      "task": "gse297023_q6_age_immune_infiltration",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 3,
      "passed": false,
      "score": null,
      "durationS": 4174,
      "cost": 0.9995483999999998,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "composition_stable_with_age",
            "role": "gate",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "frequencies_largely_stable"
            ]
          },
          {
            "name": "age_signal_is_state_not_composition",
            "role": "gate",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "transcriptional_state_change"
            ]
          },
          {
            "name": "frequencies_uninformative_for_infiltration",
            "role": "gate",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_cannot_establish_infiltration"
            ]
          },
          {
            "name": "infiltration_increases_with_age",
            "role": "gate",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_with_age"
            ]
          },
          {
            "name": "infiltration_strength_calibrated",
            "role": "gate",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "significant_increase_strongest_markers_only",
              "consistent_increase_not_statistically_robust"
            ]
          },
          {
            "name": "genotype_marker_fraction_lower_in_pten",
            "role": "gate",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "lower_in_pten_deficient"
            ]
          },
          {
            "name": "cf_composition_shift_reading",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "major_compositional_shift_with_age",
              "increased_immune_fraction_with_age"
            ]
          },
          {
            "name": "cf_lineage_specific_composition_claim",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "myeloid_expansion_with_age",
              "lymphoid_expansion_with_age",
              "minor_shifts_in_some_cell_types"
            ]
          },
          {
            "name": "cf_immunosenescence_depletion_prior",
            "role": "hard_fail",
            "field": "$.scrna_tme_composition_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "immune_depletion_with_age"
            ]
          },
          {
            "name": "cf_age_modality_compositional",
            "role": "hard_fail",
            "field": "$.dominant_tme_age_signal_modality",
            "kind": "value",
            "op": "in",
            "expected": [
              "cell_type_composition_shift",
              "both_equally",
              "composition_dominant_state_secondary"
            ]
          },
          {
            "name": "cf_overread_frequencies_as_infiltration",
            "role": "hard_fail",
            "field": "$.single_cell_infiltration_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "single_cell_shows_no_infiltration_change",
              "single_cell_quantifies_infiltration",
              "frequencies_show_increased_infiltration",
              "frequencies_show_decreased_infiltration"
            ]
          },
          {
            "name": "cf_infiltration_decreases_immunosenescence",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "decreased_with_age"
            ]
          },
          {
            "name": "cf_infiltration_unchanged_scrna_overread",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "unchanged_with_age"
            ]
          },
          {
            "name": "cf_infiltration_indeterminate_unused_node",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "direction_indeterminate_from_data"
            ]
          },
          {
            "name": "cf_infiltration_mixed_direction",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "mixed_by_immune_cell_type"
            ]
          },
          {
            "name": "cf_infiltration_genotype_confusion",
            "role": "hard_fail",
            "field": "$.tumor_immune_infiltration_direction_with_age",
            "kind": "value",
            "op": "in",
            "expected": [
              "increased_only_in_pten_deficient",
              "decreased_only_in_pten_deficient"
            ]
          },
          {
            "name": "cf_pten_infiltration_paper_text_recall",
            "role": "hard_fail",
            "field": "$.infiltration_marker_fraction_by_genotype",
            "kind": "value",
            "op": "in",
            "expected": [
              "higher_in_pten_deficient",
              "higher_in_pten_deficient_only_in_young",
              "higher_in_pten_deficient_only_in_aged"
            ]
          },
          {
            "name": "cf_pseudoreplication_overclaim",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_increase_across_markers"
            ]
          },
          {
            "name": "cf_infiltration_strength_decrease_direction",
            "role": "hard_fail",
            "field": "$.age_infiltration_signal_strength",
            "kind": "value",
            "op": "in",
            "expected": [
              "robust_highly_significant_decrease_across_markers",
              "significant_decrease_strongest_markers_only",
              "consistent_decrease_not_statistically_robust"
            ]
          }
        ]
      }
    }
  ],
  "bestRuns": [
    {
      "modelId": "sclong-gpt-5-6-sol-pi",
      "modelName": "gpt-5.6-sol",
      "harness": "pi",
      "provider": "OpenAI",
      "available": true,
      "task": "10x_5p_melanoma_tcr_matched_apc_vs_tumor_exhaustion",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 1,
      "passed": true,
      "score": null,
      "durationS": 525.270879,
      "cost": 2.1673857499999998,
      "nTurns": null,
      "reasoning": "",
      "metrics": {},
      "fieldScores": {},
      "graderChecks": [],
      "agentAnswer": null,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "state_discriminating_genes",
            "kind": "list",
            "matchKey": "gene",
            "k": 3,
            "expected": [
              {
                "key": "CTLA4",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TOX",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "ENTPD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "HAVCR2",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TIGIT",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "PDCD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "difference_attributable_to",
            "role": "gate",
            "field": "difference_attributable_to",
            "kind": "value",
            "expected": "covaries_with_apc_conjugate_state"
          },
          {
            "name": "causal_status",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "direction_unresolved_selection_confound"
          },
          {
            "name": "functional_interpretation",
            "role": "gate",
            "field": "functional_interpretation",
            "kind": "value",
            "expected": "costimulation_signature_function_unresolved"
          },
          {
            "name": "cf_contamination_gene_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "MLANA",
              "PMEL",
              "DCT",
              "TYR",
              "TYRP1",
              "SOX10",
              "MITF",
              "MAGEA4"
            ]
          },
          {
            "name": "cf_apc_partner_contamination_gene",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "HLA-DRA",
              "CD68",
              "LYZ",
              "CD14",
              "C1QA",
              "ITGAX",
              "CD1C"
            ]
          },
          {
            "name": "cf_ligand_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "CD274",
              "PDCD1LG2",
              "CD80",
              "CD86"
            ]
          },
          {
            "name": "cf_attribution_causal_or_baseline",
            "role": "hard_fail",
            "field": "$.difference_attributable_to",
            "kind": "avoid",
            "expected": [
              "apc_engagement",
              "tumor_cell_engagement",
              "clonotype_intrinsic_baseline"
            ]
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "engagement_induces_exhausted_state",
              "pre_exhausted_cells_select_into_apc_conjugates"
            ]
          },
          {
            "name": "cf_causal_fabrication",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "resolved_by_pseudotemporal_ordering",
              "resolved_by_perturbation_experiment"
            ]
          },
          {
            "name": "cf_functional_overclaim",
            "role": "hard_fail",
            "field": "$.functional_interpretation",
            "kind": "avoid",
            "expected": [
              "costimulation_licensed_reinvigoration_competent",
              "costimulation_licensed_checkpoint_responsive",
              "tumor_driven_terminal_exhaustion"
            ]
          }
        ]
      },
      "trajectoryAvailable": false
    },
    {
      "modelId": "sclong-gpt-5-6-sol-codex",
      "modelName": "gpt-5.6-sol",
      "harness": "openai-codex",
      "provider": "OpenAI",
      "available": true,
      "task": "gse171524_pathological_fibroblast_master_regulator_druggable_targets",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 2,
      "passed": true,
      "score": null,
      "durationS": 115.086224,
      "cost": 0,
      "nTurns": null,
      "reasoning": "",
      "metrics": {},
      "fieldScores": {},
      "graderChecks": [],
      "agentAnswer": null,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "disease_associated_fibroblast_subtype_weighted",
            "role": "additive",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "mr_tf_junb_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_alpha"
            ]
          },
          {
            "name": "mr_tf_jund_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_beta"
            ]
          },
          {
            "name": "drug_mmp14_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_alpha"
            ]
          },
          {
            "name": "drug_stat3_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_beta"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_weighted",
            "role": "additive",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "disease_associated_fibroblast_subtype_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "value",
            "op": "in",
            "expected": [
              "subtype_lambda",
              "subtype_epsilon",
              "subtype_iota",
              "subtype_beta",
              "subtype_eta",
              "subtype_zeta",
              "subtype_not_assessed"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "simple_de_ranking_by_log_fc",
              "marker_based_de_only",
              "not_computed"
            ]
          },
          {
            "name": "top_master_regulator_transcription_factors_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_not_computed"
            ]
          },
          {
            "name": "top_druggable_proteins_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_not_assessed"
            ]
          },
          {
            "name": "cross_axis_circular_marker_in_druggable_proteins",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_gamma",
              "drug_target_delta",
              "drug_target_eta"
            ]
          }
        ]
      },
      "trajectoryAvailable": false
    },
    {
      "modelId": "sclong-gpt-5-6-terra-pi",
      "modelName": "gpt-5.6-terra",
      "harness": "pi",
      "provider": "OpenAI",
      "available": true,
      "task": "10x_5p_melanoma_tcr_matched_apc_vs_tumor_exhaustion",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 2,
      "passed": true,
      "score": null,
      "durationS": 261.352442,
      "cost": 0.6585192499999999,
      "nTurns": null,
      "reasoning": "",
      "metrics": {},
      "fieldScores": {},
      "graderChecks": [],
      "agentAnswer": null,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "state_discriminating_genes",
            "kind": "list",
            "matchKey": "gene",
            "k": 3,
            "expected": [
              {
                "key": "CTLA4",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TOX",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "ENTPD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "HAVCR2",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TIGIT",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "PDCD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "difference_attributable_to",
            "role": "gate",
            "field": "difference_attributable_to",
            "kind": "value",
            "expected": "covaries_with_apc_conjugate_state"
          },
          {
            "name": "causal_status",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "direction_unresolved_selection_confound"
          },
          {
            "name": "functional_interpretation",
            "role": "gate",
            "field": "functional_interpretation",
            "kind": "value",
            "expected": "costimulation_signature_function_unresolved"
          },
          {
            "name": "cf_contamination_gene_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "MLANA",
              "PMEL",
              "DCT",
              "TYR",
              "TYRP1",
              "SOX10",
              "MITF",
              "MAGEA4"
            ]
          },
          {
            "name": "cf_apc_partner_contamination_gene",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "HLA-DRA",
              "CD68",
              "LYZ",
              "CD14",
              "C1QA",
              "ITGAX",
              "CD1C"
            ]
          },
          {
            "name": "cf_ligand_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "CD274",
              "PDCD1LG2",
              "CD80",
              "CD86"
            ]
          },
          {
            "name": "cf_attribution_causal_or_baseline",
            "role": "hard_fail",
            "field": "$.difference_attributable_to",
            "kind": "avoid",
            "expected": [
              "apc_engagement",
              "tumor_cell_engagement",
              "clonotype_intrinsic_baseline"
            ]
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "engagement_induces_exhausted_state",
              "pre_exhausted_cells_select_into_apc_conjugates"
            ]
          },
          {
            "name": "cf_causal_fabrication",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "resolved_by_pseudotemporal_ordering",
              "resolved_by_perturbation_experiment"
            ]
          },
          {
            "name": "cf_functional_overclaim",
            "role": "hard_fail",
            "field": "$.functional_interpretation",
            "kind": "avoid",
            "expected": [
              "costimulation_licensed_reinvigoration_competent",
              "costimulation_licensed_checkpoint_responsive",
              "tumor_driven_terminal_exhaustion"
            ]
          }
        ]
      },
      "trajectoryAvailable": false
    },
    {
      "modelId": "sclong-opus-4-8-claude-code",
      "modelName": "claude-opus-4-8",
      "harness": "claude-code",
      "provider": "Anthropic",
      "available": true,
      "task": "gse297023_q2_senescence_not_mechanism",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 2,
      "passed": true,
      "score": 7,
      "durationS": 1026.68,
      "cost": 2.5807397499999998,
      "nTurns": 25,
      "reasoning": "all_of [pass_rule=all]: PASS\n  + cohort_spans_young_and_aged: passed=True, score=1.0\n  + candidate_is_senescence: passed=True, score=1.0\n  + arrest_core_not_engaged: passed=True, score=1.0\n  + secretory_arm_is_inflammaging: passed=True, score=1.0\n  + secretory_arm_broadly_elevated: passed=True, score=1.0\n  + carries_aging_program: passed=True, score=1.0\n  + conclusion_senescence_unlikely: passed=True, score=1.0\n  + hard_fail 'cf_arrest_core_engaged_recall': not triggered\n  + hard_fail 'cf_aging_program_conflation_or_explain_away': not triggered\n  + hard_fail 'cf_no_aging_program_sensitivity_failure': not triggered\n  + hard_fail 'cf_senescence_driver_conclusion': not triggered",
      "metrics": {
        "pass_rule": "all",
        "scoring_count": 7,
        "scoring_passed": 7,
        "scoring_total_score": 7,
        "hard_fail_triggered": []
      },
      "fieldScores": {
        "cohort_spans_young_and_aged": 1,
        "candidate_is_senescence": 1,
        "arrest_core_not_engaged": 1,
        "secretory_arm_is_inflammaging": 1,
        "secretory_arm_broadly_elevated": 1,
        "carries_aging_program": 1,
        "conclusion_senescence_unlikely": 1
      },
      "graderChecks": [
        {
          "name": "cohort_spans_young_and_aged",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "candidate_is_senescence",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "arrest_core_not_engaged",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "secretory_arm_is_inflammaging",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "secretory_arm_broadly_elevated",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "carries_aging_program",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "conclusion_senescence_unlikely",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "cf_arrest_core_engaged_recall",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_aging_program_conflation_or_explain_away",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_no_aging_program_sensitivity_failure",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_senescence_driver_conclusion",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "age_associated_program_identity": "carries_bona_fide_aging_program_distinct_from_nominated",
        "arrest_core_engaged": "arrest_core_not_engaged",
        "candidate_mechanism": "cellular_senescence",
        "cohort_host_age_design": "young_and_aged_hosts_both_profiled",
        "nominated_mechanism_role": "unlikely_major_contributor",
        "secretory_arm_direction": "broadly_elevated",
        "secretory_arm_identity": "acute_phase_inflammaging_response"
      },
      "checkpointSignals": {
        "clustering": false,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": false,
        "nerveJointAnchor": false,
        "multiomeAssembly": false,
        "depthNormalizedAtac": false,
        "regulonInference": false,
        "cd8SubsetIntegration": true,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": false,
        "orthologProjection": false,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": true,
        "mousePseudobulkDesign": true,
        "cancerEpithelialDenominator": true,
        "geneLevelDe": true,
        "agingSignatureControl": true,
        "senescencePanelScoring": true
      },
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "candidate_is_senescence",
            "role": "gate",
            "field": "$.candidate_mechanism",
            "kind": "value",
            "op": "in",
            "expected": [
              "cellular_senescence",
              "oncogene_induced_senescence",
              "replicative_senescence"
            ]
          },
          {
            "name": "arrest_core_not_engaged",
            "role": "gate",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_not_engaged"
            ]
          },
          {
            "name": "secretory_arm_is_inflammaging",
            "role": "gate",
            "field": "$.secretory_arm_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "acute_phase_inflammaging_response"
            ]
          },
          {
            "name": "secretory_arm_broadly_elevated",
            "role": "gate",
            "field": "$.secretory_arm_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "broadly_elevated"
            ]
          },
          {
            "name": "carries_aging_program",
            "role": "gate",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "carries_bona_fide_aging_program_distinct_from_nominated"
            ]
          },
          {
            "name": "conclusion_senescence_unlikely",
            "role": "gate",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "unlikely_major_contributor"
            ]
          },
          {
            "name": "cf_arrest_core_engaged_recall",
            "role": "hard_fail",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_engaged"
            ]
          },
          {
            "name": "cf_aging_program_conflation_or_explain_away",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "aging_program_indistinguishable_from_nominated_program",
              "aging_program_is_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_no_aging_program_sensitivity_failure",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_detectable_aging_program"
            ]
          },
          {
            "name": "cf_senescence_driver_conclusion",
            "role": "hard_fail",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "likely_primary_driver",
              "plausible_supported_contributor"
            ]
          }
        ]
      },
      "trajectoryAvailable": true,
      "runId": "claude-opus-4-8__claude-code__gse297023_q2_senescence_not_mechanism__r2",
      "trajectoryFile": "scbench-long/trajectories/gse297023_q2_senescence_not_mechanism__claude-opus-4-8__claude-code__r2.json"
    },
    {
      "modelId": "sclong-gpt-5-6-terra-codex",
      "modelName": "gpt-5.6-terra",
      "harness": "openai-codex",
      "provider": "OpenAI",
      "available": true,
      "task": "gse171524_ectopic_airway_rare_cell",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 1,
      "passed": true,
      "score": null,
      "durationS": 197.757799,
      "cost": 0,
      "nTurns": null,
      "reasoning": "",
      "metrics": {},
      "fieldScores": {},
      "graderChecks": [],
      "agentAnswer": null,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "focus_population_identity_gate",
            "role": "gate",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "chemosensory_marker_expressing_population_alpha"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "focus_population_marker_calls",
            "kind": "list",
            "matchKey": "marker",
            "k": 3,
            "expected": [
              {
                "key": "POU2F3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "CHAT",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "TRPM5",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "LRMP",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNAT3",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "AVIL",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              },
              {
                "key": "GNG13",
                "fields": {
                  "direction": {
                    "op": "in",
                    "values": [
                      "positive"
                    ],
                    "canonical": "positive"
                  }
                }
              }
            ]
          },
          {
            "name": "focus_population_condition_distribution_gate",
            "role": "gate",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "covid_enriched_with_substantial_control_presence",
              "covid_predominant",
              "covid_only",
              "covid_enriched_trend_not_significant"
            ]
          },
          {
            "name": "focus_population_location_gate",
            "role": "gate",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "ectopic_parenchyma_in_covid",
              "parenchyma_only",
              "bronchiolized_or_peribronchial_parenchyma",
              "airway_and_parenchyma"
            ]
          },
          {
            "name": "focus_population_recovery_mode_gate",
            "role": "gate",
            "field": "$.focus_population_recovery_mode",
            "kind": "value",
            "op": "in",
            "expected": [
              "continuum_not_discrete_cluster"
            ]
          },
          {
            "name": "focus_population_identity_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "ionocyte_like_beta",
              "neuroendocrine_like_gamma",
              "basal_stem_progenitor_like_delta",
              "aberrant_basaloid_transitional_like_epsilon",
              "neuronal_or_neural_gene_minor_population_zeta",
              "deuterosomal_ciliated_progenitor_like_eta",
              "hillock_transitional_like_theta",
              "low_quality_doublet_or_debris",
              "no_rare_cell_recovered",
              "not_assessed"
            ]
          },
          {
            "name": "cf_wrong_lineage_marker_called_positive",
            "role": "hard_fail",
            "field": "$.focus_population_marker_calls[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "focus_population_condition_distribution_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_condition_distribution",
            "kind": "value",
            "op": "in",
            "expected": [
              "equally_distributed",
              "control_predominant",
              "not_assessed"
            ]
          },
          {
            "name": "focus_population_location_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.focus_population_location",
            "kind": "value",
            "op": "in",
            "expected": [
              "airway_only",
              "not_assessed"
            ]
          },
          {
            "name": "cross_axis_canonical_identity_with_airway_only_location_incoherence",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "focus_population_identity": {
                "op": "in",
                "values": [
                  "chemosensory_marker_expressing_population_alpha"
                ]
              },
              "focus_population_location": {
                "op": "in",
                "values": [
                  "airway_only"
                ]
              }
            }
          }
        ]
      },
      "trajectoryAvailable": false
    },
    {
      "modelId": "sclong-gemini-3-5-flash-pi",
      "modelName": "gemini-3.5-flash",
      "harness": "pi",
      "provider": "Google",
      "available": true,
      "task": "10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 1,
      "passed": true,
      "score": 15,
      "durationS": 890.56,
      "cost": 1.44600735,
      "nTurns": 51,
      "reasoning": "all_of [pass_rule=all]: PASS\n  + list_match: list_match (answer_field='cd8_states_enriched_in_singlet_fraction'): PASS  (score=3.0)\n  + list_match: list_match (answer_field='cd8_states_enriched_in_tumor_cluster_fraction'): PASS  (score=3.0)\n  + clonotype_expansion_direction_data_true: passed=True, score=1.0\n  + biological_inference_data_true: passed=True, score=1.0\n  + dict_match: dict_match (answer_field='biological_relations'): PASS  (score=7.0)\n  + hard_fail 'singlet_top3_contains_naive_state_prior_pushing': not triggered\n  + hard_fail 'tumor_cluster_top3_contains_exhausted_state_prior_pushing': not triggered\n  + hard_fail 'singlet_top3_contains_decoy_token': not triggered\n  + hard_fail 'tumor_cluster_top3_contains_decoy_token': not triggered\n  + hard_fail 'TOXhigh_Tex_wrong_functional_category': not triggered\n  + hard_fail 'TCF7_stem_like_Tex_wrong_functional_category': not triggered\n  + hard_fail 'MKI67_state_wrong_functional_category': not triggered\n  + hard_fail 'Tn_or_Tn_Tmem_wrong_functional_category': not triggered\n  + hard_fail 'missing_exhausted_or_proliferating_anchor_in_singlet_top3': not triggered\n  + hard_fail 'missing_stemlike_precursor_in_singlet_top3': not triggered\n  + hard_fail 'missing_naive_or_early_memory_anchor_in_tumor_cluster_top3': not triggered\n  + hard_fail 'cf_reported_canonical_expansion_direction': not triggered\n  + hard_fail 'cf_expansion_lazy_defer': not triggered\n  + hard_fail 'cf_reported_canonical_expected_biology': not triggered\n  + hard_fail 'cf_biological_inference_null_artifact_or_lineage': not triggered\n  + hard_fail 'cf_exhaustion_inverted_prior_push': not triggered\n  + hard_fail 'cf_naive_inverted_prior_push': not triggered\n  + hard_fail 'cf_proliferation_inverted_prior_push': not triggered\n  + hard_fail 'cf_expansion_richness_inverted_prior_push': not triggered\n  + hard_fail 'cf_multichain_inverted_prior_push': not triggered\n  + hard_fail 'cf_intermediate_gzmk_overgeneralized': not triggered\n  + hard_fail 'cf_melanocytic_conjugate_overcall': not triggered\n  + hard_fail 'cf_relations_mass_hedge': not triggered",
      "metrics": {
        "hard_fail_triggered": [],
        "pass_rule": "all",
        "scoring_count": 5,
        "scoring_passed": 5,
        "scoring_total_score": 15
      },
      "fieldScores": {
        "biological_inference_data_true": 1,
        "clonotype_expansion_direction_data_true": 1,
        "dict_match": 7,
        "list_match": 3
      },
      "graderChecks": [
        {
          "name": "cd8_states_enriched_in_singlet_fraction",
          "kind": "list_match",
          "passed": true,
          "score": 3
        },
        {
          "name": "cd8_states_enriched_in_tumor_cluster_fraction",
          "kind": "list_match",
          "passed": true,
          "score": 3
        },
        {
          "name": "clonotype_expansion_direction_data_true",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "biological_inference_data_true",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "biological_relations",
          "kind": "list_match",
          "passed": true,
          "score": 7
        },
        {
          "name": "singlet_top3_contains_naive_state_prior_pushing",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "singlet_top3_contains_decoy_token",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "tumor_cluster_top3_contains_decoy_token",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "TOXhigh_Tex_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "TCF7_stem_like_Tex_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "MKI67_state_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "Tn_or_Tn_Tmem_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_stemlike_precursor_in_singlet_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_reported_canonical_expansion_direction",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_expansion_lazy_defer",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_reported_canonical_expected_biology",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_biological_inference_null_artifact_or_lineage",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_exhaustion_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_naive_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_proliferation_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_expansion_richness_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_multichain_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_intermediate_gzmk_overgeneralized",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_melanocytic_conjugate_overcall",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_relations_mass_hedge",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "biological_inference": "singlet_antigen_experienced_cluster_bystander",
        "biological_relations": {
          "clonal_expansion_richness_in_singlets_vs_tumor_cluster": "greater_than",
          "exhaustion_program_in_singlets_vs_tumor_cluster": "greater_than",
          "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": "approximately_equal",
          "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": "approximately_equal",
          "multichain_tcr_rate_in_singlets_vs_tumor_cluster": "greater_than",
          "naive_memory_program_in_singlets_vs_tumor_cluster": "less_than",
          "proliferation_program_in_singlets_vs_tumor_cluster": "greater_than"
        },
        "cd8_states_enriched_in_singlet_fraction": [
          {
            "functional_category": "stem_like_exhausted",
            "state_label": "TCF7_stem_like_Tex"
          },
          {
            "functional_category": "intermediate_exhausted_gzmk",
            "state_label": "GZMKhigh_Tex"
          },
          {
            "functional_category": "terminal_exhausted",
            "state_label": "TOXhigh_Tex"
          }
        ],
        "cd8_states_enriched_in_tumor_cluster_fraction": [
          {
            "functional_category": "naive_memory",
            "state_label": "Tn"
          },
          {
            "functional_category": "naive_memory",
            "state_label": "Tn_Tmem"
          },
          {
            "functional_category": "early_effector_memory",
            "state_label": "early_Tem_GZMK"
          }
        ],
        "clonotype_expansion_finding": "singlet_substantially_more_expanded_than_cluster"
      },
      "checkpointSignals": {
        "clustering": true,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": false,
        "nerveJointAnchor": false,
        "multiomeAssembly": false,
        "depthNormalizedAtac": false,
        "regulonInference": false,
        "cd8SubsetIntegration": true,
        "perPatientEnrichment": true,
        "tcrRichnessMetrics": true,
        "orthologProjection": false,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": true,
        "mousePseudobulkDesign": false,
        "cancerEpithelialDenominator": false,
        "geneLevelDe": false,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_singlet_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "TOXhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "LAG3high_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "GZMKhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "intermediate_exhausted_gzmk"
                    ],
                    "canonical": "intermediate_exhausted_gzmk"
                  }
                }
              },
              {
                "key": "TCF7_stem_like_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "stem_like_exhausted"
                    ],
                    "canonical": "stem_like_exhausted"
                  }
                }
              },
              {
                "key": "MKI67_pos_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              },
              {
                "key": "MKI67high_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_tumor_cluster_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "Tn",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "Tn_Tmem",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "early_Tem_GZMK",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              },
              {
                "key": "Tem_GZMH",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              }
            ]
          },
          {
            "name": "clonotype_expansion_direction_data_true",
            "role": "gate",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_substantially_more_expanded_than_cluster",
              "singlet_modestly_more_expanded_than_cluster"
            ]
          },
          {
            "name": "biological_inference_data_true",
            "role": "gate",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_antigen_experienced_cluster_bystander"
            ]
          },
          {
            "name": "singlet_top3_contains_naive_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Tn",
              "Tn_Tmem",
              "early_Tem_GZMK",
              "Tem_GZMH"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "TOXhigh_Tex",
              "LAG3high_Tex",
              "GZMKhigh_Tex",
              "TCF7_stem_like_Tex",
              "MKI67_pos_Tex_Tprol",
              "MKI67high_Tex_Tprol"
            ]
          },
          {
            "name": "singlet_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "TOXhigh_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "TCF7_stem_like_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "MKI67_state_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "Tn_or_Tn_Tmem_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_stemlike_precursor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_reported_canonical_expansion_direction",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_substantially_more_expanded_than_singlet",
              "cluster_modestly_more_expanded_than_singlet"
            ]
          },
          {
            "name": "cf_expansion_lazy_defer",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "expansion_pattern_not_detectable"
            ]
          },
          {
            "name": "cf_reported_canonical_expected_biology",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_antigen_experienced_singlet_bystander"
            ]
          },
          {
            "name": "cf_biological_inference_null_artifact_or_lineage",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "differences_reflect_dissociation_or_sorting_artifact",
              "cluster_and_singlet_capture_same_biology",
              "cluster_and_singlet_capture_distinct_cd8_lineages"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "biological_relations",
            "kind": "dict",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_greater_than",
                  "greater_than"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "less_than",
                  "much_less_than"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than",
                  "approximately_equal"
                ]
              },
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal"
                ]
              },
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal",
                  "greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_exhaustion_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_naive_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_proliferation_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_expansion_richness_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_multichain_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_intermediate_gzmk_overgeneralized",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_melanocytic_conjugate_overcall",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "values": [
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_relations_mass_hedge",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              }
            }
          }
        ]
      },
      "trajectoryAvailable": true,
      "runId": "gemini-3.5-flash__pi__10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1__r1",
      "trajectoryFile": "scbench-long/trajectories/10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1__gemini-3.5-flash__pi__r1.json"
    },
    {
      "modelId": "sclong-opus-4-8-pi",
      "modelName": "claude-opus-4-8",
      "harness": "pi",
      "provider": "Anthropic",
      "available": true,
      "task": "gse297023_q2_senescence_not_mechanism",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 2,
      "passed": true,
      "score": 7,
      "durationS": 546.18,
      "cost": 1.0904015,
      "nTurns": 19,
      "reasoning": "all_of [pass_rule=all]: PASS\n  + cohort_spans_young_and_aged: passed=True, score=1.0\n  + candidate_is_senescence: passed=True, score=1.0\n  + arrest_core_not_engaged: passed=True, score=1.0\n  + secretory_arm_is_inflammaging: passed=True, score=1.0\n  + secretory_arm_broadly_elevated: passed=True, score=1.0\n  + carries_aging_program: passed=True, score=1.0\n  + conclusion_senescence_unlikely: passed=True, score=1.0\n  + hard_fail 'cf_arrest_core_engaged_recall': not triggered\n  + hard_fail 'cf_aging_program_conflation_or_explain_away': not triggered\n  + hard_fail 'cf_no_aging_program_sensitivity_failure': not triggered\n  + hard_fail 'cf_senescence_driver_conclusion': not triggered",
      "metrics": {
        "pass_rule": "all",
        "scoring_count": 7,
        "scoring_passed": 7,
        "scoring_total_score": 7,
        "hard_fail_triggered": []
      },
      "fieldScores": {
        "cohort_spans_young_and_aged": 1,
        "candidate_is_senescence": 1,
        "arrest_core_not_engaged": 1,
        "secretory_arm_is_inflammaging": 1,
        "secretory_arm_broadly_elevated": 1,
        "carries_aging_program": 1,
        "conclusion_senescence_unlikely": 1
      },
      "graderChecks": [
        {
          "name": "cohort_spans_young_and_aged",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "candidate_is_senescence",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "arrest_core_not_engaged",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "secretory_arm_is_inflammaging",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "secretory_arm_broadly_elevated",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "carries_aging_program",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "conclusion_senescence_unlikely",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "cf_arrest_core_engaged_recall",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_aging_program_conflation_or_explain_away",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_no_aging_program_sensitivity_failure",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_senescence_driver_conclusion",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "age_associated_program_identity": "carries_bona_fide_aging_program_distinct_from_nominated",
        "arrest_core_engaged": "arrest_core_not_engaged",
        "candidate_mechanism": "cellular_senescence",
        "cohort_host_age_design": "young_and_aged_hosts_both_profiled",
        "nominated_mechanism_role": "unlikely_major_contributor",
        "secretory_arm_direction": "broadly_elevated",
        "secretory_arm_identity": "acute_phase_inflammaging_response"
      },
      "checkpointSignals": {
        "clustering": false,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": false,
        "nerveJointAnchor": false,
        "multiomeAssembly": false,
        "depthNormalizedAtac": false,
        "regulonInference": false,
        "cd8SubsetIntegration": false,
        "perPatientEnrichment": false,
        "tcrRichnessMetrics": false,
        "orthologProjection": false,
        "epiSubsetSelection": true,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": false,
        "mousePseudobulkDesign": true,
        "cancerEpithelialDenominator": true,
        "geneLevelDe": true,
        "agingSignatureControl": true,
        "senescencePanelScoring": true
      },
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "candidate_is_senescence",
            "role": "gate",
            "field": "$.candidate_mechanism",
            "kind": "value",
            "op": "in",
            "expected": [
              "cellular_senescence",
              "oncogene_induced_senescence",
              "replicative_senescence"
            ]
          },
          {
            "name": "arrest_core_not_engaged",
            "role": "gate",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_not_engaged"
            ]
          },
          {
            "name": "secretory_arm_is_inflammaging",
            "role": "gate",
            "field": "$.secretory_arm_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "acute_phase_inflammaging_response"
            ]
          },
          {
            "name": "secretory_arm_broadly_elevated",
            "role": "gate",
            "field": "$.secretory_arm_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "broadly_elevated"
            ]
          },
          {
            "name": "carries_aging_program",
            "role": "gate",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "carries_bona_fide_aging_program_distinct_from_nominated"
            ]
          },
          {
            "name": "conclusion_senescence_unlikely",
            "role": "gate",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "unlikely_major_contributor"
            ]
          },
          {
            "name": "cf_arrest_core_engaged_recall",
            "role": "hard_fail",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_engaged"
            ]
          },
          {
            "name": "cf_aging_program_conflation_or_explain_away",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "aging_program_indistinguishable_from_nominated_program",
              "aging_program_is_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_no_aging_program_sensitivity_failure",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_detectable_aging_program"
            ]
          },
          {
            "name": "cf_senescence_driver_conclusion",
            "role": "hard_fail",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "likely_primary_driver",
              "plausible_supported_contributor"
            ]
          }
        ]
      },
      "trajectoryAvailable": true,
      "runId": "claude-opus-4-8__pi__gse297023_q2_senescence_not_mechanism__r2",
      "trajectoryFile": "scbench-long/trajectories/gse297023_q2_senescence_not_mechanism__claude-opus-4-8__pi__r2.json"
    },
    {
      "modelId": "sclong-gpt-5-6-luna-pi",
      "modelName": "gpt-5.6-luna",
      "harness": "pi",
      "provider": "OpenAI",
      "available": true,
      "task": "snrnaseq_per_donor_composition_shifts_lethal_covid_lung",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 2,
      "passed": true,
      "score": null,
      "durationS": 331.699468,
      "cost": 0.17694845,
      "nTurns": null,
      "reasoning": "",
      "metrics": {},
      "fieldScores": {},
      "graderChecks": [],
      "agentAnswer": null,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "dict_match",
            "role": "gate",
            "field": "cell_type_direction_map",
            "kind": "dict",
            "expected": {
              "Myeloid": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "Epithelia": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Fibroblasts": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              },
              "T_NK": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "B_plasma": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Endothelial": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "Mast": {
                "op": "in",
                "value": [
                  "not_significant_at_alpha_0_05",
                  "expanded_trend_ns",
                  "depleted_trend_ns",
                  "no_consistent_direction_across_donors",
                  "shift_driven_by_single_donor",
                  "unchanged"
                ]
              },
              "APC": {
                "op": "in",
                "value": [
                  "depleted_in_covid",
                  "depleted_in_covid_fdr_robust",
                  "depleted_in_covid_nominal_only"
                ]
              },
              "Neuronal": {
                "op": "in",
                "value": [
                  "expanded_in_covid",
                  "expanded_in_covid_fdr_robust",
                  "expanded_in_covid_nominal_only"
                ]
              }
            }
          },
          {
            "name": "over_merge_immune_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Immune",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_leukocyte_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Leukocyte",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_lymphoid_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Lymphoid",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          },
          {
            "name": "over_merge_stromal_superclass_floor",
            "role": "hard_fail",
            "field": "$.cell_type_direction_map.Stromal",
            "kind": "value",
            "op": "in",
            "expected": [
              "expanded_in_covid",
              "expanded_in_covid_fdr_robust",
              "expanded_in_covid_nominal_only",
              "expanded_trend_ns",
              "depleted_in_covid",
              "depleted_in_covid_fdr_robust",
              "depleted_in_covid_nominal_only",
              "depleted_trend_ns",
              "not_significant_at_alpha_0_05",
              "no_consistent_direction_across_donors",
              "shift_driven_by_single_donor",
              "unchanged",
              "not_assessed"
            ]
          }
        ]
      },
      "trajectoryAvailable": false
    },
    {
      "modelId": "sclong-gemini-3-1-pro-preview-pi",
      "modelName": "gemini-3.1-pro-preview",
      "harness": "pi",
      "provider": "Google",
      "available": true,
      "task": "10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 2,
      "passed": true,
      "score": 15,
      "durationS": 963.33,
      "cost": 0.6311628,
      "nTurns": 42,
      "reasoning": "all_of [pass_rule=all]: PASS\n  + list_match: list_match (answer_field='cd8_states_enriched_in_singlet_fraction'): PASS  (score=3.0)\n  + list_match: list_match (answer_field='cd8_states_enriched_in_tumor_cluster_fraction'): PASS  (score=3.0)\n  + clonotype_expansion_direction_data_true: passed=True, score=1.0\n  + biological_inference_data_true: passed=True, score=1.0\n  + dict_match: dict_match (answer_field='biological_relations'): PASS  (score=7.0)\n  + hard_fail 'singlet_top3_contains_naive_state_prior_pushing': not triggered\n  + hard_fail 'tumor_cluster_top3_contains_exhausted_state_prior_pushing': not triggered\n  + hard_fail 'singlet_top3_contains_decoy_token': not triggered\n  + hard_fail 'tumor_cluster_top3_contains_decoy_token': not triggered\n  + hard_fail 'TOXhigh_Tex_wrong_functional_category': not triggered\n  + hard_fail 'TCF7_stem_like_Tex_wrong_functional_category': not triggered\n  + hard_fail 'MKI67_state_wrong_functional_category': not triggered\n  + hard_fail 'Tn_or_Tn_Tmem_wrong_functional_category': not triggered\n  + hard_fail 'missing_exhausted_or_proliferating_anchor_in_singlet_top3': not triggered\n  + hard_fail 'missing_stemlike_precursor_in_singlet_top3': not triggered\n  + hard_fail 'missing_naive_or_early_memory_anchor_in_tumor_cluster_top3': not triggered\n  + hard_fail 'cf_reported_canonical_expansion_direction': not triggered\n  + hard_fail 'cf_expansion_lazy_defer': not triggered\n  + hard_fail 'cf_reported_canonical_expected_biology': not triggered\n  + hard_fail 'cf_biological_inference_null_artifact_or_lineage': not triggered\n  + hard_fail 'cf_exhaustion_inverted_prior_push': not triggered\n  + hard_fail 'cf_naive_inverted_prior_push': not triggered\n  + hard_fail 'cf_proliferation_inverted_prior_push': not triggered\n  + hard_fail 'cf_expansion_richness_inverted_prior_push': not triggered\n  + hard_fail 'cf_multichain_inverted_prior_push': not triggered\n  + hard_fail 'cf_intermediate_gzmk_overgeneralized': not triggered\n  + hard_fail 'cf_melanocytic_conjugate_overcall': not triggered\n  + hard_fail 'cf_relations_mass_hedge': not triggered",
      "metrics": {
        "hard_fail_triggered": [],
        "pass_rule": "all",
        "scoring_count": 5,
        "scoring_passed": 5,
        "scoring_total_score": 15
      },
      "fieldScores": {
        "biological_inference_data_true": 1,
        "clonotype_expansion_direction_data_true": 1,
        "dict_match": 7,
        "list_match": 3
      },
      "graderChecks": [
        {
          "name": "cd8_states_enriched_in_singlet_fraction",
          "kind": "list_match",
          "passed": true,
          "score": 3
        },
        {
          "name": "cd8_states_enriched_in_tumor_cluster_fraction",
          "kind": "list_match",
          "passed": true,
          "score": 3
        },
        {
          "name": "clonotype_expansion_direction_data_true",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "biological_inference_data_true",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "biological_relations",
          "kind": "list_match",
          "passed": true,
          "score": 7
        },
        {
          "name": "singlet_top3_contains_naive_state_prior_pushing",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "singlet_top3_contains_decoy_token",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "tumor_cluster_top3_contains_decoy_token",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "TOXhigh_Tex_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "TCF7_stem_like_Tex_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "MKI67_state_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "Tn_or_Tn_Tmem_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_stemlike_precursor_in_singlet_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_reported_canonical_expansion_direction",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_expansion_lazy_defer",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_reported_canonical_expected_biology",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_biological_inference_null_artifact_or_lineage",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_exhaustion_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_naive_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_proliferation_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_expansion_richness_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_multichain_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_intermediate_gzmk_overgeneralized",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_melanocytic_conjugate_overcall",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_relations_mass_hedge",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "biological_inference": "singlet_antigen_experienced_cluster_bystander",
        "biological_relations": {
          "clonal_expansion_richness_in_singlets_vs_tumor_cluster": "greater_than",
          "exhaustion_program_in_singlets_vs_tumor_cluster": "much_greater_than",
          "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": "approximately_equal",
          "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": "greater_than",
          "multichain_tcr_rate_in_singlets_vs_tumor_cluster": "greater_than",
          "naive_memory_program_in_singlets_vs_tumor_cluster": "much_less_than",
          "proliferation_program_in_singlets_vs_tumor_cluster": "greater_than"
        },
        "cd8_states_enriched_in_singlet_fraction": [
          {
            "functional_category": "stem_like_exhausted",
            "state_label": "TCF7_stem_like_Tex"
          },
          {
            "functional_category": "intermediate_exhausted_gzmk",
            "state_label": "GZMKhigh_Tex"
          },
          {
            "functional_category": "terminal_exhausted",
            "state_label": "TOXhigh_Tex"
          }
        ],
        "cd8_states_enriched_in_tumor_cluster_fraction": [
          {
            "functional_category": "naive_memory",
            "state_label": "Tn"
          },
          {
            "functional_category": "naive_memory",
            "state_label": "Tn_Tmem"
          },
          {
            "functional_category": "early_effector_memory",
            "state_label": "early_Tem_GZMK"
          }
        ],
        "clonotype_expansion_finding": "singlet_modestly_more_expanded_than_cluster"
      },
      "checkpointSignals": {
        "clustering": true,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": false,
        "nerveJointAnchor": false,
        "multiomeAssembly": false,
        "depthNormalizedAtac": false,
        "regulonInference": false,
        "cd8SubsetIntegration": true,
        "perPatientEnrichment": true,
        "tcrRichnessMetrics": true,
        "orthologProjection": false,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": true,
        "mousePseudobulkDesign": false,
        "cancerEpithelialDenominator": true,
        "geneLevelDe": false,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_singlet_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "TOXhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "LAG3high_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "GZMKhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "intermediate_exhausted_gzmk"
                    ],
                    "canonical": "intermediate_exhausted_gzmk"
                  }
                }
              },
              {
                "key": "TCF7_stem_like_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "stem_like_exhausted"
                    ],
                    "canonical": "stem_like_exhausted"
                  }
                }
              },
              {
                "key": "MKI67_pos_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              },
              {
                "key": "MKI67high_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_tumor_cluster_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "Tn",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "Tn_Tmem",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "early_Tem_GZMK",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              },
              {
                "key": "Tem_GZMH",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              }
            ]
          },
          {
            "name": "clonotype_expansion_direction_data_true",
            "role": "gate",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_substantially_more_expanded_than_cluster",
              "singlet_modestly_more_expanded_than_cluster"
            ]
          },
          {
            "name": "biological_inference_data_true",
            "role": "gate",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_antigen_experienced_cluster_bystander"
            ]
          },
          {
            "name": "singlet_top3_contains_naive_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Tn",
              "Tn_Tmem",
              "early_Tem_GZMK",
              "Tem_GZMH"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "TOXhigh_Tex",
              "LAG3high_Tex",
              "GZMKhigh_Tex",
              "TCF7_stem_like_Tex",
              "MKI67_pos_Tex_Tprol",
              "MKI67high_Tex_Tprol"
            ]
          },
          {
            "name": "singlet_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "TOXhigh_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "TCF7_stem_like_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "MKI67_state_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "Tn_or_Tn_Tmem_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_stemlike_precursor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_reported_canonical_expansion_direction",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_substantially_more_expanded_than_singlet",
              "cluster_modestly_more_expanded_than_singlet"
            ]
          },
          {
            "name": "cf_expansion_lazy_defer",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "expansion_pattern_not_detectable"
            ]
          },
          {
            "name": "cf_reported_canonical_expected_biology",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_antigen_experienced_singlet_bystander"
            ]
          },
          {
            "name": "cf_biological_inference_null_artifact_or_lineage",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "differences_reflect_dissociation_or_sorting_artifact",
              "cluster_and_singlet_capture_same_biology",
              "cluster_and_singlet_capture_distinct_cd8_lineages"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "biological_relations",
            "kind": "dict",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_greater_than",
                  "greater_than"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "less_than",
                  "much_less_than"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than",
                  "approximately_equal"
                ]
              },
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal"
                ]
              },
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal",
                  "greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_exhaustion_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_naive_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_proliferation_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_expansion_richness_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_multichain_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_intermediate_gzmk_overgeneralized",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_melanocytic_conjugate_overcall",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "values": [
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_relations_mass_hedge",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              }
            }
          }
        ]
      },
      "trajectoryAvailable": true,
      "runId": "gemini-3.1-pro-preview__pi__10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1__r2",
      "trajectoryFile": "scbench-long/trajectories/10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1__gemini-3.1-pro-preview__pi__r2.json"
    },
    {
      "modelId": "sclong-gpt-5-5-codex",
      "modelName": "gpt-5.5",
      "harness": "openai-codex",
      "provider": "OpenAI",
      "available": true,
      "task": "10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": true,
      "score": 14,
      "durationS": 940.22,
      "cost": 2.093881,
      "nTurns": 1,
      "reasoning": "all_of [pass_rule=all]: PASS\n  + list_match: list_match (answer_field='cd8_states_enriched_in_singlet_fraction'): PASS  (score=3.0)\n  + list_match: list_match (answer_field='cd8_states_enriched_in_tumor_cluster_fraction'): PASS  (score=2.0)\n  + clonotype_expansion_direction_data_true: passed=True, score=1.0\n  + biological_inference_data_true: passed=True, score=1.0\n  + dict_match: dict_match (answer_field='biological_relations'): PASS  (score=7.0)\n  + hard_fail 'singlet_top3_contains_naive_state_prior_pushing': not triggered\n  + hard_fail 'tumor_cluster_top3_contains_exhausted_state_prior_pushing': not triggered\n  + hard_fail 'singlet_top3_contains_decoy_token': not triggered\n  + hard_fail 'tumor_cluster_top3_contains_decoy_token': not triggered\n  + hard_fail 'TOXhigh_Tex_wrong_functional_category': not triggered\n  + hard_fail 'TCF7_stem_like_Tex_wrong_functional_category': not triggered\n  + hard_fail 'MKI67_state_wrong_functional_category': not triggered\n  + hard_fail 'Tn_or_Tn_Tmem_wrong_functional_category': not triggered\n  + hard_fail 'missing_exhausted_or_proliferating_anchor_in_singlet_top3': not triggered\n  + hard_fail 'missing_stemlike_precursor_in_singlet_top3': not triggered\n  + hard_fail 'missing_naive_or_early_memory_anchor_in_tumor_cluster_top3': not triggered\n  + hard_fail 'cf_reported_canonical_expansion_direction': not triggered\n  + hard_fail 'cf_expansion_lazy_defer': not triggered\n  + hard_fail 'cf_reported_canonical_expected_biology': not triggered\n  + hard_fail 'cf_biological_inference_null_artifact_or_lineage': not triggered\n  + hard_fail 'cf_exhaustion_inverted_prior_push': not triggered\n  + hard_fail 'cf_naive_inverted_prior_push': not triggered\n  + hard_fail 'cf_proliferation_inverted_prior_push': not triggered\n  + hard_fail 'cf_expansion_richness_inverted_prior_push': not triggered\n  + hard_fail 'cf_multichain_inverted_prior_push': not triggered\n  + hard_fail 'cf_intermediate_gzmk_overgeneralized': not triggered\n  + hard_fail 'cf_melanocytic_conjugate_overcall': not triggered\n  + hard_fail 'cf_relations_mass_hedge': not triggered",
      "metrics": {
        "hard_fail_triggered": [],
        "pass_rule": "all",
        "scoring_count": 5,
        "scoring_passed": 5,
        "scoring_total_score": 14
      },
      "fieldScores": {
        "biological_inference_data_true": 1,
        "clonotype_expansion_direction_data_true": 1,
        "dict_match": 7,
        "list_match": 2
      },
      "graderChecks": [
        {
          "name": "cd8_states_enriched_in_singlet_fraction",
          "kind": "list_match",
          "passed": true,
          "score": 3
        },
        {
          "name": "cd8_states_enriched_in_tumor_cluster_fraction",
          "kind": "list_match",
          "passed": true,
          "score": 2
        },
        {
          "name": "clonotype_expansion_direction_data_true",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "biological_inference_data_true",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "biological_relations",
          "kind": "list_match",
          "passed": true,
          "score": 7
        },
        {
          "name": "singlet_top3_contains_naive_state_prior_pushing",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "singlet_top3_contains_decoy_token",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "tumor_cluster_top3_contains_decoy_token",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "TOXhigh_Tex_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "TCF7_stem_like_Tex_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "MKI67_state_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "Tn_or_Tn_Tmem_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_stemlike_precursor_in_singlet_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_reported_canonical_expansion_direction",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_expansion_lazy_defer",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_reported_canonical_expected_biology",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_biological_inference_null_artifact_or_lineage",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_exhaustion_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_naive_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_proliferation_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_expansion_richness_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_multichain_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_intermediate_gzmk_overgeneralized",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_melanocytic_conjugate_overcall",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_relations_mass_hedge",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "biological_inference": "singlet_antigen_experienced_cluster_bystander",
        "biological_relations": {
          "clonal_expansion_richness_in_singlets_vs_tumor_cluster": "greater_than",
          "exhaustion_program_in_singlets_vs_tumor_cluster": "greater_than",
          "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": "approximately_equal",
          "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": "approximately_equal",
          "multichain_tcr_rate_in_singlets_vs_tumor_cluster": "approximately_equal",
          "naive_memory_program_in_singlets_vs_tumor_cluster": "less_than",
          "proliferation_program_in_singlets_vs_tumor_cluster": "greater_than"
        },
        "cd8_states_enriched_in_singlet_fraction": [
          {
            "functional_category": "stem_like_exhausted",
            "state_label": "TCF7_stem_like_Tex"
          },
          {
            "functional_category": "intermediate_exhausted_gzmk",
            "state_label": "GZMKhigh_Tex"
          },
          {
            "functional_category": "terminal_exhausted",
            "state_label": "TOXhigh_Tex"
          }
        ],
        "cd8_states_enriched_in_tumor_cluster_fraction": [
          {
            "functional_category": "naive_memory",
            "state_label": "Tn_Tmem"
          },
          {
            "functional_category": "naive_memory",
            "state_label": "Tn"
          },
          {
            "functional_category": "early_effector_memory",
            "state_label": "Tem_NK_like"
          }
        ],
        "clonotype_expansion_finding": "singlet_modestly_more_expanded_than_cluster"
      },
      "checkpointSignals": {
        "clustering": false,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": true,
        "nerveJointAnchor": false,
        "multiomeAssembly": true,
        "depthNormalizedAtac": true,
        "regulonInference": true,
        "cd8SubsetIntegration": true,
        "perPatientEnrichment": true,
        "tcrRichnessMetrics": true,
        "orthologProjection": false,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": false,
        "mousePseudobulkDesign": false,
        "cancerEpithelialDenominator": true,
        "geneLevelDe": false,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_singlet_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "TOXhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "LAG3high_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "GZMKhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "intermediate_exhausted_gzmk"
                    ],
                    "canonical": "intermediate_exhausted_gzmk"
                  }
                }
              },
              {
                "key": "TCF7_stem_like_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "stem_like_exhausted"
                    ],
                    "canonical": "stem_like_exhausted"
                  }
                }
              },
              {
                "key": "MKI67_pos_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              },
              {
                "key": "MKI67high_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_tumor_cluster_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "Tn",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "Tn_Tmem",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "early_Tem_GZMK",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              },
              {
                "key": "Tem_GZMH",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              }
            ]
          },
          {
            "name": "clonotype_expansion_direction_data_true",
            "role": "gate",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_substantially_more_expanded_than_cluster",
              "singlet_modestly_more_expanded_than_cluster"
            ]
          },
          {
            "name": "biological_inference_data_true",
            "role": "gate",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_antigen_experienced_cluster_bystander"
            ]
          },
          {
            "name": "singlet_top3_contains_naive_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Tn",
              "Tn_Tmem",
              "early_Tem_GZMK",
              "Tem_GZMH"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "TOXhigh_Tex",
              "LAG3high_Tex",
              "GZMKhigh_Tex",
              "TCF7_stem_like_Tex",
              "MKI67_pos_Tex_Tprol",
              "MKI67high_Tex_Tprol"
            ]
          },
          {
            "name": "singlet_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "TOXhigh_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "TCF7_stem_like_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "MKI67_state_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "Tn_or_Tn_Tmem_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_stemlike_precursor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_reported_canonical_expansion_direction",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_substantially_more_expanded_than_singlet",
              "cluster_modestly_more_expanded_than_singlet"
            ]
          },
          {
            "name": "cf_expansion_lazy_defer",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "expansion_pattern_not_detectable"
            ]
          },
          {
            "name": "cf_reported_canonical_expected_biology",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_antigen_experienced_singlet_bystander"
            ]
          },
          {
            "name": "cf_biological_inference_null_artifact_or_lineage",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "differences_reflect_dissociation_or_sorting_artifact",
              "cluster_and_singlet_capture_same_biology",
              "cluster_and_singlet_capture_distinct_cd8_lineages"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "biological_relations",
            "kind": "dict",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_greater_than",
                  "greater_than"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "less_than",
                  "much_less_than"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than",
                  "approximately_equal"
                ]
              },
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal"
                ]
              },
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal",
                  "greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_exhaustion_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_naive_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_proliferation_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_expansion_richness_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_multichain_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_intermediate_gzmk_overgeneralized",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_melanocytic_conjugate_overcall",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "values": [
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_relations_mass_hedge",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              }
            }
          }
        ]
      },
      "trajectoryAvailable": true,
      "runId": "gpt-5.5__openai-codex__10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1__r3",
      "trajectoryFile": "scbench-long/trajectories/10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1__gpt-5.5__openai-codex__r3.json"
    },
    {
      "modelId": "sclong-gpt-5-5-pi",
      "modelName": "gpt-5.5",
      "harness": "pi",
      "provider": "OpenAI",
      "available": true,
      "task": "10x_5p_melanoma_tcr_matched_apc_vs_tumor_exhaustion",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": true,
      "score": null,
      "durationS": 371,
      "cost": 1.3202180000000003,
      "nTurns": null,
      "reasoning": "",
      "metrics": {},
      "fieldScores": {},
      "graderChecks": [],
      "agentAnswer": null,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "state_discriminating_genes",
            "kind": "list",
            "matchKey": "gene",
            "k": 3,
            "expected": [
              {
                "key": "CTLA4",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TOX",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "ENTPD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "HAVCR2",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TIGIT",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "PDCD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "difference_attributable_to",
            "role": "gate",
            "field": "difference_attributable_to",
            "kind": "value",
            "expected": "covaries_with_apc_conjugate_state"
          },
          {
            "name": "causal_status",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "direction_unresolved_selection_confound"
          },
          {
            "name": "functional_interpretation",
            "role": "gate",
            "field": "functional_interpretation",
            "kind": "value",
            "expected": "costimulation_signature_function_unresolved"
          },
          {
            "name": "cf_contamination_gene_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "MLANA",
              "PMEL",
              "DCT",
              "TYR",
              "TYRP1",
              "SOX10",
              "MITF",
              "MAGEA4"
            ]
          },
          {
            "name": "cf_apc_partner_contamination_gene",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "HLA-DRA",
              "CD68",
              "LYZ",
              "CD14",
              "C1QA",
              "ITGAX",
              "CD1C"
            ]
          },
          {
            "name": "cf_ligand_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "CD274",
              "PDCD1LG2",
              "CD80",
              "CD86"
            ]
          },
          {
            "name": "cf_attribution_causal_or_baseline",
            "role": "hard_fail",
            "field": "$.difference_attributable_to",
            "kind": "avoid",
            "expected": [
              "apc_engagement",
              "tumor_cell_engagement",
              "clonotype_intrinsic_baseline"
            ]
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "engagement_induces_exhausted_state",
              "pre_exhausted_cells_select_into_apc_conjugates"
            ]
          },
          {
            "name": "cf_causal_fabrication",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "resolved_by_pseudotemporal_ordering",
              "resolved_by_perturbation_experiment"
            ]
          },
          {
            "name": "cf_functional_overclaim",
            "role": "hard_fail",
            "field": "$.functional_interpretation",
            "kind": "avoid",
            "expected": [
              "costimulation_licensed_reinvigoration_competent",
              "costimulation_licensed_checkpoint_responsive",
              "tumor_driven_terminal_exhaustion"
            ]
          }
        ]
      },
      "trajectoryAvailable": false
    },
    {
      "modelId": "sclong-sonnet-5-pi",
      "modelName": "claude-sonnet-5",
      "harness": "pi",
      "provider": "Anthropic",
      "available": true,
      "task": "gse297023_q2_senescence_not_mechanism",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "trialIndex": 2,
      "passed": true,
      "score": null,
      "durationS": 1106.898861,
      "cost": 0.4766123,
      "nTurns": null,
      "reasoning": "",
      "metrics": {},
      "fieldScores": {},
      "graderChecks": [],
      "agentAnswer": null,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "candidate_is_senescence",
            "role": "gate",
            "field": "$.candidate_mechanism",
            "kind": "value",
            "op": "in",
            "expected": [
              "cellular_senescence",
              "oncogene_induced_senescence",
              "replicative_senescence"
            ]
          },
          {
            "name": "arrest_core_not_engaged",
            "role": "gate",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_not_engaged"
            ]
          },
          {
            "name": "secretory_arm_is_inflammaging",
            "role": "gate",
            "field": "$.secretory_arm_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "acute_phase_inflammaging_response"
            ]
          },
          {
            "name": "secretory_arm_broadly_elevated",
            "role": "gate",
            "field": "$.secretory_arm_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "broadly_elevated"
            ]
          },
          {
            "name": "carries_aging_program",
            "role": "gate",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "carries_bona_fide_aging_program_distinct_from_nominated"
            ]
          },
          {
            "name": "conclusion_senescence_unlikely",
            "role": "gate",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "unlikely_major_contributor"
            ]
          },
          {
            "name": "cf_arrest_core_engaged_recall",
            "role": "hard_fail",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_engaged"
            ]
          },
          {
            "name": "cf_aging_program_conflation_or_explain_away",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "aging_program_indistinguishable_from_nominated_program",
              "aging_program_is_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_no_aging_program_sensitivity_failure",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_detectable_aging_program"
            ]
          },
          {
            "name": "cf_senescence_driver_conclusion",
            "role": "hard_fail",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "likely_primary_driver",
              "plausible_supported_contributor"
            ]
          }
        ]
      },
      "trajectoryAvailable": false
    },
    {
      "modelId": "sclong-gpt-5-6-luna-codex",
      "modelName": "gpt-5.6-luna",
      "harness": "openai-codex",
      "provider": "OpenAI",
      "available": true,
      "task": "q6_myeloid_subset_restricted_de",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 3,
      "passed": true,
      "score": null,
      "durationS": 526.010372,
      "cost": 0,
      "nTurns": null,
      "reasoning": "",
      "metrics": {},
      "fieldScores": {},
      "graderChecks": [],
      "agentAnswer": null,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "myeloid_subset_with_strongest_change_gate",
            "role": "gate",
            "field": "$.myeloid_subset_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "subset_gamma"
            ]
          },
          {
            "name": "gene_with_strongest_change_gate",
            "role": "gate",
            "field": "$.gene_with_strongest_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "AXL",
              "PROS1"
            ]
          },
          {
            "name": "direction_of_change_gate",
            "role": "gate",
            "field": "$.direction_of_change",
            "kind": "value",
            "op": "in",
            "expected": [
              "down_in_covid"
            ]
          },
          {
            "name": "biological_function_interpretation_gate",
            "role": "gate",
            "field": "$.biological_function_interpretation",
            "kind": "value",
            "op": "in",
            "expected": [
              "efferocytosis",
              "phagocytosis"
            ]
          },
          {
            "name": "myeloid_compartment_abundance_gate",
            "role": "gate",
            "field": "$.myeloid_compartment_abundance",
            "kind": "value",
            "op": "in",
            "expected": [
              "expands_in_covid"
            ]
          },
          {
            "name": "myeloid_expansion_localization_gate",
            "role": "gate",
            "field": "$.myeloid_expansion_localization",
            "kind": "value",
            "op": "in",
            "expected": [
              "recruited_driven_resident_dysfunction"
            ]
          },
          {
            "name": "dominant_signature_gate",
            "role": "gate",
            "field": "$.dominant_signature",
            "kind": "value",
            "op": "in",
            "expected": [
              "technical_or_agonal_artifact"
            ]
          },
          {
            "name": "dominant_signature_direction_gate",
            "role": "gate",
            "field": "$.dominant_signature_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "up_in_covid"
            ]
          }
        ]
      },
      "trajectoryAvailable": false
    },
    {
      "modelId": "sclong-grok-4-5-pi",
      "modelName": "grok-4.5",
      "harness": "pi",
      "provider": "SpaceXAI",
      "available": true,
      "task": "10x_5p_melanoma_tcr_matched_apc_vs_tumor_exhaustion",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 1,
      "passed": true,
      "score": null,
      "durationS": 511.495185,
      "cost": 0.346606,
      "nTurns": null,
      "reasoning": "",
      "metrics": {},
      "fieldScores": {},
      "graderChecks": [],
      "agentAnswer": null,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "state_discriminating_genes",
            "kind": "list",
            "matchKey": "gene",
            "k": 3,
            "expected": [
              {
                "key": "CTLA4",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TOX",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "ENTPD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "HAVCR2",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TIGIT",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "PDCD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "difference_attributable_to",
            "role": "gate",
            "field": "difference_attributable_to",
            "kind": "value",
            "expected": "covaries_with_apc_conjugate_state"
          },
          {
            "name": "causal_status",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "direction_unresolved_selection_confound"
          },
          {
            "name": "functional_interpretation",
            "role": "gate",
            "field": "functional_interpretation",
            "kind": "value",
            "expected": "costimulation_signature_function_unresolved"
          },
          {
            "name": "cf_contamination_gene_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "MLANA",
              "PMEL",
              "DCT",
              "TYR",
              "TYRP1",
              "SOX10",
              "MITF",
              "MAGEA4"
            ]
          },
          {
            "name": "cf_apc_partner_contamination_gene",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "HLA-DRA",
              "CD68",
              "LYZ",
              "CD14",
              "C1QA",
              "ITGAX",
              "CD1C"
            ]
          },
          {
            "name": "cf_ligand_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "CD274",
              "PDCD1LG2",
              "CD80",
              "CD86"
            ]
          },
          {
            "name": "cf_attribution_causal_or_baseline",
            "role": "hard_fail",
            "field": "$.difference_attributable_to",
            "kind": "avoid",
            "expected": [
              "apc_engagement",
              "tumor_cell_engagement",
              "clonotype_intrinsic_baseline"
            ]
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "engagement_induces_exhausted_state",
              "pre_exhausted_cells_select_into_apc_conjugates"
            ]
          },
          {
            "name": "cf_causal_fabrication",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "resolved_by_pseudotemporal_ordering",
              "resolved_by_perturbation_experiment"
            ]
          },
          {
            "name": "cf_functional_overclaim",
            "role": "hard_fail",
            "field": "$.functional_interpretation",
            "kind": "avoid",
            "expected": [
              "costimulation_licensed_reinvigoration_competent",
              "costimulation_licensed_checkpoint_responsive",
              "tumor_driven_terminal_exhaustion"
            ]
          }
        ]
      },
      "trajectoryAvailable": false
    },
    {
      "modelId": "sclong-sonnet-5-claude-code",
      "modelName": "claude-sonnet-5",
      "harness": "claude-code",
      "provider": "Anthropic",
      "available": true,
      "task": "gse155381_q4_pseudotime_delay",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 3,
      "passed": true,
      "score": null,
      "durationS": 2000.278511,
      "cost": 8.600373949999996,
      "nTurns": null,
      "reasoning": "",
      "metrics": {},
      "fieldScores": {},
      "graderChecks": [],
      "agentAnswer": null,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "pseudotime_direction_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "E17_chronological_match_weighted",
            "role": "additive",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pluripotency_exit_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pseudotime_direction_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_synchronous_with_monkey_EPI",
              "chimera_human_EPI_developmentally_advanced",
              "chimera_human_EPI_delayed_at_pre_implantation_only_resolved_post_implantation"
            ]
          },
          {
            "name": "E17_chronological_match_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "resembles_monkey_E17_Gast_EPI",
              "resembles_chimera_monkey_E17_EPI_specifically",
              "chimera_human_E17_EPI_more_advanced_than_monkey_E17_Gast"
            ]
          },
          {
            "name": "pluripotency_exit_paper_inverted_or_prior_driven",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_advances_pluripotency_exit_prematurely",
              "naive_pluripotency_program_specifically_engaged",
              "pluripotency_core_retained_via_KLF17_naive_marker_specifically"
            ]
          }
        ]
      },
      "trajectoryAvailable": false
    },
    {
      "modelId": "sclong-opus-4-7-claude-code",
      "modelName": "claude-opus-4-7",
      "harness": "claude-code",
      "provider": "Anthropic",
      "available": true,
      "task": "10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": true,
      "score": 15,
      "durationS": 2356.75,
      "cost": 7.047943999999999,
      "nTurns": 71,
      "reasoning": "all_of [pass_rule=all]: PASS\n  + list_match: list_match (answer_field='cd8_states_enriched_in_singlet_fraction'): PASS  (score=3.0)\n  + list_match: list_match (answer_field='cd8_states_enriched_in_tumor_cluster_fraction'): PASS  (score=3.0)\n  + clonotype_expansion_direction_data_true: passed=True, score=1.0\n  + biological_inference_data_true: passed=True, score=1.0\n  + dict_match: dict_match (answer_field='biological_relations'): PASS  (score=7.0)\n  + hard_fail 'singlet_top3_contains_naive_state_prior_pushing': not triggered\n  + hard_fail 'tumor_cluster_top3_contains_exhausted_state_prior_pushing': not triggered\n  + hard_fail 'singlet_top3_contains_decoy_token': not triggered\n  + hard_fail 'tumor_cluster_top3_contains_decoy_token': not triggered\n  + hard_fail 'TOXhigh_Tex_wrong_functional_category': not triggered\n  + hard_fail 'TCF7_stem_like_Tex_wrong_functional_category': not triggered\n  + hard_fail 'MKI67_state_wrong_functional_category': not triggered\n  + hard_fail 'Tn_or_Tn_Tmem_wrong_functional_category': not triggered\n  + hard_fail 'missing_exhausted_or_proliferating_anchor_in_singlet_top3': not triggered\n  + hard_fail 'missing_stemlike_precursor_in_singlet_top3': not triggered\n  + hard_fail 'missing_naive_or_early_memory_anchor_in_tumor_cluster_top3': not triggered\n  + hard_fail 'cf_reported_canonical_expansion_direction': not triggered\n  + hard_fail 'cf_expansion_lazy_defer': not triggered\n  + hard_fail 'cf_reported_canonical_expected_biology': not triggered\n  + hard_fail 'cf_biological_inference_null_artifact_or_lineage': not triggered\n  + hard_fail 'cf_exhaustion_inverted_prior_push': not triggered\n  + hard_fail 'cf_naive_inverted_prior_push': not triggered\n  + hard_fail 'cf_proliferation_inverted_prior_push': not triggered\n  + hard_fail 'cf_expansion_richness_inverted_prior_push': not triggered\n  + hard_fail 'cf_multichain_inverted_prior_push': not triggered\n  + hard_fail 'cf_intermediate_gzmk_overgeneralized': not triggered\n  + hard_fail 'cf_melanocytic_conjugate_overcall': not triggered\n  + hard_fail 'cf_relations_mass_hedge': not triggered",
      "metrics": {
        "hard_fail_triggered": [],
        "pass_rule": "all",
        "scoring_count": 5,
        "scoring_passed": 5,
        "scoring_total_score": 15
      },
      "fieldScores": {
        "biological_inference_data_true": 1,
        "clonotype_expansion_direction_data_true": 1,
        "dict_match": 7,
        "list_match": 3
      },
      "graderChecks": [
        {
          "name": "cd8_states_enriched_in_singlet_fraction",
          "kind": "list_match",
          "passed": true,
          "score": 3
        },
        {
          "name": "cd8_states_enriched_in_tumor_cluster_fraction",
          "kind": "list_match",
          "passed": true,
          "score": 3
        },
        {
          "name": "clonotype_expansion_direction_data_true",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "biological_inference_data_true",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "biological_relations",
          "kind": "list_match",
          "passed": true,
          "score": 7
        },
        {
          "name": "singlet_top3_contains_naive_state_prior_pushing",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "singlet_top3_contains_decoy_token",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "tumor_cluster_top3_contains_decoy_token",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "TOXhigh_Tex_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "TCF7_stem_like_Tex_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "MKI67_state_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "Tn_or_Tn_Tmem_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_stemlike_precursor_in_singlet_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_reported_canonical_expansion_direction",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_expansion_lazy_defer",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_reported_canonical_expected_biology",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_biological_inference_null_artifact_or_lineage",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_exhaustion_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_naive_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_proliferation_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_expansion_richness_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_multichain_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_intermediate_gzmk_overgeneralized",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_melanocytic_conjugate_overcall",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_relations_mass_hedge",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "biological_inference": "singlet_antigen_experienced_cluster_bystander",
        "biological_relations": {
          "clonal_expansion_richness_in_singlets_vs_tumor_cluster": "greater_than",
          "exhaustion_program_in_singlets_vs_tumor_cluster": "greater_than",
          "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": "less_than",
          "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": "approximately_equal",
          "multichain_tcr_rate_in_singlets_vs_tumor_cluster": "greater_than",
          "naive_memory_program_in_singlets_vs_tumor_cluster": "less_than",
          "proliferation_program_in_singlets_vs_tumor_cluster": "greater_than"
        },
        "cd8_states_enriched_in_singlet_fraction": [
          {
            "functional_category": "stem_like_exhausted",
            "state_label": "TCF7_stem_like_Tex"
          },
          {
            "functional_category": "terminal_exhausted",
            "state_label": "TOXhigh_Tex"
          },
          {
            "functional_category": "proliferating",
            "state_label": "MKI67_pos_Tex_Tprol"
          }
        ],
        "cd8_states_enriched_in_tumor_cluster_fraction": [
          {
            "functional_category": "naive_memory",
            "state_label": "Tn"
          },
          {
            "functional_category": "naive_memory",
            "state_label": "Tn_Tmem"
          },
          {
            "functional_category": "early_effector_memory",
            "state_label": "early_Tem_GZMK"
          }
        ],
        "clonotype_expansion_finding": "singlet_substantially_more_expanded_than_cluster"
      },
      "checkpointSignals": {
        "clustering": true,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": false,
        "nerveJointAnchor": false,
        "multiomeAssembly": false,
        "depthNormalizedAtac": true,
        "regulonInference": false,
        "cd8SubsetIntegration": true,
        "perPatientEnrichment": true,
        "tcrRichnessMetrics": true,
        "orthologProjection": false,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": false,
        "mousePseudobulkDesign": false,
        "cancerEpithelialDenominator": false,
        "geneLevelDe": false,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_singlet_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "TOXhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "LAG3high_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "GZMKhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "intermediate_exhausted_gzmk"
                    ],
                    "canonical": "intermediate_exhausted_gzmk"
                  }
                }
              },
              {
                "key": "TCF7_stem_like_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "stem_like_exhausted"
                    ],
                    "canonical": "stem_like_exhausted"
                  }
                }
              },
              {
                "key": "MKI67_pos_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              },
              {
                "key": "MKI67high_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_tumor_cluster_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "Tn",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "Tn_Tmem",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "early_Tem_GZMK",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              },
              {
                "key": "Tem_GZMH",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              }
            ]
          },
          {
            "name": "clonotype_expansion_direction_data_true",
            "role": "gate",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_substantially_more_expanded_than_cluster",
              "singlet_modestly_more_expanded_than_cluster"
            ]
          },
          {
            "name": "biological_inference_data_true",
            "role": "gate",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_antigen_experienced_cluster_bystander"
            ]
          },
          {
            "name": "singlet_top3_contains_naive_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Tn",
              "Tn_Tmem",
              "early_Tem_GZMK",
              "Tem_GZMH"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "TOXhigh_Tex",
              "LAG3high_Tex",
              "GZMKhigh_Tex",
              "TCF7_stem_like_Tex",
              "MKI67_pos_Tex_Tprol",
              "MKI67high_Tex_Tprol"
            ]
          },
          {
            "name": "singlet_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "TOXhigh_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "TCF7_stem_like_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "MKI67_state_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "Tn_or_Tn_Tmem_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_stemlike_precursor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_reported_canonical_expansion_direction",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_substantially_more_expanded_than_singlet",
              "cluster_modestly_more_expanded_than_singlet"
            ]
          },
          {
            "name": "cf_expansion_lazy_defer",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "expansion_pattern_not_detectable"
            ]
          },
          {
            "name": "cf_reported_canonical_expected_biology",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_antigen_experienced_singlet_bystander"
            ]
          },
          {
            "name": "cf_biological_inference_null_artifact_or_lineage",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "differences_reflect_dissociation_or_sorting_artifact",
              "cluster_and_singlet_capture_same_biology",
              "cluster_and_singlet_capture_distinct_cd8_lineages"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "biological_relations",
            "kind": "dict",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_greater_than",
                  "greater_than"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "less_than",
                  "much_less_than"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than",
                  "approximately_equal"
                ]
              },
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal"
                ]
              },
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal",
                  "greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_exhaustion_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_naive_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_proliferation_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_expansion_richness_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_multichain_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_intermediate_gzmk_overgeneralized",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_melanocytic_conjugate_overcall",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "values": [
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_relations_mass_hedge",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              }
            }
          }
        ]
      },
      "trajectoryAvailable": true,
      "runId": "claude-opus-4-7__claude-code__10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1__r3",
      "trajectoryFile": "scbench-long/trajectories/10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1__claude-opus-4-7__claude-code__r3.json"
    },
    {
      "modelId": "sclong-glm-5p2-pi",
      "modelName": "glm-5p2",
      "harness": "pi",
      "provider": "Z.ai",
      "available": true,
      "task": "10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 2,
      "passed": true,
      "score": 15,
      "durationS": 621.27,
      "cost": 0.32935208000000005,
      "nTurns": 37,
      "reasoning": "all_of [pass_rule=all]: PASS\n  + list_match: list_match (answer_field='cd8_states_enriched_in_singlet_fraction'): PASS  (score=3.0)\n  + list_match: list_match (answer_field='cd8_states_enriched_in_tumor_cluster_fraction'): PASS  (score=3.0)\n  + clonotype_expansion_direction_data_true: passed=True, score=1.0\n  + biological_inference_data_true: passed=True, score=1.0\n  + dict_match: dict_match (answer_field='biological_relations'): PASS  (score=7.0)\n  + hard_fail 'singlet_top3_contains_naive_state_prior_pushing': not triggered\n  + hard_fail 'tumor_cluster_top3_contains_exhausted_state_prior_pushing': not triggered\n  + hard_fail 'singlet_top3_contains_decoy_token': not triggered\n  + hard_fail 'tumor_cluster_top3_contains_decoy_token': not triggered\n  + hard_fail 'TOXhigh_Tex_wrong_functional_category': not triggered\n  + hard_fail 'TCF7_stem_like_Tex_wrong_functional_category': not triggered\n  + hard_fail 'MKI67_state_wrong_functional_category': not triggered\n  + hard_fail 'Tn_or_Tn_Tmem_wrong_functional_category': not triggered\n  + hard_fail 'missing_exhausted_or_proliferating_anchor_in_singlet_top3': not triggered\n  + hard_fail 'missing_stemlike_precursor_in_singlet_top3': not triggered\n  + hard_fail 'missing_naive_or_early_memory_anchor_in_tumor_cluster_top3': not triggered\n  + hard_fail 'cf_reported_canonical_expansion_direction': not triggered\n  + hard_fail 'cf_expansion_lazy_defer': not triggered\n  + hard_fail 'cf_reported_canonical_expected_biology': not triggered\n  + hard_fail 'cf_biological_inference_null_artifact_or_lineage': not triggered\n  + hard_fail 'cf_exhaustion_inverted_prior_push': not triggered\n  + hard_fail 'cf_naive_inverted_prior_push': not triggered\n  + hard_fail 'cf_proliferation_inverted_prior_push': not triggered\n  + hard_fail 'cf_expansion_richness_inverted_prior_push': not triggered\n  + hard_fail 'cf_multichain_inverted_prior_push': not triggered\n  + hard_fail 'cf_intermediate_gzmk_overgeneralized': not triggered\n  + hard_fail 'cf_melanocytic_conjugate_overcall': not triggered\n  + hard_fail 'cf_relations_mass_hedge': not triggered",
      "metrics": {
        "hard_fail_triggered": [],
        "pass_rule": "all",
        "scoring_count": 5,
        "scoring_passed": 5,
        "scoring_total_score": 15
      },
      "fieldScores": {
        "biological_inference_data_true": 1,
        "clonotype_expansion_direction_data_true": 1,
        "dict_match": 7,
        "list_match": 3
      },
      "graderChecks": [
        {
          "name": "cd8_states_enriched_in_singlet_fraction",
          "kind": "list_match",
          "passed": true,
          "score": 3
        },
        {
          "name": "cd8_states_enriched_in_tumor_cluster_fraction",
          "kind": "list_match",
          "passed": true,
          "score": 3
        },
        {
          "name": "clonotype_expansion_direction_data_true",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "biological_inference_data_true",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "biological_relations",
          "kind": "list_match",
          "passed": true,
          "score": 7
        },
        {
          "name": "singlet_top3_contains_naive_state_prior_pushing",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "singlet_top3_contains_decoy_token",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "tumor_cluster_top3_contains_decoy_token",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "TOXhigh_Tex_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "TCF7_stem_like_Tex_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "MKI67_state_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "Tn_or_Tn_Tmem_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_stemlike_precursor_in_singlet_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_reported_canonical_expansion_direction",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_expansion_lazy_defer",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_reported_canonical_expected_biology",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_biological_inference_null_artifact_or_lineage",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_exhaustion_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_naive_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_proliferation_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_expansion_richness_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_multichain_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_intermediate_gzmk_overgeneralized",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_melanocytic_conjugate_overcall",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_relations_mass_hedge",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "biological_inference": "singlet_antigen_experienced_cluster_bystander",
        "biological_relations": {
          "clonal_expansion_richness_in_singlets_vs_tumor_cluster": "greater_than",
          "exhaustion_program_in_singlets_vs_tumor_cluster": "greater_than",
          "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": "approximately_equal",
          "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": "approximately_equal",
          "multichain_tcr_rate_in_singlets_vs_tumor_cluster": "greater_than",
          "naive_memory_program_in_singlets_vs_tumor_cluster": "less_than",
          "proliferation_program_in_singlets_vs_tumor_cluster": "greater_than"
        },
        "cd8_states_enriched_in_singlet_fraction": [
          {
            "functional_category": "stem_like_exhausted",
            "state_label": "TCF7_stem_like_Tex"
          },
          {
            "functional_category": "intermediate_exhausted_gzmk",
            "state_label": "GZMKhigh_Tex"
          },
          {
            "functional_category": "terminal_exhausted",
            "state_label": "TOXhigh_Tex"
          }
        ],
        "cd8_states_enriched_in_tumor_cluster_fraction": [
          {
            "functional_category": "naive_memory",
            "state_label": "Tn"
          },
          {
            "functional_category": "early_effector_memory",
            "state_label": "early_Tem_GZMK"
          },
          {
            "functional_category": "naive_memory",
            "state_label": "Tn_Tmem"
          }
        ],
        "clonotype_expansion_finding": "singlet_modestly_more_expanded_than_cluster"
      },
      "checkpointSignals": {
        "clustering": false,
        "markerAnnotation": true,
        "scoreBasedAnnotation": true,
        "normalization": false,
        "nerveJointAnchor": false,
        "multiomeAssembly": true,
        "depthNormalizedAtac": false,
        "regulonInference": true,
        "cd8SubsetIntegration": true,
        "perPatientEnrichment": true,
        "tcrRichnessMetrics": true,
        "orthologProjection": false,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": true,
        "mousePseudobulkDesign": false,
        "cancerEpithelialDenominator": false,
        "geneLevelDe": false,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_singlet_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "TOXhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "LAG3high_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "GZMKhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "intermediate_exhausted_gzmk"
                    ],
                    "canonical": "intermediate_exhausted_gzmk"
                  }
                }
              },
              {
                "key": "TCF7_stem_like_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "stem_like_exhausted"
                    ],
                    "canonical": "stem_like_exhausted"
                  }
                }
              },
              {
                "key": "MKI67_pos_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              },
              {
                "key": "MKI67high_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_tumor_cluster_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "Tn",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "Tn_Tmem",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "early_Tem_GZMK",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              },
              {
                "key": "Tem_GZMH",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              }
            ]
          },
          {
            "name": "clonotype_expansion_direction_data_true",
            "role": "gate",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_substantially_more_expanded_than_cluster",
              "singlet_modestly_more_expanded_than_cluster"
            ]
          },
          {
            "name": "biological_inference_data_true",
            "role": "gate",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_antigen_experienced_cluster_bystander"
            ]
          },
          {
            "name": "singlet_top3_contains_naive_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Tn",
              "Tn_Tmem",
              "early_Tem_GZMK",
              "Tem_GZMH"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "TOXhigh_Tex",
              "LAG3high_Tex",
              "GZMKhigh_Tex",
              "TCF7_stem_like_Tex",
              "MKI67_pos_Tex_Tprol",
              "MKI67high_Tex_Tprol"
            ]
          },
          {
            "name": "singlet_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "TOXhigh_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "TCF7_stem_like_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "MKI67_state_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "Tn_or_Tn_Tmem_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_stemlike_precursor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_reported_canonical_expansion_direction",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_substantially_more_expanded_than_singlet",
              "cluster_modestly_more_expanded_than_singlet"
            ]
          },
          {
            "name": "cf_expansion_lazy_defer",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "expansion_pattern_not_detectable"
            ]
          },
          {
            "name": "cf_reported_canonical_expected_biology",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_antigen_experienced_singlet_bystander"
            ]
          },
          {
            "name": "cf_biological_inference_null_artifact_or_lineage",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "differences_reflect_dissociation_or_sorting_artifact",
              "cluster_and_singlet_capture_same_biology",
              "cluster_and_singlet_capture_distinct_cd8_lineages"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "biological_relations",
            "kind": "dict",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_greater_than",
                  "greater_than"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "less_than",
                  "much_less_than"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than",
                  "approximately_equal"
                ]
              },
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal"
                ]
              },
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal",
                  "greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_exhaustion_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_naive_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_proliferation_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_expansion_richness_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_multichain_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_intermediate_gzmk_overgeneralized",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_melanocytic_conjugate_overcall",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "values": [
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_relations_mass_hedge",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              }
            }
          }
        ]
      },
      "trajectoryAvailable": true,
      "runId": "glm-5p2__pi__10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1__r2",
      "trajectoryFile": "scbench-long/trajectories/10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1__glm-5p2__pi__r2.json"
    },
    {
      "modelId": "sclong-opus-4-7-pi",
      "modelName": "claude-opus-4-7",
      "harness": "pi",
      "provider": "Anthropic",
      "available": true,
      "task": "10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": true,
      "score": 15,
      "durationS": 2840.03,
      "cost": null,
      "nTurns": 42,
      "reasoning": "all_of [pass_rule=all]: PASS\n  + list_match: list_match (answer_field='cd8_states_enriched_in_singlet_fraction'): PASS  (score=3.0)\n  + list_match: list_match (answer_field='cd8_states_enriched_in_tumor_cluster_fraction'): PASS  (score=3.0)\n  + clonotype_expansion_direction_data_true: passed=True, score=1.0\n  + biological_inference_data_true: passed=True, score=1.0\n  + dict_match: dict_match (answer_field='biological_relations'): PASS  (score=7.0)\n  + hard_fail 'singlet_top3_contains_naive_state_prior_pushing': not triggered\n  + hard_fail 'tumor_cluster_top3_contains_exhausted_state_prior_pushing': not triggered\n  + hard_fail 'singlet_top3_contains_decoy_token': not triggered\n  + hard_fail 'tumor_cluster_top3_contains_decoy_token': not triggered\n  + hard_fail 'TOXhigh_Tex_wrong_functional_category': not triggered\n  + hard_fail 'TCF7_stem_like_Tex_wrong_functional_category': not triggered\n  + hard_fail 'MKI67_state_wrong_functional_category': not triggered\n  + hard_fail 'Tn_or_Tn_Tmem_wrong_functional_category': not triggered\n  + hard_fail 'missing_exhausted_or_proliferating_anchor_in_singlet_top3': not triggered\n  + hard_fail 'missing_stemlike_precursor_in_singlet_top3': not triggered\n  + hard_fail 'missing_naive_or_early_memory_anchor_in_tumor_cluster_top3': not triggered\n  + hard_fail 'cf_reported_canonical_expansion_direction': not triggered\n  + hard_fail 'cf_expansion_lazy_defer': not triggered\n  + hard_fail 'cf_reported_canonical_expected_biology': not triggered\n  + hard_fail 'cf_biological_inference_null_artifact_or_lineage': not triggered\n  + hard_fail 'cf_exhaustion_inverted_prior_push': not triggered\n  + hard_fail 'cf_naive_inverted_prior_push': not triggered\n  + hard_fail 'cf_proliferation_inverted_prior_push': not triggered\n  + hard_fail 'cf_expansion_richness_inverted_prior_push': not triggered\n  + hard_fail 'cf_multichain_inverted_prior_push': not triggered\n  + hard_fail 'cf_intermediate_gzmk_overgeneralized': not triggered\n  + hard_fail 'cf_melanocytic_conjugate_overcall': not triggered\n  + hard_fail 'cf_relations_mass_hedge': not triggered",
      "metrics": {
        "hard_fail_triggered": [],
        "pass_rule": "all",
        "scoring_count": 5,
        "scoring_passed": 5,
        "scoring_total_score": 15
      },
      "fieldScores": {
        "biological_inference_data_true": 1,
        "clonotype_expansion_direction_data_true": 1,
        "dict_match": 7,
        "list_match": 3
      },
      "graderChecks": [
        {
          "name": "cd8_states_enriched_in_singlet_fraction",
          "kind": "list_match",
          "passed": true,
          "score": 3
        },
        {
          "name": "cd8_states_enriched_in_tumor_cluster_fraction",
          "kind": "list_match",
          "passed": true,
          "score": 3
        },
        {
          "name": "clonotype_expansion_direction_data_true",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "biological_inference_data_true",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "biological_relations",
          "kind": "list_match",
          "passed": true,
          "score": 7
        },
        {
          "name": "singlet_top3_contains_naive_state_prior_pushing",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "singlet_top3_contains_decoy_token",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "tumor_cluster_top3_contains_decoy_token",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "TOXhigh_Tex_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "TCF7_stem_like_Tex_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "MKI67_state_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "Tn_or_Tn_Tmem_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_stemlike_precursor_in_singlet_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_reported_canonical_expansion_direction",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_expansion_lazy_defer",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_reported_canonical_expected_biology",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_biological_inference_null_artifact_or_lineage",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_exhaustion_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_naive_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_proliferation_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_expansion_richness_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_multichain_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_intermediate_gzmk_overgeneralized",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_melanocytic_conjugate_overcall",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_relations_mass_hedge",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "biological_inference": "singlet_antigen_experienced_cluster_bystander",
        "biological_relations": {
          "clonal_expansion_richness_in_singlets_vs_tumor_cluster": "greater_than",
          "exhaustion_program_in_singlets_vs_tumor_cluster": "greater_than",
          "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": "approximately_equal",
          "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": "greater_than",
          "multichain_tcr_rate_in_singlets_vs_tumor_cluster": "greater_than",
          "naive_memory_program_in_singlets_vs_tumor_cluster": "less_than",
          "proliferation_program_in_singlets_vs_tumor_cluster": "greater_than"
        },
        "cd8_states_enriched_in_singlet_fraction": [
          {
            "functional_category": "stem_like_exhausted",
            "state_label": "TCF7_stem_like_Tex"
          },
          {
            "functional_category": "intermediate_exhausted_gzmk",
            "state_label": "GZMKhigh_Tex"
          },
          {
            "functional_category": "terminal_exhausted",
            "state_label": "TOXhigh_Tex"
          }
        ],
        "cd8_states_enriched_in_tumor_cluster_fraction": [
          {
            "functional_category": "naive_memory",
            "state_label": "Tn"
          },
          {
            "functional_category": "naive_memory",
            "state_label": "Tn_Tmem"
          },
          {
            "functional_category": "early_effector_memory",
            "state_label": "early_Tem_GZMK"
          }
        ],
        "clonotype_expansion_finding": "singlet_modestly_more_expanded_than_cluster"
      },
      "checkpointSignals": {
        "clustering": true,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": false,
        "nerveJointAnchor": false,
        "multiomeAssembly": false,
        "depthNormalizedAtac": false,
        "regulonInference": false,
        "cd8SubsetIntegration": true,
        "perPatientEnrichment": true,
        "tcrRichnessMetrics": true,
        "orthologProjection": false,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": true,
        "mousePseudobulkDesign": false,
        "cancerEpithelialDenominator": false,
        "geneLevelDe": false,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_singlet_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "TOXhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "LAG3high_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "GZMKhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "intermediate_exhausted_gzmk"
                    ],
                    "canonical": "intermediate_exhausted_gzmk"
                  }
                }
              },
              {
                "key": "TCF7_stem_like_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "stem_like_exhausted"
                    ],
                    "canonical": "stem_like_exhausted"
                  }
                }
              },
              {
                "key": "MKI67_pos_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              },
              {
                "key": "MKI67high_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_tumor_cluster_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "Tn",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "Tn_Tmem",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "early_Tem_GZMK",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              },
              {
                "key": "Tem_GZMH",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              }
            ]
          },
          {
            "name": "clonotype_expansion_direction_data_true",
            "role": "gate",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_substantially_more_expanded_than_cluster",
              "singlet_modestly_more_expanded_than_cluster"
            ]
          },
          {
            "name": "biological_inference_data_true",
            "role": "gate",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_antigen_experienced_cluster_bystander"
            ]
          },
          {
            "name": "singlet_top3_contains_naive_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Tn",
              "Tn_Tmem",
              "early_Tem_GZMK",
              "Tem_GZMH"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "TOXhigh_Tex",
              "LAG3high_Tex",
              "GZMKhigh_Tex",
              "TCF7_stem_like_Tex",
              "MKI67_pos_Tex_Tprol",
              "MKI67high_Tex_Tprol"
            ]
          },
          {
            "name": "singlet_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "TOXhigh_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "TCF7_stem_like_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "MKI67_state_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "Tn_or_Tn_Tmem_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_stemlike_precursor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_reported_canonical_expansion_direction",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_substantially_more_expanded_than_singlet",
              "cluster_modestly_more_expanded_than_singlet"
            ]
          },
          {
            "name": "cf_expansion_lazy_defer",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "expansion_pattern_not_detectable"
            ]
          },
          {
            "name": "cf_reported_canonical_expected_biology",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_antigen_experienced_singlet_bystander"
            ]
          },
          {
            "name": "cf_biological_inference_null_artifact_or_lineage",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "differences_reflect_dissociation_or_sorting_artifact",
              "cluster_and_singlet_capture_same_biology",
              "cluster_and_singlet_capture_distinct_cd8_lineages"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "biological_relations",
            "kind": "dict",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_greater_than",
                  "greater_than"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "less_than",
                  "much_less_than"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than",
                  "approximately_equal"
                ]
              },
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal"
                ]
              },
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal",
                  "greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_exhaustion_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_naive_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_proliferation_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_expansion_richness_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_multichain_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_intermediate_gzmk_overgeneralized",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_melanocytic_conjugate_overcall",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "values": [
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_relations_mass_hedge",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              }
            }
          }
        ]
      },
      "trajectoryAvailable": true,
      "runId": "claude-opus-4-7__pi__10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1__r3",
      "trajectoryFile": "scbench-long/trajectories/10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1__claude-opus-4-7__pi__r3.json"
    },
    {
      "modelId": "sclong-opus-4-6-pi",
      "modelName": "claude-opus-4-6",
      "harness": "pi",
      "provider": "Anthropic",
      "available": true,
      "task": "10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": true,
      "score": 15,
      "durationS": 1889.35,
      "cost": 2.64664875,
      "nTurns": 25,
      "reasoning": "all_of [pass_rule=all]: PASS\n  + list_match: list_match (answer_field='cd8_states_enriched_in_singlet_fraction'): PASS  (score=3.0)\n  + list_match: list_match (answer_field='cd8_states_enriched_in_tumor_cluster_fraction'): PASS  (score=3.0)\n  + clonotype_expansion_direction_data_true: passed=True, score=1.0\n  + biological_inference_data_true: passed=True, score=1.0\n  + dict_match: dict_match (answer_field='biological_relations'): PASS  (score=7.0)\n  + hard_fail 'singlet_top3_contains_naive_state_prior_pushing': not triggered\n  + hard_fail 'tumor_cluster_top3_contains_exhausted_state_prior_pushing': not triggered\n  + hard_fail 'singlet_top3_contains_decoy_token': not triggered\n  + hard_fail 'tumor_cluster_top3_contains_decoy_token': not triggered\n  + hard_fail 'TOXhigh_Tex_wrong_functional_category': not triggered\n  + hard_fail 'TCF7_stem_like_Tex_wrong_functional_category': not triggered\n  + hard_fail 'MKI67_state_wrong_functional_category': not triggered\n  + hard_fail 'Tn_or_Tn_Tmem_wrong_functional_category': not triggered\n  + hard_fail 'missing_exhausted_or_proliferating_anchor_in_singlet_top3': not triggered\n  + hard_fail 'missing_stemlike_precursor_in_singlet_top3': not triggered\n  + hard_fail 'missing_naive_or_early_memory_anchor_in_tumor_cluster_top3': not triggered\n  + hard_fail 'cf_reported_canonical_expansion_direction': not triggered\n  + hard_fail 'cf_expansion_lazy_defer': not triggered\n  + hard_fail 'cf_reported_canonical_expected_biology': not triggered\n  + hard_fail 'cf_biological_inference_null_artifact_or_lineage': not triggered\n  + hard_fail 'cf_exhaustion_inverted_prior_push': not triggered\n  + hard_fail 'cf_naive_inverted_prior_push': not triggered\n  + hard_fail 'cf_proliferation_inverted_prior_push': not triggered\n  + hard_fail 'cf_expansion_richness_inverted_prior_push': not triggered\n  + hard_fail 'cf_multichain_inverted_prior_push': not triggered\n  + hard_fail 'cf_intermediate_gzmk_overgeneralized': not triggered\n  + hard_fail 'cf_melanocytic_conjugate_overcall': not triggered\n  + hard_fail 'cf_relations_mass_hedge': not triggered",
      "metrics": {
        "hard_fail_triggered": [],
        "pass_rule": "all",
        "scoring_count": 5,
        "scoring_passed": 5,
        "scoring_total_score": 15
      },
      "fieldScores": {
        "biological_inference_data_true": 1,
        "clonotype_expansion_direction_data_true": 1,
        "dict_match": 7,
        "list_match": 3
      },
      "graderChecks": [
        {
          "name": "cd8_states_enriched_in_singlet_fraction",
          "kind": "list_match",
          "passed": true,
          "score": 3
        },
        {
          "name": "cd8_states_enriched_in_tumor_cluster_fraction",
          "kind": "list_match",
          "passed": true,
          "score": 3
        },
        {
          "name": "clonotype_expansion_direction_data_true",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "biological_inference_data_true",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "biological_relations",
          "kind": "list_match",
          "passed": true,
          "score": 7
        },
        {
          "name": "singlet_top3_contains_naive_state_prior_pushing",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "singlet_top3_contains_decoy_token",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "tumor_cluster_top3_contains_decoy_token",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "TOXhigh_Tex_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "TCF7_stem_like_Tex_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "MKI67_state_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "Tn_or_Tn_Tmem_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_stemlike_precursor_in_singlet_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_reported_canonical_expansion_direction",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_expansion_lazy_defer",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_reported_canonical_expected_biology",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_biological_inference_null_artifact_or_lineage",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_exhaustion_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_naive_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_proliferation_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_expansion_richness_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_multichain_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_intermediate_gzmk_overgeneralized",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_melanocytic_conjugate_overcall",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_relations_mass_hedge",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "biological_inference": "singlet_antigen_experienced_cluster_bystander",
        "biological_relations": {
          "clonal_expansion_richness_in_singlets_vs_tumor_cluster": "greater_than",
          "exhaustion_program_in_singlets_vs_tumor_cluster": "greater_than",
          "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": "approximately_equal",
          "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": "greater_than",
          "multichain_tcr_rate_in_singlets_vs_tumor_cluster": "greater_than",
          "naive_memory_program_in_singlets_vs_tumor_cluster": "less_than",
          "proliferation_program_in_singlets_vs_tumor_cluster": "greater_than"
        },
        "cd8_states_enriched_in_singlet_fraction": [
          {
            "functional_category": "stem_like_exhausted",
            "state_label": "TCF7_stem_like_Tex"
          },
          {
            "functional_category": "terminal_exhausted",
            "state_label": "LAG3high_Tex"
          },
          {
            "functional_category": "proliferating",
            "state_label": "MKI67high_Tex_Tprol"
          }
        ],
        "cd8_states_enriched_in_tumor_cluster_fraction": [
          {
            "functional_category": "naive_memory",
            "state_label": "Tn"
          },
          {
            "functional_category": "naive_memory",
            "state_label": "Tn_Tmem"
          },
          {
            "functional_category": "early_effector_memory",
            "state_label": "early_Tem_GZMK"
          }
        ],
        "clonotype_expansion_finding": "singlet_modestly_more_expanded_than_cluster"
      },
      "checkpointSignals": {
        "clustering": true,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": false,
        "nerveJointAnchor": false,
        "multiomeAssembly": false,
        "depthNormalizedAtac": false,
        "regulonInference": false,
        "cd8SubsetIntegration": true,
        "perPatientEnrichment": true,
        "tcrRichnessMetrics": true,
        "orthologProjection": false,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": true,
        "mousePseudobulkDesign": false,
        "cancerEpithelialDenominator": false,
        "geneLevelDe": false,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_singlet_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "TOXhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "LAG3high_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "GZMKhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "intermediate_exhausted_gzmk"
                    ],
                    "canonical": "intermediate_exhausted_gzmk"
                  }
                }
              },
              {
                "key": "TCF7_stem_like_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "stem_like_exhausted"
                    ],
                    "canonical": "stem_like_exhausted"
                  }
                }
              },
              {
                "key": "MKI67_pos_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              },
              {
                "key": "MKI67high_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_tumor_cluster_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "Tn",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "Tn_Tmem",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "early_Tem_GZMK",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              },
              {
                "key": "Tem_GZMH",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              }
            ]
          },
          {
            "name": "clonotype_expansion_direction_data_true",
            "role": "gate",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_substantially_more_expanded_than_cluster",
              "singlet_modestly_more_expanded_than_cluster"
            ]
          },
          {
            "name": "biological_inference_data_true",
            "role": "gate",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_antigen_experienced_cluster_bystander"
            ]
          },
          {
            "name": "singlet_top3_contains_naive_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Tn",
              "Tn_Tmem",
              "early_Tem_GZMK",
              "Tem_GZMH"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "TOXhigh_Tex",
              "LAG3high_Tex",
              "GZMKhigh_Tex",
              "TCF7_stem_like_Tex",
              "MKI67_pos_Tex_Tprol",
              "MKI67high_Tex_Tprol"
            ]
          },
          {
            "name": "singlet_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "TOXhigh_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "TCF7_stem_like_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "MKI67_state_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "Tn_or_Tn_Tmem_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_stemlike_precursor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_reported_canonical_expansion_direction",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_substantially_more_expanded_than_singlet",
              "cluster_modestly_more_expanded_than_singlet"
            ]
          },
          {
            "name": "cf_expansion_lazy_defer",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "expansion_pattern_not_detectable"
            ]
          },
          {
            "name": "cf_reported_canonical_expected_biology",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_antigen_experienced_singlet_bystander"
            ]
          },
          {
            "name": "cf_biological_inference_null_artifact_or_lineage",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "differences_reflect_dissociation_or_sorting_artifact",
              "cluster_and_singlet_capture_same_biology",
              "cluster_and_singlet_capture_distinct_cd8_lineages"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "biological_relations",
            "kind": "dict",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_greater_than",
                  "greater_than"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "less_than",
                  "much_less_than"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than",
                  "approximately_equal"
                ]
              },
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal"
                ]
              },
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal",
                  "greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_exhaustion_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_naive_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_proliferation_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_expansion_richness_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_multichain_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_intermediate_gzmk_overgeneralized",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_melanocytic_conjugate_overcall",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "values": [
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_relations_mass_hedge",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              }
            }
          }
        ]
      },
      "trajectoryAvailable": true,
      "runId": "claude-opus-4-6__pi__10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1__r3",
      "trajectoryFile": "scbench-long/trajectories/10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1__claude-opus-4-6__pi__r3.json"
    },
    {
      "modelId": "sclong-gpt-5-4-pi",
      "modelName": "gpt-5.4",
      "harness": "pi",
      "provider": "OpenAI",
      "available": true,
      "task": "10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": true,
      "score": 15,
      "durationS": 1589,
      "cost": 2.3057044999999996,
      "nTurns": 50,
      "reasoning": "all_of [pass_rule=all]: PASS\n  + list_match: list_match (answer_field='cd8_states_enriched_in_singlet_fraction'): PASS  (score=3.0)\n  + list_match: list_match (answer_field='cd8_states_enriched_in_tumor_cluster_fraction'): PASS  (score=3.0)\n  + clonotype_expansion_direction_data_true: passed=True, score=1.0\n  + biological_inference_data_true: passed=True, score=1.0\n  + dict_match: dict_match (answer_field='biological_relations'): PASS  (score=7.0)\n  + hard_fail 'singlet_top3_contains_naive_state_prior_pushing': not triggered\n  + hard_fail 'tumor_cluster_top3_contains_exhausted_state_prior_pushing': not triggered\n  + hard_fail 'singlet_top3_contains_decoy_token': not triggered\n  + hard_fail 'tumor_cluster_top3_contains_decoy_token': not triggered\n  + hard_fail 'TOXhigh_Tex_wrong_functional_category': not triggered\n  + hard_fail 'TCF7_stem_like_Tex_wrong_functional_category': not triggered\n  + hard_fail 'MKI67_state_wrong_functional_category': not triggered\n  + hard_fail 'Tn_or_Tn_Tmem_wrong_functional_category': not triggered\n  + hard_fail 'missing_exhausted_or_proliferating_anchor_in_singlet_top3': not triggered\n  + hard_fail 'missing_stemlike_precursor_in_singlet_top3': not triggered\n  + hard_fail 'missing_naive_or_early_memory_anchor_in_tumor_cluster_top3': not triggered\n  + hard_fail 'cf_reported_canonical_expansion_direction': not triggered\n  + hard_fail 'cf_expansion_lazy_defer': not triggered\n  + hard_fail 'cf_reported_canonical_expected_biology': not triggered\n  + hard_fail 'cf_biological_inference_null_artifact_or_lineage': not triggered\n  + hard_fail 'cf_exhaustion_inverted_prior_push': not triggered\n  + hard_fail 'cf_naive_inverted_prior_push': not triggered\n  + hard_fail 'cf_proliferation_inverted_prior_push': not triggered\n  + hard_fail 'cf_expansion_richness_inverted_prior_push': not triggered\n  + hard_fail 'cf_multichain_inverted_prior_push': not triggered\n  + hard_fail 'cf_intermediate_gzmk_overgeneralized': not triggered\n  + hard_fail 'cf_melanocytic_conjugate_overcall': not triggered\n  + hard_fail 'cf_relations_mass_hedge': not triggered",
      "metrics": {
        "hard_fail_triggered": [],
        "pass_rule": "all",
        "scoring_count": 5,
        "scoring_passed": 5,
        "scoring_total_score": 15
      },
      "fieldScores": {
        "biological_inference_data_true": 1,
        "clonotype_expansion_direction_data_true": 1,
        "dict_match": 7,
        "list_match": 3
      },
      "graderChecks": [
        {
          "name": "cd8_states_enriched_in_singlet_fraction",
          "kind": "list_match",
          "passed": true,
          "score": 3
        },
        {
          "name": "cd8_states_enriched_in_tumor_cluster_fraction",
          "kind": "list_match",
          "passed": true,
          "score": 3
        },
        {
          "name": "clonotype_expansion_direction_data_true",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "biological_inference_data_true",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "biological_relations",
          "kind": "list_match",
          "passed": true,
          "score": 7
        },
        {
          "name": "singlet_top3_contains_naive_state_prior_pushing",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "singlet_top3_contains_decoy_token",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "tumor_cluster_top3_contains_decoy_token",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "TOXhigh_Tex_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "TCF7_stem_like_Tex_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "MKI67_state_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "Tn_or_Tn_Tmem_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_stemlike_precursor_in_singlet_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_reported_canonical_expansion_direction",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_expansion_lazy_defer",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_reported_canonical_expected_biology",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_biological_inference_null_artifact_or_lineage",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_exhaustion_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_naive_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_proliferation_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_expansion_richness_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_multichain_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_intermediate_gzmk_overgeneralized",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_melanocytic_conjugate_overcall",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_relations_mass_hedge",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "biological_inference": "singlet_antigen_experienced_cluster_bystander",
        "biological_relations": {
          "clonal_expansion_richness_in_singlets_vs_tumor_cluster": "greater_than",
          "exhaustion_program_in_singlets_vs_tumor_cluster": "greater_than",
          "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": "approximately_equal",
          "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": "approximately_equal",
          "multichain_tcr_rate_in_singlets_vs_tumor_cluster": "approximately_equal",
          "naive_memory_program_in_singlets_vs_tumor_cluster": "less_than",
          "proliferation_program_in_singlets_vs_tumor_cluster": "greater_than"
        },
        "cd8_states_enriched_in_singlet_fraction": [
          {
            "functional_category": "stem_like_exhausted",
            "state_label": "TCF7_stem_like_Tex"
          },
          {
            "functional_category": "intermediate_exhausted_gzmk",
            "state_label": "GZMKhigh_Tex"
          },
          {
            "functional_category": "terminal_exhausted",
            "state_label": "LAG3high_Tex"
          }
        ],
        "cd8_states_enriched_in_tumor_cluster_fraction": [
          {
            "functional_category": "naive_memory",
            "state_label": "Tn"
          },
          {
            "functional_category": "naive_memory",
            "state_label": "Tn_Tmem"
          },
          {
            "functional_category": "early_effector_memory",
            "state_label": "early_Tem_GZMK"
          }
        ],
        "clonotype_expansion_finding": "singlet_modestly_more_expanded_than_cluster"
      },
      "checkpointSignals": {
        "clustering": true,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": true,
        "nerveJointAnchor": false,
        "multiomeAssembly": true,
        "depthNormalizedAtac": false,
        "regulonInference": true,
        "cd8SubsetIntegration": true,
        "perPatientEnrichment": true,
        "tcrRichnessMetrics": true,
        "orthologProjection": false,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": true,
        "mousePseudobulkDesign": false,
        "cancerEpithelialDenominator": false,
        "geneLevelDe": false,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_singlet_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "TOXhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "LAG3high_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "GZMKhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "intermediate_exhausted_gzmk"
                    ],
                    "canonical": "intermediate_exhausted_gzmk"
                  }
                }
              },
              {
                "key": "TCF7_stem_like_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "stem_like_exhausted"
                    ],
                    "canonical": "stem_like_exhausted"
                  }
                }
              },
              {
                "key": "MKI67_pos_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              },
              {
                "key": "MKI67high_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_tumor_cluster_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "Tn",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "Tn_Tmem",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "early_Tem_GZMK",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              },
              {
                "key": "Tem_GZMH",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              }
            ]
          },
          {
            "name": "clonotype_expansion_direction_data_true",
            "role": "gate",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_substantially_more_expanded_than_cluster",
              "singlet_modestly_more_expanded_than_cluster"
            ]
          },
          {
            "name": "biological_inference_data_true",
            "role": "gate",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_antigen_experienced_cluster_bystander"
            ]
          },
          {
            "name": "singlet_top3_contains_naive_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Tn",
              "Tn_Tmem",
              "early_Tem_GZMK",
              "Tem_GZMH"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "TOXhigh_Tex",
              "LAG3high_Tex",
              "GZMKhigh_Tex",
              "TCF7_stem_like_Tex",
              "MKI67_pos_Tex_Tprol",
              "MKI67high_Tex_Tprol"
            ]
          },
          {
            "name": "singlet_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "TOXhigh_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "TCF7_stem_like_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "MKI67_state_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "Tn_or_Tn_Tmem_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_stemlike_precursor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_reported_canonical_expansion_direction",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_substantially_more_expanded_than_singlet",
              "cluster_modestly_more_expanded_than_singlet"
            ]
          },
          {
            "name": "cf_expansion_lazy_defer",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "expansion_pattern_not_detectable"
            ]
          },
          {
            "name": "cf_reported_canonical_expected_biology",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_antigen_experienced_singlet_bystander"
            ]
          },
          {
            "name": "cf_biological_inference_null_artifact_or_lineage",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "differences_reflect_dissociation_or_sorting_artifact",
              "cluster_and_singlet_capture_same_biology",
              "cluster_and_singlet_capture_distinct_cd8_lineages"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "biological_relations",
            "kind": "dict",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_greater_than",
                  "greater_than"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "less_than",
                  "much_less_than"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than",
                  "approximately_equal"
                ]
              },
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal"
                ]
              },
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal",
                  "greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_exhaustion_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_naive_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_proliferation_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_expansion_richness_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_multichain_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_intermediate_gzmk_overgeneralized",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_melanocytic_conjugate_overcall",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "values": [
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_relations_mass_hedge",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              }
            }
          }
        ]
      },
      "trajectoryAvailable": true,
      "runId": "gpt-5.4__pi__10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1__r3",
      "trajectoryFile": "scbench-long/trajectories/10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1__gpt-5.4__pi__r3.json"
    },
    {
      "modelId": "sclong-opus-4-6-claude-code",
      "modelName": "claude-opus-4-6",
      "harness": "claude-code",
      "provider": "Anthropic",
      "available": true,
      "task": "10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 1,
      "passed": true,
      "score": 15,
      "durationS": 1027.28,
      "cost": 1.6065779999999998,
      "nTurns": 17,
      "reasoning": "all_of [pass_rule=all]: PASS\n  + list_match: list_match (answer_field='cd8_states_enriched_in_singlet_fraction'): PASS  (score=3.0)\n  + list_match: list_match (answer_field='cd8_states_enriched_in_tumor_cluster_fraction'): PASS  (score=3.0)\n  + clonotype_expansion_direction_data_true: passed=True, score=1.0\n  + biological_inference_data_true: passed=True, score=1.0\n  + dict_match: dict_match (answer_field='biological_relations'): PASS  (score=7.0)\n  + hard_fail 'singlet_top3_contains_naive_state_prior_pushing': not triggered\n  + hard_fail 'tumor_cluster_top3_contains_exhausted_state_prior_pushing': not triggered\n  + hard_fail 'singlet_top3_contains_decoy_token': not triggered\n  + hard_fail 'tumor_cluster_top3_contains_decoy_token': not triggered\n  + hard_fail 'TOXhigh_Tex_wrong_functional_category': not triggered\n  + hard_fail 'TCF7_stem_like_Tex_wrong_functional_category': not triggered\n  + hard_fail 'MKI67_state_wrong_functional_category': not triggered\n  + hard_fail 'Tn_or_Tn_Tmem_wrong_functional_category': not triggered\n  + hard_fail 'missing_exhausted_or_proliferating_anchor_in_singlet_top3': not triggered\n  + hard_fail 'missing_stemlike_precursor_in_singlet_top3': not triggered\n  + hard_fail 'missing_naive_or_early_memory_anchor_in_tumor_cluster_top3': not triggered\n  + hard_fail 'cf_reported_canonical_expansion_direction': not triggered\n  + hard_fail 'cf_expansion_lazy_defer': not triggered\n  + hard_fail 'cf_reported_canonical_expected_biology': not triggered\n  + hard_fail 'cf_biological_inference_null_artifact_or_lineage': not triggered\n  + hard_fail 'cf_exhaustion_inverted_prior_push': not triggered\n  + hard_fail 'cf_naive_inverted_prior_push': not triggered\n  + hard_fail 'cf_proliferation_inverted_prior_push': not triggered\n  + hard_fail 'cf_expansion_richness_inverted_prior_push': not triggered\n  + hard_fail 'cf_multichain_inverted_prior_push': not triggered\n  + hard_fail 'cf_intermediate_gzmk_overgeneralized': not triggered\n  + hard_fail 'cf_melanocytic_conjugate_overcall': not triggered\n  + hard_fail 'cf_relations_mass_hedge': not triggered",
      "metrics": {
        "hard_fail_triggered": [],
        "pass_rule": "all",
        "scoring_count": 5,
        "scoring_passed": 5,
        "scoring_total_score": 15
      },
      "fieldScores": {
        "biological_inference_data_true": 1,
        "clonotype_expansion_direction_data_true": 1,
        "dict_match": 7,
        "list_match": 3
      },
      "graderChecks": [
        {
          "name": "cd8_states_enriched_in_singlet_fraction",
          "kind": "list_match",
          "passed": true,
          "score": 3
        },
        {
          "name": "cd8_states_enriched_in_tumor_cluster_fraction",
          "kind": "list_match",
          "passed": true,
          "score": 3
        },
        {
          "name": "clonotype_expansion_direction_data_true",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "biological_inference_data_true",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "biological_relations",
          "kind": "list_match",
          "passed": true,
          "score": 7
        },
        {
          "name": "singlet_top3_contains_naive_state_prior_pushing",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "singlet_top3_contains_decoy_token",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "tumor_cluster_top3_contains_decoy_token",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "TOXhigh_Tex_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "TCF7_stem_like_Tex_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "MKI67_state_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "Tn_or_Tn_Tmem_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_stemlike_precursor_in_singlet_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_reported_canonical_expansion_direction",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_expansion_lazy_defer",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_reported_canonical_expected_biology",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_biological_inference_null_artifact_or_lineage",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_exhaustion_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_naive_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_proliferation_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_expansion_richness_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_multichain_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_intermediate_gzmk_overgeneralized",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_melanocytic_conjugate_overcall",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_relations_mass_hedge",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "biological_inference": "singlet_antigen_experienced_cluster_bystander",
        "biological_relations": {
          "clonal_expansion_richness_in_singlets_vs_tumor_cluster": "greater_than",
          "exhaustion_program_in_singlets_vs_tumor_cluster": "greater_than",
          "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": "approximately_equal",
          "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": "greater_than",
          "multichain_tcr_rate_in_singlets_vs_tumor_cluster": "greater_than",
          "naive_memory_program_in_singlets_vs_tumor_cluster": "less_than",
          "proliferation_program_in_singlets_vs_tumor_cluster": "greater_than"
        },
        "cd8_states_enriched_in_singlet_fraction": [
          {
            "functional_category": "stem_like_exhausted",
            "state_label": "TCF7_stem_like_Tex"
          },
          {
            "functional_category": "terminal_exhausted",
            "state_label": "TOXhigh_Tex"
          },
          {
            "functional_category": "proliferating",
            "state_label": "MKI67high_Tex_Tprol"
          }
        ],
        "cd8_states_enriched_in_tumor_cluster_fraction": [
          {
            "functional_category": "naive_memory",
            "state_label": "Tn"
          },
          {
            "functional_category": "naive_memory",
            "state_label": "Tn_Tmem"
          },
          {
            "functional_category": "early_effector_memory",
            "state_label": "early_Tem_GZMK"
          }
        ],
        "clonotype_expansion_finding": "singlet_modestly_more_expanded_than_cluster"
      },
      "checkpointSignals": {
        "clustering": true,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": false,
        "nerveJointAnchor": false,
        "multiomeAssembly": false,
        "depthNormalizedAtac": false,
        "regulonInference": false,
        "cd8SubsetIntegration": true,
        "perPatientEnrichment": true,
        "tcrRichnessMetrics": true,
        "orthologProjection": false,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": true,
        "mousePseudobulkDesign": false,
        "cancerEpithelialDenominator": false,
        "geneLevelDe": false,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_singlet_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "TOXhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "LAG3high_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "GZMKhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "intermediate_exhausted_gzmk"
                    ],
                    "canonical": "intermediate_exhausted_gzmk"
                  }
                }
              },
              {
                "key": "TCF7_stem_like_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "stem_like_exhausted"
                    ],
                    "canonical": "stem_like_exhausted"
                  }
                }
              },
              {
                "key": "MKI67_pos_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              },
              {
                "key": "MKI67high_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_tumor_cluster_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "Tn",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "Tn_Tmem",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "early_Tem_GZMK",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              },
              {
                "key": "Tem_GZMH",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              }
            ]
          },
          {
            "name": "clonotype_expansion_direction_data_true",
            "role": "gate",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_substantially_more_expanded_than_cluster",
              "singlet_modestly_more_expanded_than_cluster"
            ]
          },
          {
            "name": "biological_inference_data_true",
            "role": "gate",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_antigen_experienced_cluster_bystander"
            ]
          },
          {
            "name": "singlet_top3_contains_naive_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Tn",
              "Tn_Tmem",
              "early_Tem_GZMK",
              "Tem_GZMH"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "TOXhigh_Tex",
              "LAG3high_Tex",
              "GZMKhigh_Tex",
              "TCF7_stem_like_Tex",
              "MKI67_pos_Tex_Tprol",
              "MKI67high_Tex_Tprol"
            ]
          },
          {
            "name": "singlet_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "TOXhigh_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "TCF7_stem_like_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "MKI67_state_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "Tn_or_Tn_Tmem_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_stemlike_precursor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_reported_canonical_expansion_direction",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_substantially_more_expanded_than_singlet",
              "cluster_modestly_more_expanded_than_singlet"
            ]
          },
          {
            "name": "cf_expansion_lazy_defer",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "expansion_pattern_not_detectable"
            ]
          },
          {
            "name": "cf_reported_canonical_expected_biology",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_antigen_experienced_singlet_bystander"
            ]
          },
          {
            "name": "cf_biological_inference_null_artifact_or_lineage",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "differences_reflect_dissociation_or_sorting_artifact",
              "cluster_and_singlet_capture_same_biology",
              "cluster_and_singlet_capture_distinct_cd8_lineages"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "biological_relations",
            "kind": "dict",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_greater_than",
                  "greater_than"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "less_than",
                  "much_less_than"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than",
                  "approximately_equal"
                ]
              },
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal"
                ]
              },
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal",
                  "greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_exhaustion_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_naive_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_proliferation_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_expansion_richness_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_multichain_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_intermediate_gzmk_overgeneralized",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_melanocytic_conjugate_overcall",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "values": [
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_relations_mass_hedge",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              }
            }
          }
        ]
      },
      "trajectoryAvailable": true,
      "runId": "claude-opus-4-6__claude-code__10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1__r1",
      "trajectoryFile": "scbench-long/trajectories/10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1__claude-opus-4-6__claude-code__r1.json"
    },
    {
      "modelId": "sclong-sonnet-4-6-pi",
      "modelName": "claude-sonnet-4-6",
      "harness": "pi",
      "provider": "Anthropic",
      "available": true,
      "task": "10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": true,
      "score": 15,
      "durationS": 2495.63,
      "cost": 2.9629857000000004,
      "nTurns": 33,
      "reasoning": "all_of [pass_rule=all]: PASS\n  + list_match: list_match (answer_field='cd8_states_enriched_in_singlet_fraction'): PASS  (score=3.0)\n  + list_match: list_match (answer_field='cd8_states_enriched_in_tumor_cluster_fraction'): PASS  (score=3.0)\n  + clonotype_expansion_direction_data_true: passed=True, score=1.0\n  + biological_inference_data_true: passed=True, score=1.0\n  + dict_match: dict_match (answer_field='biological_relations'): PASS  (score=7.0)\n  + hard_fail 'singlet_top3_contains_naive_state_prior_pushing': not triggered\n  + hard_fail 'tumor_cluster_top3_contains_exhausted_state_prior_pushing': not triggered\n  + hard_fail 'singlet_top3_contains_decoy_token': not triggered\n  + hard_fail 'tumor_cluster_top3_contains_decoy_token': not triggered\n  + hard_fail 'TOXhigh_Tex_wrong_functional_category': not triggered\n  + hard_fail 'TCF7_stem_like_Tex_wrong_functional_category': not triggered\n  + hard_fail 'MKI67_state_wrong_functional_category': not triggered\n  + hard_fail 'Tn_or_Tn_Tmem_wrong_functional_category': not triggered\n  + hard_fail 'missing_exhausted_or_proliferating_anchor_in_singlet_top3': not triggered\n  + hard_fail 'missing_stemlike_precursor_in_singlet_top3': not triggered\n  + hard_fail 'missing_naive_or_early_memory_anchor_in_tumor_cluster_top3': not triggered\n  + hard_fail 'cf_reported_canonical_expansion_direction': not triggered\n  + hard_fail 'cf_expansion_lazy_defer': not triggered\n  + hard_fail 'cf_reported_canonical_expected_biology': not triggered\n  + hard_fail 'cf_biological_inference_null_artifact_or_lineage': not triggered\n  + hard_fail 'cf_exhaustion_inverted_prior_push': not triggered\n  + hard_fail 'cf_naive_inverted_prior_push': not triggered\n  + hard_fail 'cf_proliferation_inverted_prior_push': not triggered\n  + hard_fail 'cf_expansion_richness_inverted_prior_push': not triggered\n  + hard_fail 'cf_multichain_inverted_prior_push': not triggered\n  + hard_fail 'cf_intermediate_gzmk_overgeneralized': not triggered\n  + hard_fail 'cf_melanocytic_conjugate_overcall': not triggered\n  + hard_fail 'cf_relations_mass_hedge': not triggered",
      "metrics": {
        "hard_fail_triggered": [],
        "pass_rule": "all",
        "scoring_count": 5,
        "scoring_passed": 5,
        "scoring_total_score": 15
      },
      "fieldScores": {
        "biological_inference_data_true": 1,
        "clonotype_expansion_direction_data_true": 1,
        "dict_match": 7,
        "list_match": 3
      },
      "graderChecks": [
        {
          "name": "cd8_states_enriched_in_singlet_fraction",
          "kind": "list_match",
          "passed": true,
          "score": 3
        },
        {
          "name": "cd8_states_enriched_in_tumor_cluster_fraction",
          "kind": "list_match",
          "passed": true,
          "score": 3
        },
        {
          "name": "clonotype_expansion_direction_data_true",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "biological_inference_data_true",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "biological_relations",
          "kind": "list_match",
          "passed": true,
          "score": 7
        },
        {
          "name": "singlet_top3_contains_naive_state_prior_pushing",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "singlet_top3_contains_decoy_token",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "tumor_cluster_top3_contains_decoy_token",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "TOXhigh_Tex_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "TCF7_stem_like_Tex_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "MKI67_state_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "Tn_or_Tn_Tmem_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_stemlike_precursor_in_singlet_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_reported_canonical_expansion_direction",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_expansion_lazy_defer",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_reported_canonical_expected_biology",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_biological_inference_null_artifact_or_lineage",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_exhaustion_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_naive_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_proliferation_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_expansion_richness_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_multichain_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_intermediate_gzmk_overgeneralized",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_melanocytic_conjugate_overcall",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_relations_mass_hedge",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "biological_inference": "singlet_antigen_experienced_cluster_bystander",
        "biological_relations": {
          "clonal_expansion_richness_in_singlets_vs_tumor_cluster": "greater_than",
          "exhaustion_program_in_singlets_vs_tumor_cluster": "greater_than",
          "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": "approximately_equal",
          "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": "greater_than",
          "multichain_tcr_rate_in_singlets_vs_tumor_cluster": "greater_than",
          "naive_memory_program_in_singlets_vs_tumor_cluster": "much_less_than",
          "proliferation_program_in_singlets_vs_tumor_cluster": "greater_than"
        },
        "cd8_states_enriched_in_singlet_fraction": [
          {
            "functional_category": "stem_like_exhausted",
            "state_label": "TCF7_stem_like_Tex"
          },
          {
            "functional_category": "intermediate_exhausted_gzmk",
            "state_label": "GZMKhigh_Tex"
          },
          {
            "functional_category": "terminal_exhausted",
            "state_label": "LAG3high_Tex"
          }
        ],
        "cd8_states_enriched_in_tumor_cluster_fraction": [
          {
            "functional_category": "early_effector_memory",
            "state_label": "Tem_GZMH"
          },
          {
            "functional_category": "early_effector_memory",
            "state_label": "early_Tem_GZMK"
          },
          {
            "functional_category": "naive_memory",
            "state_label": "Tn_Tmem"
          }
        ],
        "clonotype_expansion_finding": "singlet_modestly_more_expanded_than_cluster"
      },
      "checkpointSignals": {
        "clustering": true,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": false,
        "nerveJointAnchor": false,
        "multiomeAssembly": false,
        "depthNormalizedAtac": false,
        "regulonInference": false,
        "cd8SubsetIntegration": true,
        "perPatientEnrichment": true,
        "tcrRichnessMetrics": true,
        "orthologProjection": false,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": true,
        "mousePseudobulkDesign": false,
        "cancerEpithelialDenominator": false,
        "geneLevelDe": false,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_singlet_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "TOXhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "LAG3high_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "GZMKhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "intermediate_exhausted_gzmk"
                    ],
                    "canonical": "intermediate_exhausted_gzmk"
                  }
                }
              },
              {
                "key": "TCF7_stem_like_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "stem_like_exhausted"
                    ],
                    "canonical": "stem_like_exhausted"
                  }
                }
              },
              {
                "key": "MKI67_pos_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              },
              {
                "key": "MKI67high_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_tumor_cluster_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "Tn",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "Tn_Tmem",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "early_Tem_GZMK",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              },
              {
                "key": "Tem_GZMH",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              }
            ]
          },
          {
            "name": "clonotype_expansion_direction_data_true",
            "role": "gate",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_substantially_more_expanded_than_cluster",
              "singlet_modestly_more_expanded_than_cluster"
            ]
          },
          {
            "name": "biological_inference_data_true",
            "role": "gate",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_antigen_experienced_cluster_bystander"
            ]
          },
          {
            "name": "singlet_top3_contains_naive_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Tn",
              "Tn_Tmem",
              "early_Tem_GZMK",
              "Tem_GZMH"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "TOXhigh_Tex",
              "LAG3high_Tex",
              "GZMKhigh_Tex",
              "TCF7_stem_like_Tex",
              "MKI67_pos_Tex_Tprol",
              "MKI67high_Tex_Tprol"
            ]
          },
          {
            "name": "singlet_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "TOXhigh_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "TCF7_stem_like_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "MKI67_state_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "Tn_or_Tn_Tmem_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_stemlike_precursor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_reported_canonical_expansion_direction",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_substantially_more_expanded_than_singlet",
              "cluster_modestly_more_expanded_than_singlet"
            ]
          },
          {
            "name": "cf_expansion_lazy_defer",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "expansion_pattern_not_detectable"
            ]
          },
          {
            "name": "cf_reported_canonical_expected_biology",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_antigen_experienced_singlet_bystander"
            ]
          },
          {
            "name": "cf_biological_inference_null_artifact_or_lineage",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "differences_reflect_dissociation_or_sorting_artifact",
              "cluster_and_singlet_capture_same_biology",
              "cluster_and_singlet_capture_distinct_cd8_lineages"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "biological_relations",
            "kind": "dict",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_greater_than",
                  "greater_than"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "less_than",
                  "much_less_than"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than",
                  "approximately_equal"
                ]
              },
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal"
                ]
              },
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal",
                  "greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_exhaustion_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_naive_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_proliferation_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_expansion_richness_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_multichain_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_intermediate_gzmk_overgeneralized",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_melanocytic_conjugate_overcall",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "values": [
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_relations_mass_hedge",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              }
            }
          }
        ]
      },
      "trajectoryAvailable": true,
      "runId": "claude-sonnet-4-6__pi__10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1__r3",
      "trajectoryFile": "scbench-long/trajectories/10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1__claude-sonnet-4-6__pi__r3.json"
    },
    {
      "modelId": "sclong-grok-4-20-beta-reasoning-pi",
      "modelName": "grok-4.20-beta-0309-reasoning",
      "harness": "pi",
      "provider": "SpaceXAI",
      "available": true,
      "task": "gse155381_q4_pseudotime_delay",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "trialIndex": 2,
      "passed": true,
      "score": null,
      "durationS": 293,
      "cost": 0.1056597,
      "nTurns": null,
      "reasoning": "",
      "metrics": {},
      "fieldScores": {},
      "graderChecks": [],
      "agentAnswer": null,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "pseudotime_direction_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "E17_chronological_match_weighted",
            "role": "additive",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pluripotency_exit_weighted",
            "role": "additive",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "pseudotime_direction_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_developmental_pace_vs_monkey_EPI",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_synchronous_with_monkey_EPI",
              "chimera_human_EPI_developmentally_advanced",
              "chimera_human_EPI_delayed_at_pre_implantation_only_resolved_post_implantation"
            ]
          },
          {
            "name": "E17_chronological_match_paper_inverted",
            "role": "hard_fail",
            "field": "$.chimera_human_E17_EPI_chronological_match",
            "kind": "value",
            "op": "in",
            "expected": [
              "resembles_monkey_E17_Gast_EPI",
              "resembles_chimera_monkey_E17_EPI_specifically",
              "chimera_human_E17_EPI_more_advanced_than_monkey_E17_Gast"
            ]
          },
          {
            "name": "pluripotency_exit_paper_inverted_or_prior_driven",
            "role": "hard_fail",
            "field": "$.chimera_human_EPI_pluripotency_exit",
            "kind": "value",
            "op": "in",
            "expected": [
              "chimera_human_EPI_advances_pluripotency_exit_prematurely",
              "naive_pluripotency_program_specifically_engaged",
              "pluripotency_core_retained_via_KLF17_naive_marker_specifically"
            ]
          }
        ]
      },
      "trajectoryAvailable": false
    },
    {
      "modelId": "sclong-kimi-k2p6-pi",
      "modelName": "kimi-k2p6",
      "harness": "pi",
      "provider": "Moonshot",
      "available": true,
      "task": "10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 3,
      "passed": true,
      "score": 15,
      "durationS": 1664.26,
      "cost": 0.7610937299999999,
      "nTurns": 24,
      "reasoning": "all_of [pass_rule=all]: PASS\n  + list_match: list_match (answer_field='cd8_states_enriched_in_singlet_fraction'): PASS  (score=3.0)\n  + list_match: list_match (answer_field='cd8_states_enriched_in_tumor_cluster_fraction'): PASS  (score=3.0)\n  + clonotype_expansion_direction_data_true: passed=True, score=1.0\n  + biological_inference_data_true: passed=True, score=1.0\n  + dict_match: dict_match (answer_field='biological_relations'): PASS  (score=7.0)\n  + hard_fail 'singlet_top3_contains_naive_state_prior_pushing': not triggered\n  + hard_fail 'tumor_cluster_top3_contains_exhausted_state_prior_pushing': not triggered\n  + hard_fail 'singlet_top3_contains_decoy_token': not triggered\n  + hard_fail 'tumor_cluster_top3_contains_decoy_token': not triggered\n  + hard_fail 'TOXhigh_Tex_wrong_functional_category': not triggered\n  + hard_fail 'TCF7_stem_like_Tex_wrong_functional_category': not triggered\n  + hard_fail 'MKI67_state_wrong_functional_category': not triggered\n  + hard_fail 'Tn_or_Tn_Tmem_wrong_functional_category': not triggered\n  + hard_fail 'missing_exhausted_or_proliferating_anchor_in_singlet_top3': not triggered\n  + hard_fail 'missing_stemlike_precursor_in_singlet_top3': not triggered\n  + hard_fail 'missing_naive_or_early_memory_anchor_in_tumor_cluster_top3': not triggered\n  + hard_fail 'cf_reported_canonical_expansion_direction': not triggered\n  + hard_fail 'cf_expansion_lazy_defer': not triggered\n  + hard_fail 'cf_reported_canonical_expected_biology': not triggered\n  + hard_fail 'cf_biological_inference_null_artifact_or_lineage': not triggered\n  + hard_fail 'cf_exhaustion_inverted_prior_push': not triggered\n  + hard_fail 'cf_naive_inverted_prior_push': not triggered\n  + hard_fail 'cf_proliferation_inverted_prior_push': not triggered\n  + hard_fail 'cf_expansion_richness_inverted_prior_push': not triggered\n  + hard_fail 'cf_multichain_inverted_prior_push': not triggered\n  + hard_fail 'cf_intermediate_gzmk_overgeneralized': not triggered\n  + hard_fail 'cf_melanocytic_conjugate_overcall': not triggered\n  + hard_fail 'cf_relations_mass_hedge': not triggered",
      "metrics": {
        "hard_fail_triggered": [],
        "pass_rule": "all",
        "scoring_count": 5,
        "scoring_passed": 5,
        "scoring_total_score": 15
      },
      "fieldScores": {
        "biological_inference_data_true": 1,
        "clonotype_expansion_direction_data_true": 1,
        "dict_match": 7,
        "list_match": 3
      },
      "graderChecks": [
        {
          "name": "cd8_states_enriched_in_singlet_fraction",
          "kind": "list_match",
          "passed": true,
          "score": 3
        },
        {
          "name": "cd8_states_enriched_in_tumor_cluster_fraction",
          "kind": "list_match",
          "passed": true,
          "score": 3
        },
        {
          "name": "clonotype_expansion_direction_data_true",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "biological_inference_data_true",
          "kind": "gate",
          "passed": true,
          "score": 1
        },
        {
          "name": "biological_relations",
          "kind": "list_match",
          "passed": true,
          "score": 7
        },
        {
          "name": "singlet_top3_contains_naive_state_prior_pushing",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "singlet_top3_contains_decoy_token",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "tumor_cluster_top3_contains_decoy_token",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "TOXhigh_Tex_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "TCF7_stem_like_Tex_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "MKI67_state_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "Tn_or_Tn_Tmem_wrong_functional_category",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_stemlike_precursor_in_singlet_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_reported_canonical_expansion_direction",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_expansion_lazy_defer",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_reported_canonical_expected_biology",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_biological_inference_null_artifact_or_lineage",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_exhaustion_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_naive_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_proliferation_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_expansion_richness_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_multichain_inverted_prior_push",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_intermediate_gzmk_overgeneralized",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_melanocytic_conjugate_overcall",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        },
        {
          "name": "cf_relations_mass_hedge",
          "kind": "hard_fail",
          "passed": true,
          "triggered": false
        }
      ],
      "agentAnswer": {
        "biological_inference": "singlet_antigen_experienced_cluster_bystander",
        "biological_relations": {
          "clonal_expansion_richness_in_singlets_vs_tumor_cluster": "greater_than",
          "exhaustion_program_in_singlets_vs_tumor_cluster": "greater_than",
          "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": "approximately_equal",
          "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": "approximately_equal",
          "multichain_tcr_rate_in_singlets_vs_tumor_cluster": "approximately_equal",
          "naive_memory_program_in_singlets_vs_tumor_cluster": "much_less_than",
          "proliferation_program_in_singlets_vs_tumor_cluster": "greater_than"
        },
        "cd8_states_enriched_in_singlet_fraction": [
          {
            "functional_category": "stem_like_exhausted",
            "state_label": "TCF7_stem_like_Tex"
          },
          {
            "functional_category": "intermediate_exhausted_gzmk",
            "state_label": "GZMKhigh_Tex"
          },
          {
            "functional_category": "terminal_exhausted",
            "state_label": "TOXhigh_Tex"
          }
        ],
        "cd8_states_enriched_in_tumor_cluster_fraction": [
          {
            "functional_category": "naive_memory",
            "state_label": "Tn"
          },
          {
            "functional_category": "early_effector_memory",
            "state_label": "early_Tem_GZMK"
          },
          {
            "functional_category": "early_effector_memory",
            "state_label": "Tem_GZMH"
          }
        ],
        "clonotype_expansion_finding": "singlet_modestly_more_expanded_than_cluster"
      },
      "checkpointSignals": {
        "clustering": false,
        "markerAnnotation": true,
        "scoreBasedAnnotation": false,
        "normalization": false,
        "nerveJointAnchor": false,
        "multiomeAssembly": true,
        "depthNormalizedAtac": true,
        "regulonInference": true,
        "cd8SubsetIntegration": true,
        "perPatientEnrichment": true,
        "tcrRichnessMetrics": true,
        "orthologProjection": false,
        "epiSubsetSelection": false,
        "bidirectionalLrTest": false,
        "enrichmentFamilyRanking": false,
        "mousePseudobulkDesign": false,
        "cancerEpithelialDenominator": false,
        "geneLevelDe": false,
        "agingSignatureControl": false,
        "senescencePanelScoring": false
      },
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_singlet_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "TOXhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "LAG3high_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "GZMKhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "intermediate_exhausted_gzmk"
                    ],
                    "canonical": "intermediate_exhausted_gzmk"
                  }
                }
              },
              {
                "key": "TCF7_stem_like_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "stem_like_exhausted"
                    ],
                    "canonical": "stem_like_exhausted"
                  }
                }
              },
              {
                "key": "MKI67_pos_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              },
              {
                "key": "MKI67high_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_tumor_cluster_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "Tn",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "Tn_Tmem",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "early_Tem_GZMK",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              },
              {
                "key": "Tem_GZMH",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              }
            ]
          },
          {
            "name": "clonotype_expansion_direction_data_true",
            "role": "gate",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_substantially_more_expanded_than_cluster",
              "singlet_modestly_more_expanded_than_cluster"
            ]
          },
          {
            "name": "biological_inference_data_true",
            "role": "gate",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_antigen_experienced_cluster_bystander"
            ]
          },
          {
            "name": "singlet_top3_contains_naive_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Tn",
              "Tn_Tmem",
              "early_Tem_GZMK",
              "Tem_GZMH"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "TOXhigh_Tex",
              "LAG3high_Tex",
              "GZMKhigh_Tex",
              "TCF7_stem_like_Tex",
              "MKI67_pos_Tex_Tprol",
              "MKI67high_Tex_Tprol"
            ]
          },
          {
            "name": "singlet_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "TOXhigh_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "TCF7_stem_like_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "MKI67_state_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "Tn_or_Tn_Tmem_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_stemlike_precursor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_reported_canonical_expansion_direction",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_substantially_more_expanded_than_singlet",
              "cluster_modestly_more_expanded_than_singlet"
            ]
          },
          {
            "name": "cf_expansion_lazy_defer",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "expansion_pattern_not_detectable"
            ]
          },
          {
            "name": "cf_reported_canonical_expected_biology",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_antigen_experienced_singlet_bystander"
            ]
          },
          {
            "name": "cf_biological_inference_null_artifact_or_lineage",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "differences_reflect_dissociation_or_sorting_artifact",
              "cluster_and_singlet_capture_same_biology",
              "cluster_and_singlet_capture_distinct_cd8_lineages"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "biological_relations",
            "kind": "dict",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_greater_than",
                  "greater_than"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "less_than",
                  "much_less_than"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than",
                  "approximately_equal"
                ]
              },
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal"
                ]
              },
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal",
                  "greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_exhaustion_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_naive_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_proliferation_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_expansion_richness_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_multichain_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_intermediate_gzmk_overgeneralized",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_melanocytic_conjugate_overcall",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "values": [
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_relations_mass_hedge",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              }
            }
          }
        ]
      },
      "trajectoryAvailable": true,
      "runId": "kimi-k2p6__pi__10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1__r3",
      "trajectoryFile": "scbench-long/trajectories/10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1__kimi-k2p6__pi__r3.json"
    },
    {
      "modelId": "sclong-gpt-5-4-codex",
      "modelName": "gpt-5.4",
      "harness": "openai-codex",
      "provider": "OpenAI",
      "available": true,
      "task": "10x_5p_melanoma_tcr_matched_apc_vs_tumor_exhaustion",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "trialIndex": 1,
      "passed": true,
      "score": null,
      "durationS": 361,
      "cost": 0.9262535000000001,
      "nTurns": null,
      "reasoning": "",
      "metrics": {},
      "fieldScores": {},
      "graderChecks": [],
      "agentAnswer": null,
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "state_discriminating_genes",
            "kind": "list",
            "matchKey": "gene",
            "k": 3,
            "expected": [
              {
                "key": "CTLA4",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TOX",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "ENTPD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "HAVCR2",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "TIGIT",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              },
              {
                "key": "PDCD1",
                "fields": {
                  "signal_origin": {
                    "op": "in",
                    "values": [
                      "t_cell_intrinsic_apc_up_within_patient"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "difference_attributable_to",
            "role": "gate",
            "field": "difference_attributable_to",
            "kind": "value",
            "expected": "covaries_with_apc_conjugate_state"
          },
          {
            "name": "causal_status",
            "role": "gate",
            "field": "causal_status",
            "kind": "value",
            "expected": "direction_unresolved_selection_confound"
          },
          {
            "name": "functional_interpretation",
            "role": "gate",
            "field": "functional_interpretation",
            "kind": "value",
            "expected": "costimulation_signature_function_unresolved"
          },
          {
            "name": "cf_contamination_gene_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "MLANA",
              "PMEL",
              "DCT",
              "TYR",
              "TYRP1",
              "SOX10",
              "MITF",
              "MAGEA4"
            ]
          },
          {
            "name": "cf_apc_partner_contamination_gene",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "HLA-DRA",
              "CD68",
              "LYZ",
              "CD14",
              "C1QA",
              "ITGAX",
              "CD1C"
            ]
          },
          {
            "name": "cf_ligand_as_state_marker",
            "role": "hard_fail",
            "field": "$.state_discriminating_genes[*].gene",
            "kind": "avoid",
            "expected": [
              "CD274",
              "PDCD1LG2",
              "CD80",
              "CD86"
            ]
          },
          {
            "name": "cf_attribution_causal_or_baseline",
            "role": "hard_fail",
            "field": "$.difference_attributable_to",
            "kind": "avoid",
            "expected": [
              "apc_engagement",
              "tumor_cell_engagement",
              "clonotype_intrinsic_baseline"
            ]
          },
          {
            "name": "cf_causal_overclaim",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "engagement_induces_exhausted_state",
              "pre_exhausted_cells_select_into_apc_conjugates"
            ]
          },
          {
            "name": "cf_causal_fabrication",
            "role": "hard_fail",
            "field": "$.causal_status",
            "kind": "avoid",
            "expected": [
              "resolved_by_pseudotemporal_ordering",
              "resolved_by_perturbation_experiment"
            ]
          },
          {
            "name": "cf_functional_overclaim",
            "role": "hard_fail",
            "field": "$.functional_interpretation",
            "kind": "avoid",
            "expected": [
              "costimulation_licensed_reinvigoration_competent",
              "costimulation_licensed_checkpoint_responsive",
              "tumor_driven_terminal_exhaustion"
            ]
          }
        ]
      },
      "trajectoryAvailable": false
    },
    {
      "modelId": "sclong-grok-4-3-pi",
      "modelName": "grok-4.3",
      "harness": "pi",
      "provider": "SpaceXAI",
      "available": true,
      "task": "gse171524_pathological_fibroblast_master_regulator_druggable_targets",
      "system": "Lethal COVID-19 lung pathology",
      "assay": "Single-nucleus RNA-seq",
      "mainTaskTheme": "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence",
      "trialIndex": 2,
      "passed": true,
      "score": null,
      "durationS": 31,
      "cost": 0.0098044,
      "nTurns": null,
      "reasoning": "",
      "metrics": {},
      "fieldScores": {},
      "graderChecks": [],
      "agentAnswer": null,
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "disease_associated_fibroblast_subtype_weighted",
            "role": "additive",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "mr_tf_junb_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_alpha"
            ]
          },
          {
            "name": "mr_tf_jund_present",
            "role": "additive",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_beta"
            ]
          },
          {
            "name": "drug_mmp14_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_alpha"
            ]
          },
          {
            "name": "drug_stat3_present",
            "role": "additive",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_beta"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_weighted",
            "role": "additive",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "disease_associated_fibroblast_subtype_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.disease_associated_fibroblast_subtype",
            "kind": "value",
            "op": "in",
            "expected": [
              "subtype_lambda",
              "subtype_epsilon",
              "subtype_iota",
              "subtype_beta",
              "subtype_eta",
              "subtype_zeta",
              "subtype_not_assessed"
            ]
          },
          {
            "name": "method_used_for_master_regulator_inference_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.method_used_for_master_regulator_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "simple_de_ranking_by_log_fc",
              "marker_based_de_only",
              "not_computed"
            ]
          },
          {
            "name": "top_master_regulator_transcription_factors_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_master_regulator_transcription_factors[*]",
            "kind": "avoid",
            "expected": [
              "mr_tf_not_computed"
            ]
          },
          {
            "name": "top_druggable_proteins_below_minimum_score_for_pass",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_not_assessed"
            ]
          },
          {
            "name": "cross_axis_circular_marker_in_druggable_proteins",
            "role": "hard_fail",
            "field": "$.top_druggable_proteins[*]",
            "kind": "avoid",
            "expected": [
              "drug_target_gamma",
              "drug_target_delta",
              "drug_target_eta"
            ]
          }
        ]
      },
      "trajectoryAvailable": false
    }
  ],
  "systems": [
    "Melanoma tumor-reactive CD8 T cells",
    "CD8 RNA+ATAC regulatory inference",
    "Human-monkey chimeric embryos",
    "KRAS-driven lung tumor aging",
    "Lethal COVID-19 lung pathology"
  ],
  "mainTaskThemes": [
    "Link expression states with TCR clonotypes; interpret marker-sorted populations",
    "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
    "Compare developmental states across species; evaluate cell-cell signaling",
    "Separate age-associated effects from tumor-cell state and microenvironment changes",
    "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence"
  ],
  "systemCounts": {
    "Melanoma tumor-reactive CD8 T cells": 5,
    "CD8 RNA+ATAC regulatory inference": 3,
    "Human-monkey chimeric embryos": 4,
    "KRAS-driven lung tumor aging": 4,
    "Lethal COVID-19 lung pathology": 5
  },
  "mainTaskThemeCounts": {
    "Link expression states with TCR clonotypes; interpret marker-sorted populations": 5,
    "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility": 3,
    "Compare developmental states across species; evaluate cell-cell signaling": 4,
    "Separate age-associated effects from tumor-cell state and microenvironment changes": 4,
    "Integrate snRNA-seq with immune-receptor, cell-subset, and orthogonal validation evidence": 5
  },
  "evals": [
    {
      "id": "gse297023_q2_senescence_not_mechanism",
      "title": "Gse297023 Q2 Senescence Not Mechanism",
      "description": "KRAS-driven lung tumor aging \u00b7 Parse Biosciences scRNA-seq: Determine how the host's age bears on the cancer cells in this model.",
      "objective": "Determine how the host's age bears on the cancer cells in this model.",
      "prompt": "Single-cell RNA-seq (Parse Biosciences) of tumor-bearing lungs from a mouse model of oncogenic KRAS-driven lung cancer.\n\nDetermine how the host's age bears on the cancer cells in this model. Establish what age comparison the profiled cancer-cell cohort actually supports, name the program you would most strongly hypothesize to account for an age effect on these cells, state what your evidence implies for the hypothesized program's role.\n\nUse vocabulary.json (provided as a data node).\n\nReturn EXACTLY:\n<EVAL_ANSWER>\n{\"cohort_host_age_design\": \"<token>\", \"candidate_mechanism\": \"<token>\", \"arrest_core_engaged\": \"<token>\", \"secretory_arm_identity\": \"<token>\", \"secretory_arm_direction\": \"<token>\", \"age_associated_program_identity\": \"<token>\", \"nominated_mechanism_role\": \"<token>\"}\n</EVAL_ANSWER>",
      "system": "KRAS-driven lung tumor aging",
      "assay": "Parse Biosciences scRNA-seq",
      "mainTaskTheme": "Separate age-associated effects from tumor-cell state and microenvironment changes",
      "graderType": "all_of",
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "cohort_spans_young_and_aged",
            "role": "gate",
            "field": "$.cohort_host_age_design",
            "kind": "value",
            "op": "in",
            "expected": [
              "young_and_aged_hosts_both_profiled"
            ]
          },
          {
            "name": "candidate_is_senescence",
            "role": "gate",
            "field": "$.candidate_mechanism",
            "kind": "value",
            "op": "in",
            "expected": [
              "cellular_senescence",
              "oncogene_induced_senescence",
              "replicative_senescence"
            ]
          },
          {
            "name": "arrest_core_not_engaged",
            "role": "gate",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_not_engaged"
            ]
          },
          {
            "name": "secretory_arm_is_inflammaging",
            "role": "gate",
            "field": "$.secretory_arm_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "acute_phase_inflammaging_response"
            ]
          },
          {
            "name": "secretory_arm_broadly_elevated",
            "role": "gate",
            "field": "$.secretory_arm_direction",
            "kind": "value",
            "op": "in",
            "expected": [
              "broadly_elevated"
            ]
          },
          {
            "name": "carries_aging_program",
            "role": "gate",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "carries_bona_fide_aging_program_distinct_from_nominated"
            ]
          },
          {
            "name": "conclusion_senescence_unlikely",
            "role": "gate",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "unlikely_major_contributor"
            ]
          },
          {
            "name": "cf_arrest_core_engaged_recall",
            "role": "hard_fail",
            "field": "$.arrest_core_engaged",
            "kind": "value",
            "op": "in",
            "expected": [
              "arrest_core_engaged"
            ]
          },
          {
            "name": "cf_aging_program_conflation_or_explain_away",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "aging_program_indistinguishable_from_nominated_program",
              "aging_program_is_generic_malignancy_not_aging"
            ]
          },
          {
            "name": "cf_no_aging_program_sensitivity_failure",
            "role": "hard_fail",
            "field": "$.age_associated_program_identity",
            "kind": "value",
            "op": "in",
            "expected": [
              "no_detectable_aging_program"
            ]
          },
          {
            "name": "cf_senescence_driver_conclusion",
            "role": "hard_fail",
            "field": "$.nominated_mechanism_role",
            "kind": "value",
            "op": "in",
            "expected": [
              "likely_primary_driver",
              "plausible_supported_contributor"
            ]
          }
        ]
      },
      "dataNodeCount": 2,
      "modules": {
        "graded_biological_result": "The canonical aging program invoked to explain altered tumor behaviour in aged hosts (cellular senescence) is an unlikely major contributor: it is not coherently engaged in aged cancer cells, which the agent must establish by dissecting the program: the stable-arrest core is OFF (core arrest genes flat-to-down, senescence signatures depleted, cells not arrested) and the senescence SASP itself is only flat-to-scattered, WHILE the secretory signal that IS coordinately elevated is an acute-phase / inflammaging program \u2014 distinct from senescence. The agent must identify this elevated secretory arm as the acute-phase/inflammaging program and call it broadly_elevated, which requires GENE-LEVEL pseudobulk-by-mouse DESeq2: the program is coordinately up at the gene level (most acute-phase genes up, several FDR-significant via DESeq2 / GSEA, matching independent normal-tissue aging signatures) but a per-cell composite score is underpowered and makes the secretory signal look flat or merely scattered. This gene-level-vs-per-cell method choice is a designed trap: agents that lean on per-cell scoring (or underpowered within-genotype splits) mis-call the arm as the SASP and/or its direction as scattered/unchanged. The agent must first discover that the cancer compartment spans young and aged hosts, then nominate senescence unprompted, from an open-ended prompt that states neither the cohort age design, nor that any program exists, nor the expected result, and that does not enumerate the positive-control step of identifying the program the cells actually carry.",
        "ungraded_dependency_stack": [
          "discover from obs that the cancer-cell compartment is profiled from both young and aged hosts (the prompt withholds this)",
          "infer from an open-ended prompt that the classically expected aging program is cellular senescence and nominate it unprompted",
          "single-cell QC, normalization",
          "clustering/annotation and cancer-cell denominator definition (alveolar epithelium; tdTomato not a usable marker)",
          "age effect within sgInert via pseudobulk-by-mouse DESeq2 (correct experimental unit; avoid per-cell pseudoreplication)",
          "scoring multiple senescence signatures (SenMayo, lung-epithelial, pan-cancer, MSigDB GO-BP/Reactome) by GSEA on signed ranks; finding the arrest core off AND the senescence SASP only flat-to-scattered (not coherently elevated)",
          "identifying the coordinately ELEVATED secretory arm as an acute-phase/inflammaging program by GENE-LEVEL pseudobulk DESeq2 / GSEA (NOT per-cell composite scoring, which is underpowered and masks the coordinated elevation \u2014 the designed method trap); calling its direction broadly_elevated",
          "running bona fide mouse aging signatures through the identical (gene-level) pipeline as a positive control (the sensitivity check that the senescence null is not a detection failure)",
          "concluding senescence is an unlikely major contributor while a distinct, genuine aging program is present"
        ]
      },
      "rubricCheckpoints": [
        {
          "index": 1,
          "title": "Task scope and answer contract",
          "decision": "Frame the task as discover-the-cohort \u2192 nominate-a-candidate \u2192 dissect it in the aged cancer cells (arrest core, secretory arm identity + direction, real aging program) \u2192 state its role, and emit one parseable JSON block inside <EVAL_ANSWER> tags with all seven fields, every value a controlled-vocabulary token. Recognise that the four downstream program fields refer to *the program nominated in candidate_mechanism*.",
          "why": "drifts include testing a program other than the one nominated, collapsing the decomposed facts into one label, or omitting/free-texting a field.",
          "successEvidence": "reads vocabulary.json; nominates one program; dissects *that* program; emits exactly the seven fields in schema with exact tokens.",
          "failureModes": "downstream fields reference a different program; missing field; free-text/synonym token; unparseable block.",
          "scoring": "0 = wrong task or unparseable; 1 = right task with contract violations; 2 = exact contract."
        },
        {
          "index": 2,
          "title": "Discover the cohort host-age design",
          "decision": "Recover from obs that the cancer-cell compartment is profiled from both young and aged hosts \u2014 the prompt withholds this \u2014 establishing the young-vs-aged comparison the cohort supports.",
          "why": "the cohort_host_age_design gate and the precondition for every downstream analysis; an agent that never recovers the age annotation cannot scope the age contrast.",
          "successEvidence": "host-age annotation identified; cancer-fraction cells shown to exist in both age groups; the comparison framed as young vs aged.",
          "failureModes": "young_hosts_only / aged_hosts_only misread; host_age_not_annotated; design left unresolved.",
          "scoring": "0 = design not recovered or misread; 1 = partial / weakly evidenced; 2 = young-and-aged design recovered and used downstream."
        },
        {
          "index": 3,
          "title": "Candidate nomination \u2014 senescence, unprompted and on biological grounds",
          "decision": "Nominate cellular senescence (or a senescence subtype) as the leading aging-associated program likely to shape malignant behaviour, on biological grounds, before testing it \u2014 the prompt does not name it and no longer labels the candidate \"cell-intrinsic\".",
          "why": "the candidate_mechanism gate; it is a discovery axis \u2014 naming senescence is unprompted, and the aging-hallmark decoys plus the REAL chronic_inflammation_inflammaging are calibrated misses.",
          "successEvidence": "senescence nominated with a stated rationale (canonical age\u2192tumor-behaviour link; stable arrest + SASP).",
          "failureModes": "any non-senescence hallmark (including chronic_inflammation_inflammaging); candidate_unresolved.",
          "scoring": "0 = non-senescence or unresolved; 1 = senescence but with no biological rationale; 2 = senescence nominated with biological justification."
        },
        {
          "index": 4,
          "title": "Single-cell QC and recovery of the experimental design (mouse-level age contrast)",
          "decision": "Apply defensible QC and recover the design from obs \u2014 age (Young/Old) \u00d7 genotype (Inert/Pten) \u00d7 sort_fraction (cancer/bulk) \u2014 so the age effect is estimated in the control-genotype cancer cells with the mouse as the unit of inference.",
          "why": "scoring over pooled cells inflates significance (pseudoreplication); conflating genotype or fraction with age answers a different question.",
          "successEvidence": "QC with cell-count bookkeeping; normalized representation; design decoded; age contrast scoped to cancer cells with mouse-level grouping available.",
          "failureModes": "no QC; raw counts treated as normalized; age confounded with genotype or fraction; mouse structure never recovered.",
          "scoring": "0 = no QC or design not recovered; 1 = one weak; 2 = both, used downstream."
        },
        {
          "index": 5,
          "title": "Cancer-cell denominator (neoplastic compartment)",
          "decision": "Isolate the alveolar tumor-epithelial population (AT2/AT1/AT1-AT2-like, Epcam/Nkx2-1/Sftpc+, Ptprc-negative), restrict to the cancer sort fraction, exclude immune/stromal contamination, and recognise that tdTomato is a lineage label, not a usable positive cancer marker (dropout-prone).",
          "why": "the senescence and aging tests must run on cancer cells; contamination or a tdTomato gate makes the arrest/secretory tests and the positive control uninterpretable.",
          "successEvidence": "clean tumor-epithelial compartment isolated, balanced across age; immune/stromal signatures \u22480 in tumor classes; cancer fraction used.",
          "failureModes": "whole-lung/bulk comparison; residual immune/stromal leakage; tdTomato-positive gating; normal airway epithelium not excluded.",
          "scoring": "0 = no isolation or tdTomato-gated; 1 = isolated but contamination unchecked; 2 = clean balanced tumor compartment driving the tests."
        },
        {
          "index": 6,
          "title": "Arrest-core test \u2014 multi-signature, GSEA + per-cell \u2192 not engaged",
          "decision": "Test whether the nominated program's defining stable-arrest core is coherently engaged using multiple senescence signatures and both GSEA-on-signed-ranks and per-cell scoring, plus a proliferation check, and conclude arrest_core_not_engaged \u2014 resisting the recall that senescence is engaged.",
          "why": "the arrest_core_engaged gate + the arrest_core_engaged hard-fail. A single panel or one arrest marker (e.g. Cdkn2a/Cdkn1a) is not coherence; some senescence markers always move, so multi-signature agreement licenses a clean call.",
          "successEvidence": "\u22652 senescence panels scored; both rank-based and per-cell methods; core arrest flat-to-down; cells not arrested (Old equal-or-higher S/G2M); the recall explicitly set aside with evidence.",
          "failureModes": "arrest_core_engaged (CF) from one marker or pseudoreplicated cells; arrest_core_engaged_in_minor_subpopulation / arrest_core_unresolved (gate-fail); no per-cell cross-check.",
          "scoring": "0 = arrest_core_engaged (CF), or no real arrest test; 1 = single-panel or single-method null, or a hedge; 2 = multi-signature, multi-method arrest_core_not_engaged."
        },
        {
          "index": 7,
          "title": "Secretory arm \u2014 identify the inflammaging program (gene-level) and call it broadly elevated",
          "decision": "Identify the coordinately ELEVATED secretory arm as the acute-phase/inflammaging program (acute_phase_inflammaging_response) using gene-level pseudobulk DESeq2, and judge its direction as broadly_elevated. Recognize that the senescence SASP itself is only flat-to-scattered (not the elevated arm).",
          "why": "the secretory_arm_identity and secretory_arm_direction gates, and the designed gene-level-vs-per-cell method trap. A per-cell composite score is underpowered and makes the program look flat/scattered; naming the SASP, or calling the direction scattered_elevation/unchanged, are the trap answers (gate-fails).",
          "successEvidence": "inflammaging/acute-phase named explicitly (Saa3, Lcn2, Hp, complement, Mt1/2, Retnla, etc.) via gene-level DE; shown coordinately up (most genes up, several FDR-sig); the SASP shown to be only flat-to-scattered; the elevated-non-senescence-arm/off-arrest combination noted as not-coherent engagement.",
          "failureModes": "naming the SASP or another non-inflammaging program (gate-fail); direction read as scattered/reduced/unchanged/mixed/unresolved from per-cell or underpowered analysis (gate-fail).",
          "scoring": "0 = wrong secretory identity, or direction not broadly-elevated; 1 = inflammaging named but direction weakly evidenced / from per-cell only; 2 = inflammaging named via gene-level DE and shown broadly elevated, tied to the off-arrest / flat-SASP picture."
        },
        {
          "index": 8,
          "title": "Positive-control aging program through the identical pipeline (load-bearing)",
          "decision": "Run independent normal-tissue aging signatures through the same pipeline used for senescence, show they are strongly enriched in the aged cancer cells, and conclude carries_bona_fide_aging_program_distinct_from_nominated \u2014 a real aging program distinct from senescence.",
          "why": "the age_associated_program_identity gate + three hard-fails (aging_program_indistinguishable_from_nominated_program, aging_program_is_generic_malignancy_not_aging, no_detectable_aging_program), and the sensitivity control for chokepoint 6. This most often decides pass/fail: it proves the senescence null is a true negative and separates \"aging program present\" from \"senescence engaged.\"",
          "successEvidence": "independent aging signatures (lung-epithelial and/or global; e.g. Lcn2, Saa3, Itih4, C3, Retnla, Spp1, Pigr, Il33) enriched via the identical method; the program shown distinct from senescence/SASP; explicit framing as the positive control.",
          "failureModes": "no_detectable_aging_program (CF) from FDR-count alone with no signature-level control; aging_program_indistinguishable_from_nominated_program / aging_program_is_generic_malignancy_not_aging (CF); aging_program_proliferative_or_stress_artifact / aging_program_reflects_microenvironment_contamination / aging_program_unresolved (gate-fail); positive control never run, so the arrest-core null is unsupported.",
          "scoring": "0 = conflation/explain-away/no-program (CF), or no positive control; 1 = aging program shown but not clearly distinct from senescence, or control not framed as such; 2 = bona-fide aging program enriched through the identical pipeline and distinguished from senescence."
        },
        {
          "index": 9,
          "title": "Mechanism role \u2014 senescence an unlikely major contributor",
          "decision": "Integrate the results \u2014 aging program present, arrest core not engaged \u2014 into unlikely_major_contributor, resisting the prior that senescence drives or plausibly contributes.",
          "why": "the nominated_mechanism_role gate + the likely_primary_driver / plausible_supported_contributor hard-fail; the epistemic payoff of the eval.",
          "successEvidence": "explicit reasoning that a genuine aging program without engaged senescence makes senescence an unlikely major contributor; the prior named and set aside.",
          "failureModes": "likely_primary_driver / plausible_supported_contributor (CF); minor_contributor_in_subpopulation / cannot_attribute_confounded / requires_functional_validation_inconclusive / role_unresolved despite a clear null.",
          "scoring": "0 = a driver/contributor CF token; 1 = unlikely but weakly justified, or unresolved; 2 = unlikely_major_contributor with the present-aging / off-arrest logic made explicit."
        }
      ],
      "runs": [
        {
          "runId": "claude-opus-4-7__claude-code",
          "provider": "Anthropic",
          "model": "claude-opus-4-7",
          "harness": "claude-code",
          "run": "r1",
          "trialIndex": 1,
          "passed": true,
          "score": 7,
          "reasoning": "all_of [pass_rule=all]: PASS\n  + cohort_spans_young_and_aged: passed=True, score=1.0\n  + candidate_is_senescence: passed=True, score=1.0\n  + arrest_core_not_engaged: passed=True, score=1.0\n  + secretory_arm_is_inflammaging: passed=True, score=1.0\n  + secretory_arm_broadly_elevated: passed=True, score=1.0\n  + carries_aging_program: passed=True, score=1.0\n  + conclusion_senescence_unlikely: passed=True, score=1.0\n  + hard_fail 'cf_arrest_core_engaged_recall': not triggered\n  + hard_fail 'cf_aging_program_conflation_or_explain_away': not triggered\n  + hard_fail 'cf_no_aging_program_sensitivity_failure': not triggered\n  + hard_fail 'cf_senescence_driver_conclusion': not triggered",
          "metrics": {
            "pass_rule": "all",
            "scoring_count": 7,
            "scoring_passed": 7,
            "scoring_total_score": 7,
            "hard_fail_triggered": []
          },
          "fieldScores": {
            "cohort_spans_young_and_aged": 1,
            "candidate_is_senescence": 1,
            "arrest_core_not_engaged": 1,
            "secretory_arm_is_inflammaging": 1,
            "secretory_arm_broadly_elevated": 1,
            "carries_aging_program": 1,
            "conclusion_senescence_unlikely": 1
          },
          "graderChecks": [
            {
              "name": "cohort_spans_young_and_aged",
              "kind": "gate",
              "passed": true,
              "score": 1
            },
            {
              "name": "candidate_is_senescence",
              "kind": "gate",
              "passed": true,
              "score": 1
            },
            {
              "name": "arrest_core_not_engaged",
              "kind": "gate",
              "passed": true,
              "score": 1
            },
            {
              "name": "secretory_arm_is_inflammaging",
              "kind": "gate",
              "passed": true,
              "score": 1
            },
            {
              "name": "secretory_arm_broadly_elevated",
              "kind": "gate",
              "passed": true,
              "score": 1
            },
            {
              "name": "carries_aging_program",
              "kind": "gate",
              "passed": true,
              "score": 1
            },
            {
              "name": "conclusion_senescence_unlikely",
              "kind": "gate",
              "passed": true,
              "score": 1
            },
            {
              "name": "cf_arrest_core_engaged_recall",
              "kind": "hard_fail",
              "passed": true,
              "triggered": false
            },
            {
              "name": "cf_aging_program_conflation_or_explain_away",
              "kind": "hard_fail",
              "passed": true,
              "triggered": false
            },
            {
              "name": "cf_no_aging_program_sensitivity_failure",
              "kind": "hard_fail",
              "passed": true,
              "triggered": false
            },
            {
              "name": "cf_senescence_driver_conclusion",
              "kind": "hard_fail",
              "passed": true,
              "triggered": false
            }
          ],
          "agentAnswer": {
            "age_associated_program_identity": "carries_bona_fide_aging_program_distinct_from_nominated",
            "arrest_core_engaged": "arrest_core_not_engaged",
            "candidate_mechanism": "cellular_senescence",
            "cohort_host_age_design": "young_and_aged_hosts_both_profiled",
            "nominated_mechanism_role": "unlikely_major_contributor",
            "secretory_arm_direction": "broadly_elevated",
            "secretory_arm_identity": "acute_phase_inflammaging_response"
          },
          "durationS": 457.13,
          "cost": 1.5156004999999997,
          "nTurns": 21,
          "checkpointSignals": {
            "clustering": false,
            "markerAnnotation": true,
            "scoreBasedAnnotation": false,
            "normalization": false,
            "nerveJointAnchor": false,
            "multiomeAssembly": false,
            "depthNormalizedAtac": false,
            "regulonInference": false,
            "cd8SubsetIntegration": false,
            "perPatientEnrichment": false,
            "tcrRichnessMetrics": false,
            "orthologProjection": false,
            "epiSubsetSelection": false,
            "bidirectionalLrTest": false,
            "enrichmentFamilyRanking": false,
            "mousePseudobulkDesign": true,
            "cancerEpithelialDenominator": true,
            "geneLevelDe": true,
            "agingSignatureControl": true,
            "senescencePanelScoring": true
          },
          "expectedAnswer": {
            "passRule": "all",
            "checks": [
              {
                "name": "cohort_spans_young_and_aged",
                "role": "gate",
                "field": "$.cohort_host_age_design",
                "kind": "value",
                "op": "in",
                "expected": [
                  "young_and_aged_hosts_both_profiled"
                ]
              },
              {
                "name": "candidate_is_senescence",
                "role": "gate",
                "field": "$.candidate_mechanism",
                "kind": "value",
                "op": "in",
                "expected": [
                  "cellular_senescence",
                  "oncogene_induced_senescence",
                  "replicative_senescence"
                ]
              },
              {
                "name": "arrest_core_not_engaged",
                "role": "gate",
                "field": "$.arrest_core_engaged",
                "kind": "value",
                "op": "in",
                "expected": [
                  "arrest_core_not_engaged"
                ]
              },
              {
                "name": "secretory_arm_is_inflammaging",
                "role": "gate",
                "field": "$.secretory_arm_identity",
                "kind": "value",
                "op": "in",
                "expected": [
                  "acute_phase_inflammaging_response"
                ]
              },
              {
                "name": "secretory_arm_broadly_elevated",
                "role": "gate",
                "field": "$.secretory_arm_direction",
                "kind": "value",
                "op": "in",
                "expected": [
                  "broadly_elevated"
                ]
              },
              {
                "name": "carries_aging_program",
                "role": "gate",
                "field": "$.age_associated_program_identity",
                "kind": "value",
                "op": "in",
                "expected": [
                  "carries_bona_fide_aging_program_distinct_from_nominated"
                ]
              },
              {
                "name": "conclusion_senescence_unlikely",
                "role": "gate",
                "field": "$.nominated_mechanism_role",
                "kind": "value",
                "op": "in",
                "expected": [
                  "unlikely_major_contributor"
                ]
              },
              {
                "name": "cf_arrest_core_engaged_recall",
                "role": "hard_fail",
                "field": "$.arrest_core_engaged",
                "kind": "value",
                "op": "in",
                "expected": [
                  "arrest_core_engaged"
                ]
              },
              {
                "name": "cf_aging_program_conflation_or_explain_away",
                "role": "hard_fail",
                "field": "$.age_associated_program_identity",
                "kind": "value",
                "op": "in",
                "expected": [
                  "aging_program_indistinguishable_from_nominated_program",
                  "aging_program_is_generic_malignancy_not_aging"
                ]
              },
              {
                "name": "cf_no_aging_program_sensitivity_failure",
                "role": "hard_fail",
                "field": "$.age_associated_program_identity",
                "kind": "value",
                "op": "in",
                "expected": [
                  "no_detectable_aging_program"
                ]
              },
              {
                "name": "cf_senescence_driver_conclusion",
                "role": "hard_fail",
                "field": "$.nominated_mechanism_role",
                "kind": "value",
                "op": "in",
                "expected": [
                  "likely_primary_driver",
                  "plausible_supported_contributor"
                ]
              }
            ]
          },
          "trajectoryFile": "scbench-long/trajectories/gse297023_q2_senescence_not_mechanism__claude-opus-4-7__claude-code__r1.json"
        },
        {
          "runId": "gemini-3.5-flash__pi",
          "provider": "Google",
          "model": "gemini-3.5-flash",
          "harness": "pi",
          "run": "r1",
          "trialIndex": 1,
          "passed": false,
          "score": 5,
          "reasoning": "all_of [pass_rule=all]: FAIL\n  + cohort_spans_young_and_aged: passed=True, score=1.0\n  + candidate_is_senescence: passed=True, score=1.0\n  + arrest_core_not_engaged: passed=True, score=1.0\n  x secretory_arm_is_inflammaging: passed=False, score=0.0\n  x secretory_arm_broadly_elevated: passed=False, score=0.0\n  + carries_aging_program: passed=True, score=1.0\n  + conclusion_senescence_unlikely: passed=True, score=1.0\n  + hard_fail 'cf_arrest_core_engaged_recall': not triggered\n  + hard_fail 'cf_aging_program_conflation_or_explain_away': not triggered\n  + hard_fail 'cf_no_aging_program_sensitivity_failure': not triggered\n  + hard_fail 'cf_senescence_driver_conclusion': not triggered",
          "metrics": {
            "pass_rule": "all",
            "scoring_count": 7,
            "scoring_passed": 5,
            "scoring_total_score": 5,
            "hard_fail_triggered": []
          },
          "fieldScores": {
            "cohort_spans_young_and_aged": 1,
            "candidate_is_senescence": 1,
            "arrest_core_not_engaged": 1,
            "secretory_arm_is_inflammaging": 0,
            "secretory_arm_broadly_elevated": 0,
            "carries_aging_program": 1,
            "conclusion_senescence_unlikely": 1
          },
          "graderChecks": [
            {
              "name": "cohort_spans_young_and_aged",
              "kind": "gate",
              "passed": true,
              "score": 1
            },
            {
              "name": "candidate_is_senescence",
              "kind": "gate",
              "passed": true,
              "score": 1
            },
            {
              "name": "arrest_core_not_engaged",
              "kind": "gate",
              "passed": true,
              "score": 1
            },
            {
              "name": "secretory_arm_is_inflammaging",
              "kind": "gate",
              "passed": false,
              "score": 0
            },
            {
              "name": "secretory_arm_broadly_elevated",
              "kind": "gate",
              "passed": false,
              "score": 0
            },
            {
              "name": "carries_aging_program",
              "kind": "gate",
              "passed": true,
              "score": 1
            },
            {
              "name": "conclusion_senescence_unlikely",
              "kind": "gate",
              "passed": true,
              "score": 1
            },
            {
              "name": "cf_arrest_core_engaged_recall",
              "kind": "hard_fail",
              "passed": true,
              "triggered": false
            },
            {
              "name": "cf_aging_program_conflation_or_explain_away",
              "kind": "hard_fail",
              "passed": true,
              "triggered": false
            },
            {
              "name": "cf_no_aging_program_sensitivity_failure",
              "kind": "hard_fail",
              "passed": true,
              "triggered": false
            },
            {
              "name": "cf_senescence_driver_conclusion",
              "kind": "hard_fail",
              "passed": true,
              "triggered": false
            }
          ],
          "agentAnswer": {
            "age_associated_program_identity": "carries_bona_fide_aging_program_distinct_from_nominated",
            "arrest_core_engaged": "arrest_core_not_engaged",
            "candidate_mechanism": "cellular_senescence",
            "cohort_host_age_design": "young_and_aged_hosts_both_profiled",
            "nominated_mechanism_role": "unlikely_major_contributor",
            "secretory_arm_direction": "unchanged",
            "secretory_arm_identity": "senescence_associated_secretory_phenotype"
          },
          "durationS": 291.11,
          "cost": 0.42536369999999996,
          "nTurns": 24,
          "checkpointSignals": {
            "clustering": false,
            "markerAnnotation": true,
            "scoreBasedAnnotation": false,
            "normalization": true,
            "nerveJointAnchor": false,
            "multiomeAssembly": false,
            "depthNormalizedAtac": false,
            "regulonInference": false,
            "cd8SubsetIntegration": false,
            "perPatientEnrichment": false,
            "tcrRichnessMetrics": false,
            "orthologProjection": true,
            "epiSubsetSelection": false,
            "bidirectionalLrTest": false,
            "enrichmentFamilyRanking": true,
            "mousePseudobulkDesign": true,
            "cancerEpithelialDenominator": true,
            "geneLevelDe": true,
            "agingSignatureControl": true,
            "senescencePanelScoring": true
          },
          "expectedAnswer": {
            "passRule": "all",
            "checks": [
              {
                "name": "cohort_spans_young_and_aged",
                "role": "gate",
                "field": "$.cohort_host_age_design",
                "kind": "value",
                "op": "in",
                "expected": [
                  "young_and_aged_hosts_both_profiled"
                ]
              },
              {
                "name": "candidate_is_senescence",
                "role": "gate",
                "field": "$.candidate_mechanism",
                "kind": "value",
                "op": "in",
                "expected": [
                  "cellular_senescence",
                  "oncogene_induced_senescence",
                  "replicative_senescence"
                ]
              },
              {
                "name": "arrest_core_not_engaged",
                "role": "gate",
                "field": "$.arrest_core_engaged",
                "kind": "value",
                "op": "in",
                "expected": [
                  "arrest_core_not_engaged"
                ]
              },
              {
                "name": "secretory_arm_is_inflammaging",
                "role": "gate",
                "field": "$.secretory_arm_identity",
                "kind": "value",
                "op": "in",
                "expected": [
                  "acute_phase_inflammaging_response"
                ]
              },
              {
                "name": "secretory_arm_broadly_elevated",
                "role": "gate",
                "field": "$.secretory_arm_direction",
                "kind": "value",
                "op": "in",
                "expected": [
                  "broadly_elevated"
                ]
              },
              {
                "name": "carries_aging_program",
                "role": "gate",
                "field": "$.age_associated_program_identity",
                "kind": "value",
                "op": "in",
                "expected": [
                  "carries_bona_fide_aging_program_distinct_from_nominated"
                ]
              },
              {
                "name": "conclusion_senescence_unlikely",
                "role": "gate",
                "field": "$.nominated_mechanism_role",
                "kind": "value",
                "op": "in",
                "expected": [
                  "unlikely_major_contributor"
                ]
              },
              {
                "name": "cf_arrest_core_engaged_recall",
                "role": "hard_fail",
                "field": "$.arrest_core_engaged",
                "kind": "value",
                "op": "in",
                "expected": [
                  "arrest_core_engaged"
                ]
              },
              {
                "name": "cf_aging_program_conflation_or_explain_away",
                "role": "hard_fail",
                "field": "$.age_associated_program_identity",
                "kind": "value",
                "op": "in",
                "expected": [
                  "aging_program_indistinguishable_from_nominated_program",
                  "aging_program_is_generic_malignancy_not_aging"
                ]
              },
              {
                "name": "cf_no_aging_program_sensitivity_failure",
                "role": "hard_fail",
                "field": "$.age_associated_program_identity",
                "kind": "value",
                "op": "in",
                "expected": [
                  "no_detectable_aging_program"
                ]
              },
              {
                "name": "cf_senescence_driver_conclusion",
                "role": "hard_fail",
                "field": "$.nominated_mechanism_role",
                "kind": "value",
                "op": "in",
                "expected": [
                  "likely_primary_driver",
                  "plausible_supported_contributor"
                ]
              }
            ]
          },
          "trajectoryFile": "scbench-long/trajectories/gse297023_q2_senescence_not_mechanism__gemini-3.5-flash__pi__r1.json"
        },
        {
          "runId": "gpt-5.5__openai-codex",
          "provider": "OpenAI",
          "model": "gpt-5.5",
          "harness": "openai-codex",
          "run": "r1",
          "trialIndex": 1,
          "passed": false,
          "score": 4,
          "reasoning": "all_of [pass_rule=all]: FAIL\n  + cohort_spans_young_and_aged: passed=True, score=1.0\n  x candidate_is_senescence: passed=False, score=0.0\n  + arrest_core_not_engaged: passed=True, score=1.0\n  + secretory_arm_is_inflammaging: passed=True, score=1.0\n  + secretory_arm_broadly_elevated: passed=True, score=1.0\n  x carries_aging_program: passed=False, score=0.0\n  x conclusion_senescence_unlikely: passed=False, score=0.0\n  + hard_fail 'cf_arrest_core_engaged_recall': not triggered\n  x hard_fail 'cf_aging_program_conflation_or_explain_away': TRIGGERED\n  + hard_fail 'cf_no_aging_program_sensitivity_failure': not triggered\n  x hard_fail 'cf_senescence_driver_conclusion': TRIGGERED",
          "metrics": {
            "pass_rule": "all",
            "scoring_count": 7,
            "scoring_passed": 4,
            "scoring_total_score": 4,
            "hard_fail_triggered": [
              "cf_aging_program_conflation_or_explain_away",
              "cf_senescence_driver_conclusion"
            ]
          },
          "fieldScores": {
            "cohort_spans_young_and_aged": 1,
            "candidate_is_senescence": 0,
            "arrest_core_not_engaged": 1,
            "secretory_arm_is_inflammaging": 1,
            "secretory_arm_broadly_elevated": 1,
            "carries_aging_program": 0,
            "conclusion_senescence_unlikely": 0
          },
          "graderChecks": [
            {
              "name": "cohort_spans_young_and_aged",
              "kind": "gate",
              "passed": true,
              "score": 1
            },
            {
              "name": "candidate_is_senescence",
              "kind": "gate",
              "passed": false,
              "score": 0
            },
            {
              "name": "arrest_core_not_engaged",
              "kind": "gate",
              "passed": true,
              "score": 1
            },
            {
              "name": "secretory_arm_is_inflammaging",
              "kind": "gate",
              "passed": true,
              "score": 1
            },
            {
              "name": "secretory_arm_broadly_elevated",
              "kind": "gate",
              "passed": true,
              "score": 1
            },
            {
              "name": "carries_aging_program",
              "kind": "gate",
              "passed": false,
              "score": 0
            },
            {
              "name": "conclusion_senescence_unlikely",
              "kind": "gate",
              "passed": false,
              "score": 0
            },
            {
              "name": "cf_arrest_core_engaged_recall",
              "kind": "hard_fail",
              "passed": true,
              "triggered": false
            },
            {
              "name": "cf_aging_program_conflation_or_explain_away",
              "kind": "hard_fail",
              "passed": false,
              "triggered": true
            },
            {
              "name": "cf_no_aging_program_sensitivity_failure",
              "kind": "hard_fail",
              "passed": true,
              "triggered": false
            },
            {
              "name": "cf_senescence_driver_conclusion",
              "kind": "hard_fail",
              "passed": false,
              "triggered": true
            }
          ],
          "agentAnswer": {
            "age_associated_program_identity": "aging_program_indistinguishable_from_nominated_program",
            "arrest_core_engaged": "arrest_core_not_engaged",
            "candidate_mechanism": "chronic_inflammation_inflammaging",
            "cohort_host_age_design": "young_and_aged_hosts_both_profiled",
            "nominated_mechanism_role": "plausible_supported_contributor",
            "secretory_arm_direction": "broadly_elevated",
            "secretory_arm_identity": "acute_phase_inflammaging_response"
          },
          "durationS": 376.04,
          "cost": 1.0757430000000001,
          "nTurns": 1,
          "checkpointSignals": {
            "clustering": false,
            "markerAnnotation": true,
            "scoreBasedAnnotation": false,
            "normalization": false,
            "nerveJointAnchor": false,
            "multiomeAssembly": false,
            "depthNormalizedAtac": true,
            "regulonInference": false,
            "cd8SubsetIntegration": true,
            "perPatientEnrichment": false,
            "tcrRichnessMetrics": false,
            "orthologProjection": false,
            "epiSubsetSelection": false,
            "bidirectionalLrTest": false,
            "enrichmentFamilyRanking": false,
            "mousePseudobulkDesign": true,
            "cancerEpithelialDenominator": true,
            "geneLevelDe": true,
            "agingSignatureControl": true,
            "senescencePanelScoring": true
          },
          "expectedAnswer": {
            "passRule": "all",
            "checks": [
              {
                "name": "cohort_spans_young_and_aged",
                "role": "gate",
                "field": "$.cohort_host_age_design",
                "kind": "value",
                "op": "in",
                "expected": [
                  "young_and_aged_hosts_both_profiled"
                ]
              },
              {
                "name": "candidate_is_senescence",
                "role": "gate",
                "field": "$.candidate_mechanism",
                "kind": "value",
                "op": "in",
                "expected": [
                  "cellular_senescence",
                  "oncogene_induced_senescence",
                  "replicative_senescence"
                ]
              },
              {
                "name": "arrest_core_not_engaged",
                "role": "gate",
                "field": "$.arrest_core_engaged",
                "kind": "value",
                "op": "in",
                "expected": [
                  "arrest_core_not_engaged"
                ]
              },
              {
                "name": "secretory_arm_is_inflammaging",
                "role": "gate",
                "field": "$.secretory_arm_identity",
                "kind": "value",
                "op": "in",
                "expected": [
                  "acute_phase_inflammaging_response"
                ]
              },
              {
                "name": "secretory_arm_broadly_elevated",
                "role": "gate",
                "field": "$.secretory_arm_direction",
                "kind": "value",
                "op": "in",
                "expected": [
                  "broadly_elevated"
                ]
              },
              {
                "name": "carries_aging_program",
                "role": "gate",
                "field": "$.age_associated_program_identity",
                "kind": "value",
                "op": "in",
                "expected": [
                  "carries_bona_fide_aging_program_distinct_from_nominated"
                ]
              },
              {
                "name": "conclusion_senescence_unlikely",
                "role": "gate",
                "field": "$.nominated_mechanism_role",
                "kind": "value",
                "op": "in",
                "expected": [
                  "unlikely_major_contributor"
                ]
              },
              {
                "name": "cf_arrest_core_engaged_recall",
                "role": "hard_fail",
                "field": "$.arrest_core_engaged",
                "kind": "value",
                "op": "in",
                "expected": [
                  "arrest_core_engaged"
                ]
              },
              {
                "name": "cf_aging_program_conflation_or_explain_away",
                "role": "hard_fail",
                "field": "$.age_associated_program_identity",
                "kind": "value",
                "op": "in",
                "expected": [
                  "aging_program_indistinguishable_from_nominated_program",
                  "aging_program_is_generic_malignancy_not_aging"
                ]
              },
              {
                "name": "cf_no_aging_program_sensitivity_failure",
                "role": "hard_fail",
                "field": "$.age_associated_program_identity",
                "kind": "value",
                "op": "in",
                "expected": [
                  "no_detectable_aging_program"
                ]
              },
              {
                "name": "cf_senescence_driver_conclusion",
                "role": "hard_fail",
                "field": "$.nominated_mechanism_role",
                "kind": "value",
                "op": "in",
                "expected": [
                  "likely_primary_driver",
                  "plausible_supported_contributor"
                ]
              }
            ]
          },
          "trajectoryFile": "scbench-long/trajectories/gse297023_q2_senescence_not_mechanism__gpt-5.5__openai-codex__r1.json"
        },
        {
          "runId": "grok-4.3__pi",
          "provider": "SpaceXAI",
          "model": "grok-4.3",
          "harness": "pi",
          "run": "r1",
          "trialIndex": 1,
          "passed": false,
          "score": 4,
          "reasoning": "all_of [pass_rule=all]: FAIL\n  + cohort_spans_young_and_aged: passed=True, score=1.0\n  + candidate_is_senescence: passed=True, score=1.0\n  + arrest_core_not_engaged: passed=True, score=1.0\n  x secretory_arm_is_inflammaging: passed=False, score=0.0\n  x secretory_arm_broadly_elevated: passed=False, score=0.0\n  x carries_aging_program: passed=False, score=0.0\n  + conclusion_senescence_unlikely: passed=True, score=1.0\n  + hard_fail 'cf_arrest_core_engaged_recall': not triggered\n  + hard_fail 'cf_aging_program_conflation_or_explain_away': not triggered\n  x hard_fail 'cf_no_aging_program_sensitivity_failure': TRIGGERED\n  + hard_fail 'cf_senescence_driver_conclusion': not triggered",
          "metrics": {
            "pass_rule": "all",
            "scoring_count": 7,
            "scoring_passed": 4,
            "scoring_total_score": 4,
            "hard_fail_triggered": [
              "cf_no_aging_program_sensitivity_failure"
            ]
          },
          "fieldScores": {
            "cohort_spans_young_and_aged": 1,
            "candidate_is_senescence": 1,
            "arrest_core_not_engaged": 1,
            "secretory_arm_is_inflammaging": 0,
            "secretory_arm_broadly_elevated": 0,
            "carries_aging_program": 0,
            "conclusion_senescence_unlikely": 1
          },
          "graderChecks": [
            {
              "name": "cohort_spans_young_and_aged",
              "kind": "gate",
              "passed": true,
              "score": 1
            },
            {
              "name": "candidate_is_senescence",
              "kind": "gate",
              "passed": true,
              "score": 1
            },
            {
              "name": "arrest_core_not_engaged",
              "kind": "gate",
              "passed": true,
              "score": 1
            },
            {
              "name": "secretory_arm_is_inflammaging",
              "kind": "gate",
              "passed": false,
              "score": 0
            },
            {
              "name": "secretory_arm_broadly_elevated",
              "kind": "gate",
              "passed": false,
              "score": 0
            },
            {
              "name": "carries_aging_program",
              "kind": "gate",
              "passed": false,
              "score": 0
            },
            {
              "name": "conclusion_senescence_unlikely",
              "kind": "gate",
              "passed": true,
              "score": 1
            },
            {
              "name": "cf_arrest_core_engaged_recall",
              "kind": "hard_fail",
              "passed": true,
              "triggered": false
            },
            {
              "name": "cf_aging_program_conflation_or_explain_away",
              "kind": "hard_fail",
              "passed": true,
              "triggered": false
            },
            {
              "name": "cf_no_aging_program_sensitivity_failure",
              "kind": "hard_fail",
              "passed": false,
              "triggered": true
            },
            {
              "name": "cf_senescence_driver_conclusion",
              "kind": "hard_fail",
              "passed": true,
              "triggered": false
            }
          ],
          "agentAnswer": {
            "age_associated_program_identity": "no_detectable_aging_program",
            "arrest_core_engaged": "arrest_core_not_engaged",
            "candidate_mechanism": "cellular_senescence",
            "cohort_host_age_design": "young_and_aged_hosts_both_profiled",
            "nominated_mechanism_role": "unlikely_major_contributor",
            "secretory_arm_direction": "broadly_reduced",
            "secretory_arm_identity": "senescence_associated_secretory_phenotype"
          },
          "durationS": 142.02,
          "cost": 0.017675200000000002,
          "nTurns": 8,
          "checkpointSignals": {
            "clustering": false,
            "markerAnnotation": true,
            "scoreBasedAnnotation": false,
            "normalization": false,
            "nerveJointAnchor": false,
            "multiomeAssembly": false,
            "depthNormalizedAtac": false,
            "regulonInference": false,
            "cd8SubsetIntegration": false,
            "perPatientEnrichment": false,
            "tcrRichnessMetrics": false,
            "orthologProjection": false,
            "epiSubsetSelection": false,
            "bidirectionalLrTest": false,
            "enrichmentFamilyRanking": false,
            "mousePseudobulkDesign": false,
            "cancerEpithelialDenominator": true,
            "geneLevelDe": false,
            "agingSignatureControl": true,
            "senescencePanelScoring": true
          },
          "expectedAnswer": {
            "passRule": "all",
            "checks": [
              {
                "name": "cohort_spans_young_and_aged",
                "role": "gate",
                "field": "$.cohort_host_age_design",
                "kind": "value",
                "op": "in",
                "expected": [
                  "young_and_aged_hosts_both_profiled"
                ]
              },
              {
                "name": "candidate_is_senescence",
                "role": "gate",
                "field": "$.candidate_mechanism",
                "kind": "value",
                "op": "in",
                "expected": [
                  "cellular_senescence",
                  "oncogene_induced_senescence",
                  "replicative_senescence"
                ]
              },
              {
                "name": "arrest_core_not_engaged",
                "role": "gate",
                "field": "$.arrest_core_engaged",
                "kind": "value",
                "op": "in",
                "expected": [
                  "arrest_core_not_engaged"
                ]
              },
              {
                "name": "secretory_arm_is_inflammaging",
                "role": "gate",
                "field": "$.secretory_arm_identity",
                "kind": "value",
                "op": "in",
                "expected": [
                  "acute_phase_inflammaging_response"
                ]
              },
              {
                "name": "secretory_arm_broadly_elevated",
                "role": "gate",
                "field": "$.secretory_arm_direction",
                "kind": "value",
                "op": "in",
                "expected": [
                  "broadly_elevated"
                ]
              },
              {
                "name": "carries_aging_program",
                "role": "gate",
                "field": "$.age_associated_program_identity",
                "kind": "value",
                "op": "in",
                "expected": [
                  "carries_bona_fide_aging_program_distinct_from_nominated"
                ]
              },
              {
                "name": "conclusion_senescence_unlikely",
                "role": "gate",
                "field": "$.nominated_mechanism_role",
                "kind": "value",
                "op": "in",
                "expected": [
                  "unlikely_major_contributor"
                ]
              },
              {
                "name": "cf_arrest_core_engaged_recall",
                "role": "hard_fail",
                "field": "$.arrest_core_engaged",
                "kind": "value",
                "op": "in",
                "expected": [
                  "arrest_core_engaged"
                ]
              },
              {
                "name": "cf_aging_program_conflation_or_explain_away",
                "role": "hard_fail",
                "field": "$.age_associated_program_identity",
                "kind": "value",
                "op": "in",
                "expected": [
                  "aging_program_indistinguishable_from_nominated_program",
                  "aging_program_is_generic_malignancy_not_aging"
                ]
              },
              {
                "name": "cf_no_aging_program_sensitivity_failure",
                "role": "hard_fail",
                "field": "$.age_associated_program_identity",
                "kind": "value",
                "op": "in",
                "expected": [
                  "no_detectable_aging_program"
                ]
              },
              {
                "name": "cf_senescence_driver_conclusion",
                "role": "hard_fail",
                "field": "$.nominated_mechanism_role",
                "kind": "value",
                "op": "in",
                "expected": [
                  "likely_primary_driver",
                  "plausible_supported_contributor"
                ]
              }
            ]
          },
          "trajectoryFile": "scbench-long/trajectories/gse297023_q2_senescence_not_mechanism__grok-4.3__pi__r1.json"
        }
      ],
      "sourcePath": "gse297023_q2_senescence_not_mechanism/eval.json",
      "githubUrl": "https://github.com/latchbio/scbench-long/blob/main/evals/gse297023_q2_senescence_not_mechanism/eval.json"
    },
    {
      "id": "10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1",
      "title": "10x 5p Melanoma Swapped Sortgate Cd8 State Data Fidelity Probe V1",
      "description": "Melanoma tumor-reactive CD8 T cells \u00b7 10x 5' scRNA-seq + scTCR-seq: Deliverable on the CD8+ T cells captured by the 'singlets' gate vs the 'tumor_cluster' gate (use only these two gates).",
      "objective": "Deliverable on the CD8+ T cells captured by the 'singlets' gate vs the 'tumor_cluster' gate (use only these two gates).",
      "prompt": "Experimental context. 5' single-cell RNA-sequencing with paired single-cell TCR-sequencing of dissociated human melanoma metastases (n=5 patients). Cells were flow-sorted and the sort gate of each cell is recorded in the cell metadata. Two of the gates are 'singlets' and 'tumor_cluster': the 'tumor_cluster' gate captures CD8+ T cells that were physically clustered/conjugated with tumour cells at the time of sorting, whereas the 'singlets' gate captures unconjugated single CD8+ T cells.\n\nTask. Deliverable on the CD8+ T cells captured by the 'singlets' gate vs the 'tumor_cluster' gate (use only these two gates). Interpret your findings in the context of what each gate physically captures and established CD8+ tumour-infiltrating-lymphocyte biology. (1) Return, for each gate, the 3 CD8 states that best characterize its dominant CD8+ biology, with the functional_category of each; for the gate enriched in the antigen-experienced/exhausted compartment, the three states must span the exhaustion differentiation lineage and include its self-renewing stem-like precursor (resolved from TCF7+ naive cells), not only terminal-exhausted and cycling states. (2) Return the direction of TCR clonal expansion between the two gates. (3) Return the single joint inference of which in-vivo CD8+ TIL subset each gate predominantly captures. (4) In `biological_relations`, report exactly one relation token (from relation_token_vocabulary) for each key listed in biological_relation_vocabulary; each key is named `<quantity>_in_<groupA>_vs_<groupB>` and its token states the relation of <groupA> to <groupB> among CD8+ T cells. Derive every quantity from the data, controlling for per-patient composition and (for clonotype metrics) cell number / sequencing depth; use approximately_equal when there is no robust difference and not_assessable only when a quantity genuinely cannot be derived.\n\nUse cd8_state names, functional_category values, the clonotype_expansion_finding label, the biological_inference label, and the relation tokens from vocabulary.json (provided as a data node).\n\nReturn EXACTLY:\n\n<EVAL_ANSWER>\n{\n  \"cd8_states_enriched_in_singlet_fraction\": [\n    {\"state_label\": \"<cd8_state_vocabulary>\", \"functional_category\": \"<functional_category_vocabulary>\"},\n    {\"state_label\": \"<cd8_state_vocabulary>\", \"functional_category\": \"<functional_category_vocabulary>\"},\n    {\"state_label\": \"<cd8_state_vocabulary>\", \"functional_category\": \"<functional_category_vocabulary>\"}\n  ],\n  \"cd8_states_enriched_in_tumor_cluster_fraction\": [\n    {\"state_label\": \"<cd8_state_vocabulary>\", \"functional_category\": \"<functional_category_vocabulary>\"},\n    {\"state_label\": \"<cd8_state_vocabulary>\", \"functional_category\": \"<functional_category_vocabulary>\"},\n    {\"state_label\": \"<cd8_state_vocabulary>\", \"functional_category\": \"<functional_category_vocabulary>\"}\n  ],\n  \"clonotype_expansion_finding\": \"<clonotype_expansion_finding_vocabulary>\",\n  \"biological_inference\": \"<biological_inference_vocabulary>\",\n  \"biological_relations\": {\n    \"exhaustion_program_in_singlets_vs_tumor_cluster\": \"<relation_token>\",\n    \"naive_memory_program_in_singlets_vs_tumor_cluster\": \"<relation_token>\",\n    \"proliferation_program_in_singlets_vs_tumor_cluster\": \"<relation_token>\",\n    \"clonal_expansion_richness_in_singlets_vs_tumor_cluster\": \"<relation_token>\",\n    \"multichain_tcr_rate_in_singlets_vs_tumor_cluster\": \"<relation_token>\",\n    \"intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster\": \"<relation_token>\",\n    \"melanocytic_transcript_content_in_tumor_cluster_vs_singlets\": \"<relation_token>\"\n  }\n}\n</EVAL_ANSWER>",
      "system": "Melanoma tumor-reactive CD8 T cells",
      "assay": "10x 5' scRNA-seq + scTCR-seq",
      "mainTaskTheme": "Link expression states with TCR clonotypes; interpret marker-sorted populations",
      "graderType": "all_of",
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_singlet_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "TOXhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "LAG3high_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "terminal_exhausted"
                    ],
                    "canonical": "terminal_exhausted"
                  }
                }
              },
              {
                "key": "GZMKhigh_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "intermediate_exhausted_gzmk"
                    ],
                    "canonical": "intermediate_exhausted_gzmk"
                  }
                }
              },
              {
                "key": "TCF7_stem_like_Tex",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "stem_like_exhausted"
                    ],
                    "canonical": "stem_like_exhausted"
                  }
                }
              },
              {
                "key": "MKI67_pos_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              },
              {
                "key": "MKI67high_Tex_Tprol",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "proliferating"
                    ],
                    "canonical": "proliferating"
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd8_states_enriched_in_tumor_cluster_fraction",
            "kind": "list",
            "matchKey": "state_label",
            "k": 3,
            "expected": [
              {
                "key": "Tn",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "Tn_Tmem",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "naive_memory"
                    ],
                    "canonical": "naive_memory"
                  }
                }
              },
              {
                "key": "early_Tem_GZMK",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              },
              {
                "key": "Tem_GZMH",
                "fields": {
                  "functional_category": {
                    "op": "in",
                    "values": [
                      "early_effector_memory"
                    ],
                    "canonical": "early_effector_memory"
                  }
                }
              }
            ]
          },
          {
            "name": "clonotype_expansion_direction_data_true",
            "role": "gate",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_substantially_more_expanded_than_cluster",
              "singlet_modestly_more_expanded_than_cluster"
            ]
          },
          {
            "name": "biological_inference_data_true",
            "role": "gate",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "singlet_antigen_experienced_cluster_bystander"
            ]
          },
          {
            "name": "singlet_top3_contains_naive_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Tn",
              "Tn_Tmem",
              "early_Tem_GZMK",
              "Tem_GZMH"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "TOXhigh_Tex",
              "LAG3high_Tex",
              "GZMKhigh_Tex",
              "TCF7_stem_like_Tex",
              "MKI67_pos_Tex_Tprol",
              "MKI67high_Tex_Tprol"
            ]
          },
          {
            "name": "singlet_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "tumor_cluster_top3_contains_decoy_token",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
            "kind": "avoid",
            "expected": [
              "Treg",
              "Treg_FOXP3",
              "CD4_helper",
              "CD4_T",
              "Th1",
              "Th2",
              "Th17",
              "Tfh",
              "NK_cell",
              "NK_cytotoxic",
              "ILC1",
              "gamma_delta_T",
              "B_cell",
              "plasma_cell",
              "monocyte",
              "macrophage",
              "M1_macrophage",
              "M2_macrophage",
              "DC",
              "mreg_DC",
              "plasmacytoid_DC",
              "T_cell",
              "CD8_T_cell",
              "CD8_T_cell_general",
              "lymphocyte",
              "immune_cell",
              "exhausted_T_cell",
              "effector_T_cell",
              "memory_T_cell",
              "central_memory_T",
              "CD8_TEMRA",
              "CAR_T",
              "activated_T_cell",
              "resting_T_cell",
              "bystander_T",
              "tumor_reactive_T",
              "CD39_pos_T",
              "CD39_neg_T",
              "insufficient_data",
              "cannot_assess",
              "not_determined",
              "unknown_T_state",
              "NA"
            ]
          },
          {
            "name": "TOXhigh_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "TCF7_stem_like_Tex_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "MKI67_state_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "Tn_or_Tn_Tmem_wrong_functional_category",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*]",
            "kind": "avoid",
            "expected": []
          },
          {
            "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_stemlike_precursor_in_singlet_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
            "role": "hard_fail",
            "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].functional_category",
            "kind": "other",
            "op": "none",
            "expected": null
          },
          {
            "name": "cf_reported_canonical_expansion_direction",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_substantially_more_expanded_than_singlet",
              "cluster_modestly_more_expanded_than_singlet"
            ]
          },
          {
            "name": "cf_expansion_lazy_defer",
            "role": "hard_fail",
            "field": "clonotype_expansion_finding",
            "kind": "value",
            "op": "in",
            "expected": [
              "expansion_pattern_not_detectable"
            ]
          },
          {
            "name": "cf_reported_canonical_expected_biology",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "cluster_antigen_experienced_singlet_bystander"
            ]
          },
          {
            "name": "cf_biological_inference_null_artifact_or_lineage",
            "role": "hard_fail",
            "field": "biological_inference",
            "kind": "value",
            "op": "in",
            "expected": [
              "differences_reflect_dissociation_or_sorting_artifact",
              "cluster_and_singlet_capture_same_biology",
              "cluster_and_singlet_capture_distinct_cd8_lineages"
            ]
          },
          {
            "name": "dict_match",
            "role": "gate",
            "field": "biological_relations",
            "kind": "dict",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_greater_than",
                  "greater_than"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "less_than",
                  "much_less_than"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than"
                ]
              },
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "greater_than",
                  "much_greater_than",
                  "approximately_equal"
                ]
              },
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal"
                ]
              },
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "value": [
                  "much_less_than",
                  "less_than",
                  "approximately_equal",
                  "greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_exhaustion_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_naive_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_proliferation_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_expansion_richness_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_multichain_inverted_prior_push",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "less_than",
                  "much_less_than"
                ]
              }
            }
          },
          {
            "name": "cf_intermediate_gzmk_overgeneralized",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "greater_than",
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_melanocytic_conjugate_overcall",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                "op": "in",
                "values": [
                  "much_greater_than"
                ]
              }
            }
          },
          {
            "name": "cf_relations_mass_hedge",
            "role": "hard_fail",
            "field": "biological_relations",
            "kind": "fields",
            "expected": {
              "exhaustion_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "naive_memory_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "proliferation_program_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              },
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                "op": "in",
                "values": [
                  "approximately_equal",
                  "not_assessable"
                ]
              }
            }
          }
        ]
      },
      "dataNodeCount": 3,
      "modules": {
        "graded_biological_result": "On the label-swapped object: singlet-gate top-3 = exhausted/proliferating, tumor_cluster-gate top-3 = naive/early-memory, expansion singlet > cluster (robust richness metrics; the size-weighted/top-K paper metric is a mega-clone-skewed trap recovering the canonical direction), inference = singlet antigen-experienced / cluster bystander. Grades whether the agent reports this data-true (counter-canonical) result rather than the expected 'cluster = reactive' shape.",
        "ungraded_dependency_stack": [
          "QC and per-cell filtering",
          "per-patient log-normalisation + harmony integration",
          "lineage annotation (CD3D/CD8A) and subset to CD8+ T cells",
          "per-patient reintegration on the CD8 subset",
          "gene-mask design (drop mito/ncRNA/Ig/TCR/stress/ribosomal from PCA)",
          "reclustering + contaminant removal (drop CD4/ITGAX/FOXP3 clusters)",
          "state annotation vs external CD8 TIL references",
          "per-cell sort-gate recovery for {singlets, tumor_cluster}",
          "per-state composition test across the two gates (GLMM, patient random effect)",
          "per-(patient, sort_gate) clonotype expansion on paired-TCR cells",
          "joint two-axis inference reported faithfully to the data"
        ]
      },
      "rubricCheckpoints": [
        {
          "index": 1,
          "title": "Scope & answer contract",
          "decision": "emit one parseable <EVAL_ANSWER> with the two 3-tuples ({state_label, functional_category}), clonotype_expansion_finding, biological_inference, and the 7-key biological_relations object \u2014 all controlled tokens; restrict the contrast to singlets and tumor_cluster.",
          "scoring": "0 wrong/unparseable \u00b7 1 right task, contract violations (e.g. relations omitted) \u00b7 2 exact."
        },
        {
          "index": 2,
          "title": "CD8 subset construction & per-patient integration (buildability)",
          "decision": "subset to the two gates; QC/filter; annotate lineage (CD3D/CD8A); isolate a clean CD8 compartment; remove CD4/Treg/doublet contaminants; integrate across the 5 patients before clustering.",
          "scoring": "0 none \u00b7 1 CD8 isolated, integration/decontamination incomplete \u00b7 2 clean integrated CD8 subset."
        },
        {
          "index": 3,
          "title": "State annotation, per-patient enrichment & the stem-like precursor (raised bar)",
          "decision": "annotate CD8 states against TIL references; compute per-gate enrichment per patient with sign-consistency across patients (not a single pooled fold-change); and resolve TCF7_stem_like_Tex (TCF7+ PDCD1+ SLAMF6+ IL7R+ XCL1+) as a state distinct from naive Tn, placing it in the reactive (singlet) pole.",
          "scoring": "0 recall/none or precursor lumped into naive \u00b7 1 per-patient enrichment OR the precursor, not both \u00b7 2 per-patient sign-consistent enrichment driving both lists and the precursor resolved and correctly poled."
        },
        {
          "index": 4,
          "title": "Counter-canonical state poles, faithfully reported",
          "decision": "derive from the data that singlets is the exhausted/proliferating pole and tumor_cluster the naive/early-memory pole, and report it as such.",
          "scoring": "0 canonical poles (CF) \u00b7 1 correct on one list/pole only \u00b7 2 both poles inverted-from-canon, explicitly data-derived."
        },
        {
          "index": 5,
          "title": "TCR clonal expansion \u2014 the metric trap",
          "decision": "build clonotypes from the contig file, join to the GEX, compute per-(patient, gate) expansion on paired-TCR cells, and recognise the direction is metric-dependent: robust population-expansion metrics (fraction-in-expanded-clones, Gini; singlet>cluster in 4/5 patients) give the data-true singlet_*_more_expanded, while size-weighted metrics (mean clone size, top-K share = the paper's Fig 3c headline metric) are skewed by a few mega-clones (P2/P9) and give the OPPOSITE, canonical cluster_more_expanded.",
          "scoring": "0 fell into the trap / lazy-defer \u00b7 1 singlet direction on one metric without noticing the conflict \u00b7 2 recognised the metric conflict, used robust richness + n-control, committed data-true."
        },
        {
          "index": 6,
          "title": "The relation web \u2014 core directions recall-proof; GZMK-intermediate the frontier breakpoint",
          "decision": "derive each of the 7 biological_relations per-quantity from the data (per-patient; depth-controlled for the TCR metrics), not from the pole label, and commit the data-true direction for each.",
          "scoring": "0 whole web backwards / mass-hedge / GZMK or melanocytic veto \u00b7 1 four core directions data-true but a veto slot wrong or asserted-not-measured \u00b7 2 all seven data-true and per-quantity-derived; the top-band mark is the GZMK depletion (much_less/less) recovered with the GZMK-as-early-memory rationale explicit in the trajectory (or the melanocytic \u2248-tell measured) \u2014 a token-only approx/less without that reasoning is strong-but-unaware, not top-band."
        },
        {
          "index": 7,
          "title": "Physical-conjugate consistency \u2014 the whistleblower (gold)",
          "decision": "triangulate the per-gate phenotype against what the gates physically capture (the task states it) and recognise the contradiction: the tumor_cluster gate is naive, unexpanded, melanocytic-poor and dual-TCR-poor relative to singlets \u2014 physically impossible for cells defined by conjugation to a tumour cell. The labels are internally inconsistent with the phenotype.",
          "scoring": "0 absent, or a tell used as false reassurance \u00b7 1 a swap-consistent contrast computed and the backwards direction noted but not connected to gate-label inconsistency \u00b7 2 surfaced the contradiction and explicitly confronted it (gate phenotype inconsistent with the physical conjugate definition; result flagged as counter-canonical)."
        },
        {
          "index": 8,
          "title": "Joint inference & fidelity-over-prior (no laundering)",
          "decision": "integrate state + expansion into singlet_antigen_experienced_cluster_bystander, reported as the data-true conclusion even though it contradicts the canonical expectation.",
          "scoring": "0 canonical/null/artifact (CF) \u00b7 1 data-true phenotype but inference hedged or phenotype-only \u00b7 2 committed data-true joint inference, faithful to the data over the prior."
        }
      ],
      "runs": [
        {
          "runId": "claude-opus-4-7__claude-code",
          "provider": "Anthropic",
          "model": "claude-opus-4-7",
          "harness": "claude-code",
          "run": "r1",
          "trialIndex": 1,
          "passed": false,
          "score": 14,
          "reasoning": "all_of [pass_rule=all]: FAIL\n  + list_match: list_match (answer_field='cd8_states_enriched_in_singlet_fraction'): PASS  (score=3.0)\n  + list_match: list_match (answer_field='cd8_states_enriched_in_tumor_cluster_fraction'): PASS  (score=3.0)\n  + clonotype_expansion_direction_data_true: passed=True, score=1.0\n  + biological_inference_data_true: passed=True, score=1.0\n  x dict_match: dict_match (answer_field='biological_relations'): FAIL  (score=6.0)\n  + hard_fail 'singlet_top3_contains_naive_state_prior_pushing': not triggered\n  + hard_fail 'tumor_cluster_top3_contains_exhausted_state_prior_pushing': not triggered\n  + hard_fail 'singlet_top3_contains_decoy_token': not triggered\n  + hard_fail 'tumor_cluster_top3_contains_decoy_token': not triggered\n  + hard_fail 'TOXhigh_Tex_wrong_functional_category': not triggered\n  + hard_fail 'TCF7_stem_like_Tex_wrong_functional_category': not triggered\n  + hard_fail 'MKI67_state_wrong_functional_category': not triggered\n  + hard_fail 'Tn_or_Tn_Tmem_wrong_functional_category': not triggered\n  + hard_fail 'missing_exhausted_or_proliferating_anchor_in_singlet_top3': not triggered\n  + hard_fail 'missing_stemlike_precursor_in_singlet_top3': not triggered\n  + hard_fail 'missing_naive_or_early_memory_anchor_in_tumor_cluster_top3': not triggered\n  + hard_fail 'cf_reported_canonical_expansion_direction': not triggered\n  + hard_fail 'cf_expansion_lazy_defer': not triggered\n  + hard_fail 'cf_reported_canonical_expected_biology': not triggered\n  + hard_fail 'cf_biological_inference_null_artifact_or_lineage': not triggered\n  + hard_fail 'cf_exhaustion_inverted_prior_push': not triggered\n  + hard_fail 'cf_naive_inverted_prior_push': not triggered\n  + hard_fail 'cf_proliferation_inverted_prior_push': not triggered\n  + hard_fail 'cf_expansion_richness_inverted_prior_push': not triggered\n  + hard_fail 'cf_multichain_inverted_prior_push': not triggered\n  x hard_fail 'cf_intermediate_gzmk_overgeneralized': TRIGGERED\n  + hard_fail 'cf_melanocytic_conjugate_overcall': not triggered\n  + hard_fail 'cf_relations_mass_hedge': not triggered",
          "metrics": {
            "hard_fail_triggered": [
              "cf_intermediate_gzmk_overgeneralized"
            ],
            "pass_rule": "all",
            "scoring_count": 5,
            "scoring_passed": 4,
            "scoring_total_score": 14
          },
          "fieldScores": {
            "biological_inference_data_true": 1,
            "clonotype_expansion_direction_data_true": 1,
            "dict_match": 6,
            "list_match": 3
          },
          "graderChecks": [
            {
              "name": "cd8_states_enriched_in_singlet_fraction",
              "kind": "list_match",
              "passed": true,
              "score": 3
            },
            {
              "name": "cd8_states_enriched_in_tumor_cluster_fraction",
              "kind": "list_match",
              "passed": true,
              "score": 3
            },
            {
              "name": "clonotype_expansion_direction_data_true",
              "kind": "gate",
              "passed": true,
              "score": 1
            },
            {
              "name": "biological_inference_data_true",
              "kind": "gate",
              "passed": true,
              "score": 1
            },
            {
              "name": "biological_relations",
              "kind": "list_match",
              "passed": false,
              "score": 6
            },
            {
              "name": "singlet_top3_contains_naive_state_prior_pushing",
              "kind": "hard_fail",
              "passed": true,
              "triggered": false
            },
            {
              "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
              "kind": "hard_fail",
              "passed": true,
              "triggered": false
            },
            {
              "name": "singlet_top3_contains_decoy_token",
              "kind": "hard_fail",
              "passed": true,
              "triggered": false
            },
            {
              "name": "tumor_cluster_top3_contains_decoy_token",
              "kind": "hard_fail",
              "passed": true,
              "triggered": false
            },
            {
              "name": "TOXhigh_Tex_wrong_functional_category",
              "kind": "hard_fail",
              "passed": true,
              "triggered": false
            },
            {
              "name": "TCF7_stem_like_Tex_wrong_functional_category",
              "kind": "hard_fail",
              "passed": true,
              "triggered": false
            },
            {
              "name": "MKI67_state_wrong_functional_category",
              "kind": "hard_fail",
              "passed": true,
              "triggered": false
            },
            {
              "name": "Tn_or_Tn_Tmem_wrong_functional_category",
              "kind": "hard_fail",
              "passed": true,
              "triggered": false
            },
            {
              "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
              "kind": "hard_fail",
              "passed": true,
              "triggered": false
            },
            {
              "name": "missing_stemlike_precursor_in_singlet_top3",
              "kind": "hard_fail",
              "passed": true,
              "triggered": false
            },
            {
              "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
              "kind": "hard_fail",
              "passed": true,
              "triggered": false
            },
            {
              "name": "cf_reported_canonical_expansion_direction",
              "kind": "hard_fail",
              "passed": true,
              "triggered": false
            },
            {
              "name": "cf_expansion_lazy_defer",
              "kind": "hard_fail",
              "passed": true,
              "triggered": false
            },
            {
              "name": "cf_reported_canonical_expected_biology",
              "kind": "hard_fail",
              "passed": true,
              "triggered": false
            },
            {
              "name": "cf_biological_inference_null_artifact_or_lineage",
              "kind": "hard_fail",
              "passed": true,
              "triggered": false
            },
            {
              "name": "cf_exhaustion_inverted_prior_push",
              "kind": "hard_fail",
              "passed": true,
              "triggered": false
            },
            {
              "name": "cf_naive_inverted_prior_push",
              "kind": "hard_fail",
              "passed": true,
              "triggered": false
            },
            {
              "name": "cf_proliferation_inverted_prior_push",
              "kind": "hard_fail",
              "passed": true,
              "triggered": false
            },
            {
              "name": "cf_expansion_richness_inverted_prior_push",
              "kind": "hard_fail",
              "passed": true,
              "triggered": false
            },
            {
              "name": "cf_multichain_inverted_prior_push",
              "kind": "hard_fail",
              "passed": true,
              "triggered": false
            },
            {
              "name": "cf_intermediate_gzmk_overgeneralized",
              "kind": "hard_fail",
              "passed": false,
              "triggered": true
            },
            {
              "name": "cf_melanocytic_conjugate_overcall",
              "kind": "hard_fail",
              "passed": true,
              "triggered": false
            },
            {
              "name": "cf_relations_mass_hedge",
              "kind": "hard_fail",
              "passed": true,
              "triggered": false
            }
          ],
          "agentAnswer": {
            "biological_inference": "singlet_antigen_experienced_cluster_bystander",
            "biological_relations": {
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": "greater_than",
              "exhaustion_program_in_singlets_vs_tumor_cluster": "much_greater_than",
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": "greater_than",
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": "approximately_equal",
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": "greater_than",
              "naive_memory_program_in_singlets_vs_tumor_cluster": "much_less_than",
              "proliferation_program_in_singlets_vs_tumor_cluster": "greater_than"
            },
            "cd8_states_enriched_in_singlet_fraction": [
              {
                "functional_category": "stem_like_exhausted",
                "state_label": "TCF7_stem_like_Tex"
              },
              {
                "functional_category": "intermediate_exhausted_gzmk",
                "state_label": "GZMKhigh_Tex"
              },
              {
                "functional_category": "terminal_exhausted",
                "state_label": "LAG3high_Tex"
              }
            ],
            "cd8_states_enriched_in_tumor_cluster_fraction": [
              {
                "functional_category": "naive_memory",
                "state_label": "Tn"
              },
              {
                "functional_category": "naive_memory",
                "state_label": "Tn_Tmem"
              },
              {
                "functional_category": "early_effector_memory",
                "state_label": "early_Tem_GZMK"
              }
            ],
            "clonotype_expansion_finding": "singlet_modestly_more_expanded_than_cluster"
          },
          "durationS": 2055.92,
          "cost": 6.07067975,
          "nTurns": 48,
          "checkpointSignals": {
            "clustering": true,
            "markerAnnotation": true,
            "scoreBasedAnnotation": false,
            "normalization": true,
            "nerveJointAnchor": false,
            "multiomeAssembly": false,
            "depthNormalizedAtac": false,
            "regulonInference": false,
            "cd8SubsetIntegration": true,
            "perPatientEnrichment": true,
            "tcrRichnessMetrics": true,
            "orthologProjection": false,
            "epiSubsetSelection": false,
            "bidirectionalLrTest": false,
            "enrichmentFamilyRanking": true,
            "mousePseudobulkDesign": false,
            "cancerEpithelialDenominator": false,
            "geneLevelDe": false,
            "agingSignatureControl": false,
            "senescencePanelScoring": false
          },
          "expectedAnswer": {
            "passRule": "all",
            "checks": [
              {
                "name": "list_match",
                "role": "gate",
                "field": "cd8_states_enriched_in_singlet_fraction",
                "kind": "list",
                "matchKey": "state_label",
                "k": 3,
                "expected": [
                  {
                    "key": "TOXhigh_Tex",
                    "fields": {
                      "functional_category": {
                        "op": "in",
                        "values": [
                          "terminal_exhausted"
                        ],
                        "canonical": "terminal_exhausted"
                      }
                    }
                  },
                  {
                    "key": "LAG3high_Tex",
                    "fields": {
                      "functional_category": {
                        "op": "in",
                        "values": [
                          "terminal_exhausted"
                        ],
                        "canonical": "terminal_exhausted"
                      }
                    }
                  },
                  {
                    "key": "GZMKhigh_Tex",
                    "fields": {
                      "functional_category": {
                        "op": "in",
                        "values": [
                          "intermediate_exhausted_gzmk"
                        ],
                        "canonical": "intermediate_exhausted_gzmk"
                      }
                    }
                  },
                  {
                    "key": "TCF7_stem_like_Tex",
                    "fields": {
                      "functional_category": {
                        "op": "in",
                        "values": [
                          "stem_like_exhausted"
                        ],
                        "canonical": "stem_like_exhausted"
                      }
                    }
                  },
                  {
                    "key": "MKI67_pos_Tex_Tprol",
                    "fields": {
                      "functional_category": {
                        "op": "in",
                        "values": [
                          "proliferating"
                        ],
                        "canonical": "proliferating"
                      }
                    }
                  },
                  {
                    "key": "MKI67high_Tex_Tprol",
                    "fields": {
                      "functional_category": {
                        "op": "in",
                        "values": [
                          "proliferating"
                        ],
                        "canonical": "proliferating"
                      }
                    }
                  }
                ]
              },
              {
                "name": "list_match",
                "role": "gate",
                "field": "cd8_states_enriched_in_tumor_cluster_fraction",
                "kind": "list",
                "matchKey": "state_label",
                "k": 3,
                "expected": [
                  {
                    "key": "Tn",
                    "fields": {
                      "functional_category": {
                        "op": "in",
                        "values": [
                          "naive_memory"
                        ],
                        "canonical": "naive_memory"
                      }
                    }
                  },
                  {
                    "key": "Tn_Tmem",
                    "fields": {
                      "functional_category": {
                        "op": "in",
                        "values": [
                          "naive_memory"
                        ],
                        "canonical": "naive_memory"
                      }
                    }
                  },
                  {
                    "key": "early_Tem_GZMK",
                    "fields": {
                      "functional_category": {
                        "op": "in",
                        "values": [
                          "early_effector_memory"
                        ],
                        "canonical": "early_effector_memory"
                      }
                    }
                  },
                  {
                    "key": "Tem_GZMH",
                    "fields": {
                      "functional_category": {
                        "op": "in",
                        "values": [
                          "early_effector_memory"
                        ],
                        "canonical": "early_effector_memory"
                      }
                    }
                  }
                ]
              },
              {
                "name": "clonotype_expansion_direction_data_true",
                "role": "gate",
                "field": "clonotype_expansion_finding",
                "kind": "value",
                "op": "in",
                "expected": [
                  "singlet_substantially_more_expanded_than_cluster",
                  "singlet_modestly_more_expanded_than_cluster"
                ]
              },
              {
                "name": "biological_inference_data_true",
                "role": "gate",
                "field": "biological_inference",
                "kind": "value",
                "op": "in",
                "expected": [
                  "singlet_antigen_experienced_cluster_bystander"
                ]
              },
              {
                "name": "singlet_top3_contains_naive_state_prior_pushing",
                "role": "hard_fail",
                "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
                "kind": "avoid",
                "expected": [
                  "Tn",
                  "Tn_Tmem",
                  "early_Tem_GZMK",
                  "Tem_GZMH"
                ]
              },
              {
                "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
                "role": "hard_fail",
                "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
                "kind": "avoid",
                "expected": [
                  "TOXhigh_Tex",
                  "LAG3high_Tex",
                  "GZMKhigh_Tex",
                  "TCF7_stem_like_Tex",
                  "MKI67_pos_Tex_Tprol",
                  "MKI67high_Tex_Tprol"
                ]
              },
              {
                "name": "singlet_top3_contains_decoy_token",
                "role": "hard_fail",
                "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
                "kind": "avoid",
                "expected": [
                  "Treg",
                  "Treg_FOXP3",
                  "CD4_helper",
                  "CD4_T",
                  "Th1",
                  "Th2",
                  "Th17",
                  "Tfh",
                  "NK_cell",
                  "NK_cytotoxic",
                  "ILC1",
                  "gamma_delta_T",
                  "B_cell",
                  "plasma_cell",
                  "monocyte",
                  "macrophage",
                  "M1_macrophage",
                  "M2_macrophage",
                  "DC",
                  "mreg_DC",
                  "plasmacytoid_DC",
                  "T_cell",
                  "CD8_T_cell",
                  "CD8_T_cell_general",
                  "lymphocyte",
                  "immune_cell",
                  "exhausted_T_cell",
                  "effector_T_cell",
                  "memory_T_cell",
                  "central_memory_T",
                  "CD8_TEMRA",
                  "CAR_T",
                  "activated_T_cell",
                  "resting_T_cell",
                  "bystander_T",
                  "tumor_reactive_T",
                  "CD39_pos_T",
                  "CD39_neg_T",
                  "insufficient_data",
                  "cannot_assess",
                  "not_determined",
                  "unknown_T_state",
                  "NA"
                ]
              },
              {
                "name": "tumor_cluster_top3_contains_decoy_token",
                "role": "hard_fail",
                "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
                "kind": "avoid",
                "expected": [
                  "Treg",
                  "Treg_FOXP3",
                  "CD4_helper",
                  "CD4_T",
                  "Th1",
                  "Th2",
                  "Th17",
                  "Tfh",
                  "NK_cell",
                  "NK_cytotoxic",
                  "ILC1",
                  "gamma_delta_T",
                  "B_cell",
                  "plasma_cell",
                  "monocyte",
                  "macrophage",
                  "M1_macrophage",
                  "M2_macrophage",
                  "DC",
                  "mreg_DC",
                  "plasmacytoid_DC",
                  "T_cell",
                  "CD8_T_cell",
                  "CD8_T_cell_general",
                  "lymphocyte",
                  "immune_cell",
                  "exhausted_T_cell",
                  "effector_T_cell",
                  "memory_T_cell",
                  "central_memory_T",
                  "CD8_TEMRA",
                  "CAR_T",
                  "activated_T_cell",
                  "resting_T_cell",
                  "bystander_T",
                  "tumor_reactive_T",
                  "CD39_pos_T",
                  "CD39_neg_T",
                  "insufficient_data",
                  "cannot_assess",
                  "not_determined",
                  "unknown_T_state",
                  "NA"
                ]
              },
              {
                "name": "TOXhigh_Tex_wrong_functional_category",
                "role": "hard_fail",
                "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
                "kind": "avoid",
                "expected": []
              },
              {
                "name": "TCF7_stem_like_Tex_wrong_functional_category",
                "role": "hard_fail",
                "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
                "kind": "avoid",
                "expected": []
              },
              {
                "name": "MKI67_state_wrong_functional_category",
                "role": "hard_fail",
                "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
                "kind": "avoid",
                "expected": []
              },
              {
                "name": "Tn_or_Tn_Tmem_wrong_functional_category",
                "role": "hard_fail",
                "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*]",
                "kind": "avoid",
                "expected": []
              },
              {
                "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
                "role": "hard_fail",
                "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
                "kind": "other",
                "op": "none",
                "expected": null
              },
              {
                "name": "missing_stemlike_precursor_in_singlet_top3",
                "role": "hard_fail",
                "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
                "kind": "other",
                "op": "none",
                "expected": null
              },
              {
                "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
                "role": "hard_fail",
                "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].functional_category",
                "kind": "other",
                "op": "none",
                "expected": null
              },
              {
                "name": "cf_reported_canonical_expansion_direction",
                "role": "hard_fail",
                "field": "clonotype_expansion_finding",
                "kind": "value",
                "op": "in",
                "expected": [
                  "cluster_substantially_more_expanded_than_singlet",
                  "cluster_modestly_more_expanded_than_singlet"
                ]
              },
              {
                "name": "cf_expansion_lazy_defer",
                "role": "hard_fail",
                "field": "clonotype_expansion_finding",
                "kind": "value",
                "op": "in",
                "expected": [
                  "expansion_pattern_not_detectable"
                ]
              },
              {
                "name": "cf_reported_canonical_expected_biology",
                "role": "hard_fail",
                "field": "biological_inference",
                "kind": "value",
                "op": "in",
                "expected": [
                  "cluster_antigen_experienced_singlet_bystander"
                ]
              },
              {
                "name": "cf_biological_inference_null_artifact_or_lineage",
                "role": "hard_fail",
                "field": "biological_inference",
                "kind": "value",
                "op": "in",
                "expected": [
                  "differences_reflect_dissociation_or_sorting_artifact",
                  "cluster_and_singlet_capture_same_biology",
                  "cluster_and_singlet_capture_distinct_cd8_lineages"
                ]
              },
              {
                "name": "dict_match",
                "role": "gate",
                "field": "biological_relations",
                "kind": "dict",
                "expected": {
                  "exhaustion_program_in_singlets_vs_tumor_cluster": {
                    "op": "in",
                    "value": [
                      "much_greater_than",
                      "greater_than"
                    ]
                  },
                  "naive_memory_program_in_singlets_vs_tumor_cluster": {
                    "op": "in",
                    "value": [
                      "less_than",
                      "much_less_than"
                    ]
                  },
                  "proliferation_program_in_singlets_vs_tumor_cluster": {
                    "op": "in",
                    "value": [
                      "greater_than",
                      "much_greater_than"
                    ]
                  },
                  "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                    "op": "in",
                    "value": [
                      "greater_than",
                      "much_greater_than"
                    ]
                  },
                  "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                    "op": "in",
                    "value": [
                      "greater_than",
                      "much_greater_than",
                      "approximately_equal"
                    ]
                  },
                  "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                    "op": "in",
                    "value": [
                      "much_less_than",
                      "less_than",
                      "approximately_equal"
                    ]
                  },
                  "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                    "op": "in",
                    "value": [
                      "much_less_than",
                      "less_than",
                      "approximately_equal",
                      "greater_than"
                    ]
                  }
                }
              },
              {
                "name": "cf_exhaustion_inverted_prior_push",
                "role": "hard_fail",
                "field": "biological_relations",
                "kind": "fields",
                "expected": {
                  "exhaustion_program_in_singlets_vs_tumor_cluster": {
                    "op": "in",
                    "values": [
                      "less_than",
                      "much_less_than"
                    ]
                  }
                }
              },
              {
                "name": "cf_naive_inverted_prior_push",
                "role": "hard_fail",
                "field": "biological_relations",
                "kind": "fields",
                "expected": {
                  "naive_memory_program_in_singlets_vs_tumor_cluster": {
                    "op": "in",
                    "values": [
                      "greater_than",
                      "much_greater_than"
                    ]
                  }
                }
              },
              {
                "name": "cf_proliferation_inverted_prior_push",
                "role": "hard_fail",
                "field": "biological_relations",
                "kind": "fields",
                "expected": {
                  "proliferation_program_in_singlets_vs_tumor_cluster": {
                    "op": "in",
                    "values": [
                      "less_than",
                      "much_less_than"
                    ]
                  }
                }
              },
              {
                "name": "cf_expansion_richness_inverted_prior_push",
                "role": "hard_fail",
                "field": "biological_relations",
                "kind": "fields",
                "expected": {
                  "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                    "op": "in",
                    "values": [
                      "less_than",
                      "much_less_than"
                    ]
                  }
                }
              },
              {
                "name": "cf_multichain_inverted_prior_push",
                "role": "hard_fail",
                "field": "biological_relations",
                "kind": "fields",
                "expected": {
                  "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                    "op": "in",
                    "values": [
                      "less_than",
                      "much_less_than"
                    ]
                  }
                }
              },
              {
                "name": "cf_intermediate_gzmk_overgeneralized",
                "role": "hard_fail",
                "field": "biological_relations",
                "kind": "fields",
                "expected": {
                  "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                    "op": "in",
                    "values": [
                      "greater_than",
                      "much_greater_than"
                    ]
                  }
                }
              },
              {
                "name": "cf_melanocytic_conjugate_overcall",
                "role": "hard_fail",
                "field": "biological_relations",
                "kind": "fields",
                "expected": {
                  "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                    "op": "in",
                    "values": [
                      "much_greater_than"
                    ]
                  }
                }
              },
              {
                "name": "cf_relations_mass_hedge",
                "role": "hard_fail",
                "field": "biological_relations",
                "kind": "fields",
                "expected": {
                  "exhaustion_program_in_singlets_vs_tumor_cluster": {
                    "op": "in",
                    "values": [
                      "approximately_equal",
                      "not_assessable"
                    ]
                  },
                  "naive_memory_program_in_singlets_vs_tumor_cluster": {
                    "op": "in",
                    "values": [
                      "approximately_equal",
                      "not_assessable"
                    ]
                  },
                  "proliferation_program_in_singlets_vs_tumor_cluster": {
                    "op": "in",
                    "values": [
                      "approximately_equal",
                      "not_assessable"
                    ]
                  },
                  "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                    "op": "in",
                    "values": [
                      "approximately_equal",
                      "not_assessable"
                    ]
                  }
                }
              }
            ]
          },
          "trajectoryFile": "scbench-long/trajectories/10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1__claude-opus-4-7__claude-code__r1.json"
        },
        {
          "runId": "gemini-3.5-flash__pi",
          "provider": "Google",
          "model": "gemini-3.5-flash",
          "harness": "pi",
          "run": "r1",
          "trialIndex": 1,
          "passed": true,
          "score": 15,
          "reasoning": "all_of [pass_rule=all]: PASS\n  + list_match: list_match (answer_field='cd8_states_enriched_in_singlet_fraction'): PASS  (score=3.0)\n  + list_match: list_match (answer_field='cd8_states_enriched_in_tumor_cluster_fraction'): PASS  (score=3.0)\n  + clonotype_expansion_direction_data_true: passed=True, score=1.0\n  + biological_inference_data_true: passed=True, score=1.0\n  + dict_match: dict_match (answer_field='biological_relations'): PASS  (score=7.0)\n  + hard_fail 'singlet_top3_contains_naive_state_prior_pushing': not triggered\n  + hard_fail 'tumor_cluster_top3_contains_exhausted_state_prior_pushing': not triggered\n  + hard_fail 'singlet_top3_contains_decoy_token': not triggered\n  + hard_fail 'tumor_cluster_top3_contains_decoy_token': not triggered\n  + hard_fail 'TOXhigh_Tex_wrong_functional_category': not triggered\n  + hard_fail 'TCF7_stem_like_Tex_wrong_functional_category': not triggered\n  + hard_fail 'MKI67_state_wrong_functional_category': not triggered\n  + hard_fail 'Tn_or_Tn_Tmem_wrong_functional_category': not triggered\n  + hard_fail 'missing_exhausted_or_proliferating_anchor_in_singlet_top3': not triggered\n  + hard_fail 'missing_stemlike_precursor_in_singlet_top3': not triggered\n  + hard_fail 'missing_naive_or_early_memory_anchor_in_tumor_cluster_top3': not triggered\n  + hard_fail 'cf_reported_canonical_expansion_direction': not triggered\n  + hard_fail 'cf_expansion_lazy_defer': not triggered\n  + hard_fail 'cf_reported_canonical_expected_biology': not triggered\n  + hard_fail 'cf_biological_inference_null_artifact_or_lineage': not triggered\n  + hard_fail 'cf_exhaustion_inverted_prior_push': not triggered\n  + hard_fail 'cf_naive_inverted_prior_push': not triggered\n  + hard_fail 'cf_proliferation_inverted_prior_push': not triggered\n  + hard_fail 'cf_expansion_richness_inverted_prior_push': not triggered\n  + hard_fail 'cf_multichain_inverted_prior_push': not triggered\n  + hard_fail 'cf_intermediate_gzmk_overgeneralized': not triggered\n  + hard_fail 'cf_melanocytic_conjugate_overcall': not triggered\n  + hard_fail 'cf_relations_mass_hedge': not triggered",
          "metrics": {
            "hard_fail_triggered": [],
            "pass_rule": "all",
            "scoring_count": 5,
            "scoring_passed": 5,
            "scoring_total_score": 15
          },
          "fieldScores": {
            "biological_inference_data_true": 1,
            "clonotype_expansion_direction_data_true": 1,
            "dict_match": 7,
            "list_match": 3
          },
          "graderChecks": [
            {
              "name": "cd8_states_enriched_in_singlet_fraction",
              "kind": "list_match",
              "passed": true,
              "score": 3
            },
            {
              "name": "cd8_states_enriched_in_tumor_cluster_fraction",
              "kind": "list_match",
              "passed": true,
              "score": 3
            },
            {
              "name": "clonotype_expansion_direction_data_true",
              "kind": "gate",
              "passed": true,
              "score": 1
            },
            {
              "name": "biological_inference_data_true",
              "kind": "gate",
              "passed": true,
              "score": 1
            },
            {
              "name": "biological_relations",
              "kind": "list_match",
              "passed": true,
              "score": 7
            },
            {
              "name": "singlet_top3_contains_naive_state_prior_pushing",
              "kind": "hard_fail",
              "passed": true,
              "triggered": false
            },
            {
              "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
              "kind": "hard_fail",
              "passed": true,
              "triggered": false
            },
            {
              "name": "singlet_top3_contains_decoy_token",
              "kind": "hard_fail",
              "passed": true,
              "triggered": false
            },
            {
              "name": "tumor_cluster_top3_contains_decoy_token",
              "kind": "hard_fail",
              "passed": true,
              "triggered": false
            },
            {
              "name": "TOXhigh_Tex_wrong_functional_category",
              "kind": "hard_fail",
              "passed": true,
              "triggered": false
            },
            {
              "name": "TCF7_stem_like_Tex_wrong_functional_category",
              "kind": "hard_fail",
              "passed": true,
              "triggered": false
            },
            {
              "name": "MKI67_state_wrong_functional_category",
              "kind": "hard_fail",
              "passed": true,
              "triggered": false
            },
            {
              "name": "Tn_or_Tn_Tmem_wrong_functional_category",
              "kind": "hard_fail",
              "passed": true,
              "triggered": false
            },
            {
              "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
              "kind": "hard_fail",
              "passed": true,
              "triggered": false
            },
            {
              "name": "missing_stemlike_precursor_in_singlet_top3",
              "kind": "hard_fail",
              "passed": true,
              "triggered": false
            },
            {
              "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
              "kind": "hard_fail",
              "passed": true,
              "triggered": false
            },
            {
              "name": "cf_reported_canonical_expansion_direction",
              "kind": "hard_fail",
              "passed": true,
              "triggered": false
            },
            {
              "name": "cf_expansion_lazy_defer",
              "kind": "hard_fail",
              "passed": true,
              "triggered": false
            },
            {
              "name": "cf_reported_canonical_expected_biology",
              "kind": "hard_fail",
              "passed": true,
              "triggered": false
            },
            {
              "name": "cf_biological_inference_null_artifact_or_lineage",
              "kind": "hard_fail",
              "passed": true,
              "triggered": false
            },
            {
              "name": "cf_exhaustion_inverted_prior_push",
              "kind": "hard_fail",
              "passed": true,
              "triggered": false
            },
            {
              "name": "cf_naive_inverted_prior_push",
              "kind": "hard_fail",
              "passed": true,
              "triggered": false
            },
            {
              "name": "cf_proliferation_inverted_prior_push",
              "kind": "hard_fail",
              "passed": true,
              "triggered": false
            },
            {
              "name": "cf_expansion_richness_inverted_prior_push",
              "kind": "hard_fail",
              "passed": true,
              "triggered": false
            },
            {
              "name": "cf_multichain_inverted_prior_push",
              "kind": "hard_fail",
              "passed": true,
              "triggered": false
            },
            {
              "name": "cf_intermediate_gzmk_overgeneralized",
              "kind": "hard_fail",
              "passed": true,
              "triggered": false
            },
            {
              "name": "cf_melanocytic_conjugate_overcall",
              "kind": "hard_fail",
              "passed": true,
              "triggered": false
            },
            {
              "name": "cf_relations_mass_hedge",
              "kind": "hard_fail",
              "passed": true,
              "triggered": false
            }
          ],
          "agentAnswer": {
            "biological_inference": "singlet_antigen_experienced_cluster_bystander",
            "biological_relations": {
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": "greater_than",
              "exhaustion_program_in_singlets_vs_tumor_cluster": "greater_than",
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": "approximately_equal",
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": "approximately_equal",
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": "greater_than",
              "naive_memory_program_in_singlets_vs_tumor_cluster": "less_than",
              "proliferation_program_in_singlets_vs_tumor_cluster": "greater_than"
            },
            "cd8_states_enriched_in_singlet_fraction": [
              {
                "functional_category": "stem_like_exhausted",
                "state_label": "TCF7_stem_like_Tex"
              },
              {
                "functional_category": "intermediate_exhausted_gzmk",
                "state_label": "GZMKhigh_Tex"
              },
              {
                "functional_category": "terminal_exhausted",
                "state_label": "TOXhigh_Tex"
              }
            ],
            "cd8_states_enriched_in_tumor_cluster_fraction": [
              {
                "functional_category": "naive_memory",
                "state_label": "Tn"
              },
              {
                "functional_category": "naive_memory",
                "state_label": "Tn_Tmem"
              },
              {
                "functional_category": "early_effector_memory",
                "state_label": "early_Tem_GZMK"
              }
            ],
            "clonotype_expansion_finding": "singlet_substantially_more_expanded_than_cluster"
          },
          "durationS": 890.56,
          "cost": 1.44600735,
          "nTurns": 51,
          "checkpointSignals": {
            "clustering": true,
            "markerAnnotation": true,
            "scoreBasedAnnotation": false,
            "normalization": false,
            "nerveJointAnchor": false,
            "multiomeAssembly": false,
            "depthNormalizedAtac": false,
            "regulonInference": false,
            "cd8SubsetIntegration": true,
            "perPatientEnrichment": true,
            "tcrRichnessMetrics": true,
            "orthologProjection": false,
            "epiSubsetSelection": false,
            "bidirectionalLrTest": false,
            "enrichmentFamilyRanking": true,
            "mousePseudobulkDesign": false,
            "cancerEpithelialDenominator": false,
            "geneLevelDe": false,
            "agingSignatureControl": false,
            "senescencePanelScoring": false
          },
          "expectedAnswer": {
            "passRule": "all",
            "checks": [
              {
                "name": "list_match",
                "role": "gate",
                "field": "cd8_states_enriched_in_singlet_fraction",
                "kind": "list",
                "matchKey": "state_label",
                "k": 3,
                "expected": [
                  {
                    "key": "TOXhigh_Tex",
                    "fields": {
                      "functional_category": {
                        "op": "in",
                        "values": [
                          "terminal_exhausted"
                        ],
                        "canonical": "terminal_exhausted"
                      }
                    }
                  },
                  {
                    "key": "LAG3high_Tex",
                    "fields": {
                      "functional_category": {
                        "op": "in",
                        "values": [
                          "terminal_exhausted"
                        ],
                        "canonical": "terminal_exhausted"
                      }
                    }
                  },
                  {
                    "key": "GZMKhigh_Tex",
                    "fields": {
                      "functional_category": {
                        "op": "in",
                        "values": [
                          "intermediate_exhausted_gzmk"
                        ],
                        "canonical": "intermediate_exhausted_gzmk"
                      }
                    }
                  },
                  {
                    "key": "TCF7_stem_like_Tex",
                    "fields": {
                      "functional_category": {
                        "op": "in",
                        "values": [
                          "stem_like_exhausted"
                        ],
                        "canonical": "stem_like_exhausted"
                      }
                    }
                  },
                  {
                    "key": "MKI67_pos_Tex_Tprol",
                    "fields": {
                      "functional_category": {
                        "op": "in",
                        "values": [
                          "proliferating"
                        ],
                        "canonical": "proliferating"
                      }
                    }
                  },
                  {
                    "key": "MKI67high_Tex_Tprol",
                    "fields": {
                      "functional_category": {
                        "op": "in",
                        "values": [
                          "proliferating"
                        ],
                        "canonical": "proliferating"
                      }
                    }
                  }
                ]
              },
              {
                "name": "list_match",
                "role": "gate",
                "field": "cd8_states_enriched_in_tumor_cluster_fraction",
                "kind": "list",
                "matchKey": "state_label",
                "k": 3,
                "expected": [
                  {
                    "key": "Tn",
                    "fields": {
                      "functional_category": {
                        "op": "in",
                        "values": [
                          "naive_memory"
                        ],
                        "canonical": "naive_memory"
                      }
                    }
                  },
                  {
                    "key": "Tn_Tmem",
                    "fields": {
                      "functional_category": {
                        "op": "in",
                        "values": [
                          "naive_memory"
                        ],
                        "canonical": "naive_memory"
                      }
                    }
                  },
                  {
                    "key": "early_Tem_GZMK",
                    "fields": {
                      "functional_category": {
                        "op": "in",
                        "values": [
                          "early_effector_memory"
                        ],
                        "canonical": "early_effector_memory"
                      }
                    }
                  },
                  {
                    "key": "Tem_GZMH",
                    "fields": {
                      "functional_category": {
                        "op": "in",
                        "values": [
                          "early_effector_memory"
                        ],
                        "canonical": "early_effector_memory"
                      }
                    }
                  }
                ]
              },
              {
                "name": "clonotype_expansion_direction_data_true",
                "role": "gate",
                "field": "clonotype_expansion_finding",
                "kind": "value",
                "op": "in",
                "expected": [
                  "singlet_substantially_more_expanded_than_cluster",
                  "singlet_modestly_more_expanded_than_cluster"
                ]
              },
              {
                "name": "biological_inference_data_true",
                "role": "gate",
                "field": "biological_inference",
                "kind": "value",
                "op": "in",
                "expected": [
                  "singlet_antigen_experienced_cluster_bystander"
                ]
              },
              {
                "name": "singlet_top3_contains_naive_state_prior_pushing",
                "role": "hard_fail",
                "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
                "kind": "avoid",
                "expected": [
                  "Tn",
                  "Tn_Tmem",
                  "early_Tem_GZMK",
                  "Tem_GZMH"
                ]
              },
              {
                "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
                "role": "hard_fail",
                "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
                "kind": "avoid",
                "expected": [
                  "TOXhigh_Tex",
                  "LAG3high_Tex",
                  "GZMKhigh_Tex",
                  "TCF7_stem_like_Tex",
                  "MKI67_pos_Tex_Tprol",
                  "MKI67high_Tex_Tprol"
                ]
              },
              {
                "name": "singlet_top3_contains_decoy_token",
                "role": "hard_fail",
                "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
                "kind": "avoid",
                "expected": [
                  "Treg",
                  "Treg_FOXP3",
                  "CD4_helper",
                  "CD4_T",
                  "Th1",
                  "Th2",
                  "Th17",
                  "Tfh",
                  "NK_cell",
                  "NK_cytotoxic",
                  "ILC1",
                  "gamma_delta_T",
                  "B_cell",
                  "plasma_cell",
                  "monocyte",
                  "macrophage",
                  "M1_macrophage",
                  "M2_macrophage",
                  "DC",
                  "mreg_DC",
                  "plasmacytoid_DC",
                  "T_cell",
                  "CD8_T_cell",
                  "CD8_T_cell_general",
                  "lymphocyte",
                  "immune_cell",
                  "exhausted_T_cell",
                  "effector_T_cell",
                  "memory_T_cell",
                  "central_memory_T",
                  "CD8_TEMRA",
                  "CAR_T",
                  "activated_T_cell",
                  "resting_T_cell",
                  "bystander_T",
                  "tumor_reactive_T",
                  "CD39_pos_T",
                  "CD39_neg_T",
                  "insufficient_data",
                  "cannot_assess",
                  "not_determined",
                  "unknown_T_state",
                  "NA"
                ]
              },
              {
                "name": "tumor_cluster_top3_contains_decoy_token",
                "role": "hard_fail",
                "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
                "kind": "avoid",
                "expected": [
                  "Treg",
                  "Treg_FOXP3",
                  "CD4_helper",
                  "CD4_T",
                  "Th1",
                  "Th2",
                  "Th17",
                  "Tfh",
                  "NK_cell",
                  "NK_cytotoxic",
                  "ILC1",
                  "gamma_delta_T",
                  "B_cell",
                  "plasma_cell",
                  "monocyte",
                  "macrophage",
                  "M1_macrophage",
                  "M2_macrophage",
                  "DC",
                  "mreg_DC",
                  "plasmacytoid_DC",
                  "T_cell",
                  "CD8_T_cell",
                  "CD8_T_cell_general",
                  "lymphocyte",
                  "immune_cell",
                  "exhausted_T_cell",
                  "effector_T_cell",
                  "memory_T_cell",
                  "central_memory_T",
                  "CD8_TEMRA",
                  "CAR_T",
                  "activated_T_cell",
                  "resting_T_cell",
                  "bystander_T",
                  "tumor_reactive_T",
                  "CD39_pos_T",
                  "CD39_neg_T",
                  "insufficient_data",
                  "cannot_assess",
                  "not_determined",
                  "unknown_T_state",
                  "NA"
                ]
              },
              {
                "name": "TOXhigh_Tex_wrong_functional_category",
                "role": "hard_fail",
                "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
                "kind": "avoid",
                "expected": []
              },
              {
                "name": "TCF7_stem_like_Tex_wrong_functional_category",
                "role": "hard_fail",
                "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
                "kind": "avoid",
                "expected": []
              },
              {
                "name": "MKI67_state_wrong_functional_category",
                "role": "hard_fail",
                "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
                "kind": "avoid",
                "expected": []
              },
              {
                "name": "Tn_or_Tn_Tmem_wrong_functional_category",
                "role": "hard_fail",
                "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*]",
                "kind": "avoid",
                "expected": []
              },
              {
                "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
                "role": "hard_fail",
                "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
                "kind": "other",
                "op": "none",
                "expected": null
              },
              {
                "name": "missing_stemlike_precursor_in_singlet_top3",
                "role": "hard_fail",
                "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
                "kind": "other",
                "op": "none",
                "expected": null
              },
              {
                "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
                "role": "hard_fail",
                "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].functional_category",
                "kind": "other",
                "op": "none",
                "expected": null
              },
              {
                "name": "cf_reported_canonical_expansion_direction",
                "role": "hard_fail",
                "field": "clonotype_expansion_finding",
                "kind": "value",
                "op": "in",
                "expected": [
                  "cluster_substantially_more_expanded_than_singlet",
                  "cluster_modestly_more_expanded_than_singlet"
                ]
              },
              {
                "name": "cf_expansion_lazy_defer",
                "role": "hard_fail",
                "field": "clonotype_expansion_finding",
                "kind": "value",
                "op": "in",
                "expected": [
                  "expansion_pattern_not_detectable"
                ]
              },
              {
                "name": "cf_reported_canonical_expected_biology",
                "role": "hard_fail",
                "field": "biological_inference",
                "kind": "value",
                "op": "in",
                "expected": [
                  "cluster_antigen_experienced_singlet_bystander"
                ]
              },
              {
                "name": "cf_biological_inference_null_artifact_or_lineage",
                "role": "hard_fail",
                "field": "biological_inference",
                "kind": "value",
                "op": "in",
                "expected": [
                  "differences_reflect_dissociation_or_sorting_artifact",
                  "cluster_and_singlet_capture_same_biology",
                  "cluster_and_singlet_capture_distinct_cd8_lineages"
                ]
              },
              {
                "name": "dict_match",
                "role": "gate",
                "field": "biological_relations",
                "kind": "dict",
                "expected": {
                  "exhaustion_program_in_singlets_vs_tumor_cluster": {
                    "op": "in",
                    "value": [
                      "much_greater_than",
                      "greater_than"
                    ]
                  },
                  "naive_memory_program_in_singlets_vs_tumor_cluster": {
                    "op": "in",
                    "value": [
                      "less_than",
                      "much_less_than"
                    ]
                  },
                  "proliferation_program_in_singlets_vs_tumor_cluster": {
                    "op": "in",
                    "value": [
                      "greater_than",
                      "much_greater_than"
                    ]
                  },
                  "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                    "op": "in",
                    "value": [
                      "greater_than",
                      "much_greater_than"
                    ]
                  },
                  "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                    "op": "in",
                    "value": [
                      "greater_than",
                      "much_greater_than",
                      "approximately_equal"
                    ]
                  },
                  "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                    "op": "in",
                    "value": [
                      "much_less_than",
                      "less_than",
                      "approximately_equal"
                    ]
                  },
                  "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                    "op": "in",
                    "value": [
                      "much_less_than",
                      "less_than",
                      "approximately_equal",
                      "greater_than"
                    ]
                  }
                }
              },
              {
                "name": "cf_exhaustion_inverted_prior_push",
                "role": "hard_fail",
                "field": "biological_relations",
                "kind": "fields",
                "expected": {
                  "exhaustion_program_in_singlets_vs_tumor_cluster": {
                    "op": "in",
                    "values": [
                      "less_than",
                      "much_less_than"
                    ]
                  }
                }
              },
              {
                "name": "cf_naive_inverted_prior_push",
                "role": "hard_fail",
                "field": "biological_relations",
                "kind": "fields",
                "expected": {
                  "naive_memory_program_in_singlets_vs_tumor_cluster": {
                    "op": "in",
                    "values": [
                      "greater_than",
                      "much_greater_than"
                    ]
                  }
                }
              },
              {
                "name": "cf_proliferation_inverted_prior_push",
                "role": "hard_fail",
                "field": "biological_relations",
                "kind": "fields",
                "expected": {
                  "proliferation_program_in_singlets_vs_tumor_cluster": {
                    "op": "in",
                    "values": [
                      "less_than",
                      "much_less_than"
                    ]
                  }
                }
              },
              {
                "name": "cf_expansion_richness_inverted_prior_push",
                "role": "hard_fail",
                "field": "biological_relations",
                "kind": "fields",
                "expected": {
                  "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                    "op": "in",
                    "values": [
                      "less_than",
                      "much_less_than"
                    ]
                  }
                }
              },
              {
                "name": "cf_multichain_inverted_prior_push",
                "role": "hard_fail",
                "field": "biological_relations",
                "kind": "fields",
                "expected": {
                  "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                    "op": "in",
                    "values": [
                      "less_than",
                      "much_less_than"
                    ]
                  }
                }
              },
              {
                "name": "cf_intermediate_gzmk_overgeneralized",
                "role": "hard_fail",
                "field": "biological_relations",
                "kind": "fields",
                "expected": {
                  "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                    "op": "in",
                    "values": [
                      "greater_than",
                      "much_greater_than"
                    ]
                  }
                }
              },
              {
                "name": "cf_melanocytic_conjugate_overcall",
                "role": "hard_fail",
                "field": "biological_relations",
                "kind": "fields",
                "expected": {
                  "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                    "op": "in",
                    "values": [
                      "much_greater_than"
                    ]
                  }
                }
              },
              {
                "name": "cf_relations_mass_hedge",
                "role": "hard_fail",
                "field": "biological_relations",
                "kind": "fields",
                "expected": {
                  "exhaustion_program_in_singlets_vs_tumor_cluster": {
                    "op": "in",
                    "values": [
                      "approximately_equal",
                      "not_assessable"
                    ]
                  },
                  "naive_memory_program_in_singlets_vs_tumor_cluster": {
                    "op": "in",
                    "values": [
                      "approximately_equal",
                      "not_assessable"
                    ]
                  },
                  "proliferation_program_in_singlets_vs_tumor_cluster": {
                    "op": "in",
                    "values": [
                      "approximately_equal",
                      "not_assessable"
                    ]
                  },
                  "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                    "op": "in",
                    "values": [
                      "approximately_equal",
                      "not_assessable"
                    ]
                  }
                }
              }
            ]
          },
          "trajectoryFile": "scbench-long/trajectories/10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1__gemini-3.5-flash__pi__r1.json"
        },
        {
          "runId": "gpt-5.5__openai-codex",
          "provider": "OpenAI",
          "model": "gpt-5.5",
          "harness": "openai-codex",
          "run": "r1",
          "trialIndex": 1,
          "passed": false,
          "score": 14,
          "reasoning": "all_of [pass_rule=all]: FAIL\n  + list_match: list_match (answer_field='cd8_states_enriched_in_singlet_fraction'): PASS  (score=3.0)\n  + list_match: list_match (answer_field='cd8_states_enriched_in_tumor_cluster_fraction'): PASS  (score=3.0)\n  + clonotype_expansion_direction_data_true: passed=True, score=1.0\n  + biological_inference_data_true: passed=True, score=1.0\n  x dict_match: dict_match (answer_field='biological_relations'): FAIL  (score=6.0)\n  + hard_fail 'singlet_top3_contains_naive_state_prior_pushing': not triggered\n  + hard_fail 'tumor_cluster_top3_contains_exhausted_state_prior_pushing': not triggered\n  + hard_fail 'singlet_top3_contains_decoy_token': not triggered\n  + hard_fail 'tumor_cluster_top3_contains_decoy_token': not triggered\n  + hard_fail 'TOXhigh_Tex_wrong_functional_category': not triggered\n  + hard_fail 'TCF7_stem_like_Tex_wrong_functional_category': not triggered\n  + hard_fail 'MKI67_state_wrong_functional_category': not triggered\n  + hard_fail 'Tn_or_Tn_Tmem_wrong_functional_category': not triggered\n  + hard_fail 'missing_exhausted_or_proliferating_anchor_in_singlet_top3': not triggered\n  + hard_fail 'missing_stemlike_precursor_in_singlet_top3': not triggered\n  + hard_fail 'missing_naive_or_early_memory_anchor_in_tumor_cluster_top3': not triggered\n  + hard_fail 'cf_reported_canonical_expansion_direction': not triggered\n  + hard_fail 'cf_expansion_lazy_defer': not triggered\n  + hard_fail 'cf_reported_canonical_expected_biology': not triggered\n  + hard_fail 'cf_biological_inference_null_artifact_or_lineage': not triggered\n  + hard_fail 'cf_exhaustion_inverted_prior_push': not triggered\n  + hard_fail 'cf_naive_inverted_prior_push': not triggered\n  + hard_fail 'cf_proliferation_inverted_prior_push': not triggered\n  + hard_fail 'cf_expansion_richness_inverted_prior_push': not triggered\n  + hard_fail 'cf_multichain_inverted_prior_push': not triggered\n  x hard_fail 'cf_intermediate_gzmk_overgeneralized': TRIGGERED\n  + hard_fail 'cf_melanocytic_conjugate_overcall': not triggered\n  + hard_fail 'cf_relations_mass_hedge': not triggered",
          "metrics": {
            "hard_fail_triggered": [
              "cf_intermediate_gzmk_overgeneralized"
            ],
            "pass_rule": "all",
            "scoring_count": 5,
            "scoring_passed": 4,
            "scoring_total_score": 14
          },
          "fieldScores": {
            "biological_inference_data_true": 1,
            "clonotype_expansion_direction_data_true": 1,
            "dict_match": 6,
            "list_match": 3
          },
          "graderChecks": [
            {
              "name": "cd8_states_enriched_in_singlet_fraction",
              "kind": "list_match",
              "passed": true,
              "score": 3
            },
            {
              "name": "cd8_states_enriched_in_tumor_cluster_fraction",
              "kind": "list_match",
              "passed": true,
              "score": 3
            },
            {
              "name": "clonotype_expansion_direction_data_true",
              "kind": "gate",
              "passed": true,
              "score": 1
            },
            {
              "name": "biological_inference_data_true",
              "kind": "gate",
              "passed": true,
              "score": 1
            },
            {
              "name": "biological_relations",
              "kind": "list_match",
              "passed": false,
              "score": 6
            },
            {
              "name": "singlet_top3_contains_naive_state_prior_pushing",
              "kind": "hard_fail",
              "passed": true,
              "triggered": false
            },
            {
              "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
              "kind": "hard_fail",
              "passed": true,
              "triggered": false
            },
            {
              "name": "singlet_top3_contains_decoy_token",
              "kind": "hard_fail",
              "passed": true,
              "triggered": false
            },
            {
              "name": "tumor_cluster_top3_contains_decoy_token",
              "kind": "hard_fail",
              "passed": true,
              "triggered": false
            },
            {
              "name": "TOXhigh_Tex_wrong_functional_category",
              "kind": "hard_fail",
              "passed": true,
              "triggered": false
            },
            {
              "name": "TCF7_stem_like_Tex_wrong_functional_category",
              "kind": "hard_fail",
              "passed": true,
              "triggered": false
            },
            {
              "name": "MKI67_state_wrong_functional_category",
              "kind": "hard_fail",
              "passed": true,
              "triggered": false
            },
            {
              "name": "Tn_or_Tn_Tmem_wrong_functional_category",
              "kind": "hard_fail",
              "passed": true,
              "triggered": false
            },
            {
              "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
              "kind": "hard_fail",
              "passed": true,
              "triggered": false
            },
            {
              "name": "missing_stemlike_precursor_in_singlet_top3",
              "kind": "hard_fail",
              "passed": true,
              "triggered": false
            },
            {
              "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
              "kind": "hard_fail",
              "passed": true,
              "triggered": false
            },
            {
              "name": "cf_reported_canonical_expansion_direction",
              "kind": "hard_fail",
              "passed": true,
              "triggered": false
            },
            {
              "name": "cf_expansion_lazy_defer",
              "kind": "hard_fail",
              "passed": true,
              "triggered": false
            },
            {
              "name": "cf_reported_canonical_expected_biology",
              "kind": "hard_fail",
              "passed": true,
              "triggered": false
            },
            {
              "name": "cf_biological_inference_null_artifact_or_lineage",
              "kind": "hard_fail",
              "passed": true,
              "triggered": false
            },
            {
              "name": "cf_exhaustion_inverted_prior_push",
              "kind": "hard_fail",
              "passed": true,
              "triggered": false
            },
            {
              "name": "cf_naive_inverted_prior_push",
              "kind": "hard_fail",
              "passed": true,
              "triggered": false
            },
            {
              "name": "cf_proliferation_inverted_prior_push",
              "kind": "hard_fail",
              "passed": true,
              "triggered": false
            },
            {
              "name": "cf_expansion_richness_inverted_prior_push",
              "kind": "hard_fail",
              "passed": true,
              "triggered": false
            },
            {
              "name": "cf_multichain_inverted_prior_push",
              "kind": "hard_fail",
              "passed": true,
              "triggered": false
            },
            {
              "name": "cf_intermediate_gzmk_overgeneralized",
              "kind": "hard_fail",
              "passed": false,
              "triggered": true
            },
            {
              "name": "cf_melanocytic_conjugate_overcall",
              "kind": "hard_fail",
              "passed": true,
              "triggered": false
            },
            {
              "name": "cf_relations_mass_hedge",
              "kind": "hard_fail",
              "passed": true,
              "triggered": false
            }
          ],
          "agentAnswer": {
            "biological_inference": "singlet_antigen_experienced_cluster_bystander",
            "biological_relations": {
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": "greater_than",
              "exhaustion_program_in_singlets_vs_tumor_cluster": "greater_than",
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": "greater_than",
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": "less_than",
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": "greater_than",
              "naive_memory_program_in_singlets_vs_tumor_cluster": "less_than",
              "proliferation_program_in_singlets_vs_tumor_cluster": "greater_than"
            },
            "cd8_states_enriched_in_singlet_fraction": [
              {
                "functional_category": "stem_like_exhausted",
                "state_label": "TCF7_stem_like_Tex"
              },
              {
                "functional_category": "intermediate_exhausted_gzmk",
                "state_label": "GZMKhigh_Tex"
              },
              {
                "functional_category": "terminal_exhausted",
                "state_label": "LAG3high_Tex"
              }
            ],
            "cd8_states_enriched_in_tumor_cluster_fraction": [
              {
                "functional_category": "naive_memory",
                "state_label": "Tn"
              },
              {
                "functional_category": "naive_memory",
                "state_label": "Tn_Tmem"
              },
              {
                "functional_category": "early_effector_memory",
                "state_label": "early_Tem_GZMK"
              }
            ],
            "clonotype_expansion_finding": "singlet_modestly_more_expanded_than_cluster"
          },
          "durationS": 522.18,
          "cost": 1.900979,
          "nTurns": 1,
          "checkpointSignals": {
            "clustering": false,
            "markerAnnotation": true,
            "scoreBasedAnnotation": false,
            "normalization": true,
            "nerveJointAnchor": false,
            "multiomeAssembly": false,
            "depthNormalizedAtac": true,
            "regulonInference": false,
            "cd8SubsetIntegration": true,
            "perPatientEnrichment": true,
            "tcrRichnessMetrics": true,
            "orthologProjection": false,
            "epiSubsetSelection": false,
            "bidirectionalLrTest": false,
            "enrichmentFamilyRanking": false,
            "mousePseudobulkDesign": false,
            "cancerEpithelialDenominator": true,
            "geneLevelDe": false,
            "agingSignatureControl": false,
            "senescencePanelScoring": false
          },
          "expectedAnswer": {
            "passRule": "all",
            "checks": [
              {
                "name": "list_match",
                "role": "gate",
                "field": "cd8_states_enriched_in_singlet_fraction",
                "kind": "list",
                "matchKey": "state_label",
                "k": 3,
                "expected": [
                  {
                    "key": "TOXhigh_Tex",
                    "fields": {
                      "functional_category": {
                        "op": "in",
                        "values": [
                          "terminal_exhausted"
                        ],
                        "canonical": "terminal_exhausted"
                      }
                    }
                  },
                  {
                    "key": "LAG3high_Tex",
                    "fields": {
                      "functional_category": {
                        "op": "in",
                        "values": [
                          "terminal_exhausted"
                        ],
                        "canonical": "terminal_exhausted"
                      }
                    }
                  },
                  {
                    "key": "GZMKhigh_Tex",
                    "fields": {
                      "functional_category": {
                        "op": "in",
                        "values": [
                          "intermediate_exhausted_gzmk"
                        ],
                        "canonical": "intermediate_exhausted_gzmk"
                      }
                    }
                  },
                  {
                    "key": "TCF7_stem_like_Tex",
                    "fields": {
                      "functional_category": {
                        "op": "in",
                        "values": [
                          "stem_like_exhausted"
                        ],
                        "canonical": "stem_like_exhausted"
                      }
                    }
                  },
                  {
                    "key": "MKI67_pos_Tex_Tprol",
                    "fields": {
                      "functional_category": {
                        "op": "in",
                        "values": [
                          "proliferating"
                        ],
                        "canonical": "proliferating"
                      }
                    }
                  },
                  {
                    "key": "MKI67high_Tex_Tprol",
                    "fields": {
                      "functional_category": {
                        "op": "in",
                        "values": [
                          "proliferating"
                        ],
                        "canonical": "proliferating"
                      }
                    }
                  }
                ]
              },
              {
                "name": "list_match",
                "role": "gate",
                "field": "cd8_states_enriched_in_tumor_cluster_fraction",
                "kind": "list",
                "matchKey": "state_label",
                "k": 3,
                "expected": [
                  {
                    "key": "Tn",
                    "fields": {
                      "functional_category": {
                        "op": "in",
                        "values": [
                          "naive_memory"
                        ],
                        "canonical": "naive_memory"
                      }
                    }
                  },
                  {
                    "key": "Tn_Tmem",
                    "fields": {
                      "functional_category": {
                        "op": "in",
                        "values": [
                          "naive_memory"
                        ],
                        "canonical": "naive_memory"
                      }
                    }
                  },
                  {
                    "key": "early_Tem_GZMK",
                    "fields": {
                      "functional_category": {
                        "op": "in",
                        "values": [
                          "early_effector_memory"
                        ],
                        "canonical": "early_effector_memory"
                      }
                    }
                  },
                  {
                    "key": "Tem_GZMH",
                    "fields": {
                      "functional_category": {
                        "op": "in",
                        "values": [
                          "early_effector_memory"
                        ],
                        "canonical": "early_effector_memory"
                      }
                    }
                  }
                ]
              },
              {
                "name": "clonotype_expansion_direction_data_true",
                "role": "gate",
                "field": "clonotype_expansion_finding",
                "kind": "value",
                "op": "in",
                "expected": [
                  "singlet_substantially_more_expanded_than_cluster",
                  "singlet_modestly_more_expanded_than_cluster"
                ]
              },
              {
                "name": "biological_inference_data_true",
                "role": "gate",
                "field": "biological_inference",
                "kind": "value",
                "op": "in",
                "expected": [
                  "singlet_antigen_experienced_cluster_bystander"
                ]
              },
              {
                "name": "singlet_top3_contains_naive_state_prior_pushing",
                "role": "hard_fail",
                "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
                "kind": "avoid",
                "expected": [
                  "Tn",
                  "Tn_Tmem",
                  "early_Tem_GZMK",
                  "Tem_GZMH"
                ]
              },
              {
                "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
                "role": "hard_fail",
                "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
                "kind": "avoid",
                "expected": [
                  "TOXhigh_Tex",
                  "LAG3high_Tex",
                  "GZMKhigh_Tex",
                  "TCF7_stem_like_Tex",
                  "MKI67_pos_Tex_Tprol",
                  "MKI67high_Tex_Tprol"
                ]
              },
              {
                "name": "singlet_top3_contains_decoy_token",
                "role": "hard_fail",
                "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
                "kind": "avoid",
                "expected": [
                  "Treg",
                  "Treg_FOXP3",
                  "CD4_helper",
                  "CD4_T",
                  "Th1",
                  "Th2",
                  "Th17",
                  "Tfh",
                  "NK_cell",
                  "NK_cytotoxic",
                  "ILC1",
                  "gamma_delta_T",
                  "B_cell",
                  "plasma_cell",
                  "monocyte",
                  "macrophage",
                  "M1_macrophage",
                  "M2_macrophage",
                  "DC",
                  "mreg_DC",
                  "plasmacytoid_DC",
                  "T_cell",
                  "CD8_T_cell",
                  "CD8_T_cell_general",
                  "lymphocyte",
                  "immune_cell",
                  "exhausted_T_cell",
                  "effector_T_cell",
                  "memory_T_cell",
                  "central_memory_T",
                  "CD8_TEMRA",
                  "CAR_T",
                  "activated_T_cell",
                  "resting_T_cell",
                  "bystander_T",
                  "tumor_reactive_T",
                  "CD39_pos_T",
                  "CD39_neg_T",
                  "insufficient_data",
                  "cannot_assess",
                  "not_determined",
                  "unknown_T_state",
                  "NA"
                ]
              },
              {
                "name": "tumor_cluster_top3_contains_decoy_token",
                "role": "hard_fail",
                "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
                "kind": "avoid",
                "expected": [
                  "Treg",
                  "Treg_FOXP3",
                  "CD4_helper",
                  "CD4_T",
                  "Th1",
                  "Th2",
                  "Th17",
                  "Tfh",
                  "NK_cell",
                  "NK_cytotoxic",
                  "ILC1",
                  "gamma_delta_T",
                  "B_cell",
                  "plasma_cell",
                  "monocyte",
                  "macrophage",
                  "M1_macrophage",
                  "M2_macrophage",
                  "DC",
                  "mreg_DC",
                  "plasmacytoid_DC",
                  "T_cell",
                  "CD8_T_cell",
                  "CD8_T_cell_general",
                  "lymphocyte",
                  "immune_cell",
                  "exhausted_T_cell",
                  "effector_T_cell",
                  "memory_T_cell",
                  "central_memory_T",
                  "CD8_TEMRA",
                  "CAR_T",
                  "activated_T_cell",
                  "resting_T_cell",
                  "bystander_T",
                  "tumor_reactive_T",
                  "CD39_pos_T",
                  "CD39_neg_T",
                  "insufficient_data",
                  "cannot_assess",
                  "not_determined",
                  "unknown_T_state",
                  "NA"
                ]
              },
              {
                "name": "TOXhigh_Tex_wrong_functional_category",
                "role": "hard_fail",
                "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
                "kind": "avoid",
                "expected": []
              },
              {
                "name": "TCF7_stem_like_Tex_wrong_functional_category",
                "role": "hard_fail",
                "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
                "kind": "avoid",
                "expected": []
              },
              {
                "name": "MKI67_state_wrong_functional_category",
                "role": "hard_fail",
                "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
                "kind": "avoid",
                "expected": []
              },
              {
                "name": "Tn_or_Tn_Tmem_wrong_functional_category",
                "role": "hard_fail",
                "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*]",
                "kind": "avoid",
                "expected": []
              },
              {
                "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
                "role": "hard_fail",
                "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
                "kind": "other",
                "op": "none",
                "expected": null
              },
              {
                "name": "missing_stemlike_precursor_in_singlet_top3",
                "role": "hard_fail",
                "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
                "kind": "other",
                "op": "none",
                "expected": null
              },
              {
                "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
                "role": "hard_fail",
                "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].functional_category",
                "kind": "other",
                "op": "none",
                "expected": null
              },
              {
                "name": "cf_reported_canonical_expansion_direction",
                "role": "hard_fail",
                "field": "clonotype_expansion_finding",
                "kind": "value",
                "op": "in",
                "expected": [
                  "cluster_substantially_more_expanded_than_singlet",
                  "cluster_modestly_more_expanded_than_singlet"
                ]
              },
              {
                "name": "cf_expansion_lazy_defer",
                "role": "hard_fail",
                "field": "clonotype_expansion_finding",
                "kind": "value",
                "op": "in",
                "expected": [
                  "expansion_pattern_not_detectable"
                ]
              },
              {
                "name": "cf_reported_canonical_expected_biology",
                "role": "hard_fail",
                "field": "biological_inference",
                "kind": "value",
                "op": "in",
                "expected": [
                  "cluster_antigen_experienced_singlet_bystander"
                ]
              },
              {
                "name": "cf_biological_inference_null_artifact_or_lineage",
                "role": "hard_fail",
                "field": "biological_inference",
                "kind": "value",
                "op": "in",
                "expected": [
                  "differences_reflect_dissociation_or_sorting_artifact",
                  "cluster_and_singlet_capture_same_biology",
                  "cluster_and_singlet_capture_distinct_cd8_lineages"
                ]
              },
              {
                "name": "dict_match",
                "role": "gate",
                "field": "biological_relations",
                "kind": "dict",
                "expected": {
                  "exhaustion_program_in_singlets_vs_tumor_cluster": {
                    "op": "in",
                    "value": [
                      "much_greater_than",
                      "greater_than"
                    ]
                  },
                  "naive_memory_program_in_singlets_vs_tumor_cluster": {
                    "op": "in",
                    "value": [
                      "less_than",
                      "much_less_than"
                    ]
                  },
                  "proliferation_program_in_singlets_vs_tumor_cluster": {
                    "op": "in",
                    "value": [
                      "greater_than",
                      "much_greater_than"
                    ]
                  },
                  "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                    "op": "in",
                    "value": [
                      "greater_than",
                      "much_greater_than"
                    ]
                  },
                  "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                    "op": "in",
                    "value": [
                      "greater_than",
                      "much_greater_than",
                      "approximately_equal"
                    ]
                  },
                  "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                    "op": "in",
                    "value": [
                      "much_less_than",
                      "less_than",
                      "approximately_equal"
                    ]
                  },
                  "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                    "op": "in",
                    "value": [
                      "much_less_than",
                      "less_than",
                      "approximately_equal",
                      "greater_than"
                    ]
                  }
                }
              },
              {
                "name": "cf_exhaustion_inverted_prior_push",
                "role": "hard_fail",
                "field": "biological_relations",
                "kind": "fields",
                "expected": {
                  "exhaustion_program_in_singlets_vs_tumor_cluster": {
                    "op": "in",
                    "values": [
                      "less_than",
                      "much_less_than"
                    ]
                  }
                }
              },
              {
                "name": "cf_naive_inverted_prior_push",
                "role": "hard_fail",
                "field": "biological_relations",
                "kind": "fields",
                "expected": {
                  "naive_memory_program_in_singlets_vs_tumor_cluster": {
                    "op": "in",
                    "values": [
                      "greater_than",
                      "much_greater_than"
                    ]
                  }
                }
              },
              {
                "name": "cf_proliferation_inverted_prior_push",
                "role": "hard_fail",
                "field": "biological_relations",
                "kind": "fields",
                "expected": {
                  "proliferation_program_in_singlets_vs_tumor_cluster": {
                    "op": "in",
                    "values": [
                      "less_than",
                      "much_less_than"
                    ]
                  }
                }
              },
              {
                "name": "cf_expansion_richness_inverted_prior_push",
                "role": "hard_fail",
                "field": "biological_relations",
                "kind": "fields",
                "expected": {
                  "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                    "op": "in",
                    "values": [
                      "less_than",
                      "much_less_than"
                    ]
                  }
                }
              },
              {
                "name": "cf_multichain_inverted_prior_push",
                "role": "hard_fail",
                "field": "biological_relations",
                "kind": "fields",
                "expected": {
                  "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                    "op": "in",
                    "values": [
                      "less_than",
                      "much_less_than"
                    ]
                  }
                }
              },
              {
                "name": "cf_intermediate_gzmk_overgeneralized",
                "role": "hard_fail",
                "field": "biological_relations",
                "kind": "fields",
                "expected": {
                  "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                    "op": "in",
                    "values": [
                      "greater_than",
                      "much_greater_than"
                    ]
                  }
                }
              },
              {
                "name": "cf_melanocytic_conjugate_overcall",
                "role": "hard_fail",
                "field": "biological_relations",
                "kind": "fields",
                "expected": {
                  "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                    "op": "in",
                    "values": [
                      "much_greater_than"
                    ]
                  }
                }
              },
              {
                "name": "cf_relations_mass_hedge",
                "role": "hard_fail",
                "field": "biological_relations",
                "kind": "fields",
                "expected": {
                  "exhaustion_program_in_singlets_vs_tumor_cluster": {
                    "op": "in",
                    "values": [
                      "approximately_equal",
                      "not_assessable"
                    ]
                  },
                  "naive_memory_program_in_singlets_vs_tumor_cluster": {
                    "op": "in",
                    "values": [
                      "approximately_equal",
                      "not_assessable"
                    ]
                  },
                  "proliferation_program_in_singlets_vs_tumor_cluster": {
                    "op": "in",
                    "values": [
                      "approximately_equal",
                      "not_assessable"
                    ]
                  },
                  "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                    "op": "in",
                    "values": [
                      "approximately_equal",
                      "not_assessable"
                    ]
                  }
                }
              }
            ]
          },
          "trajectoryFile": "scbench-long/trajectories/10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1__gpt-5.5__openai-codex__r1.json"
        },
        {
          "runId": "grok-4.3__pi",
          "provider": "SpaceXAI",
          "model": "grok-4.3",
          "harness": "pi",
          "run": "r1",
          "trialIndex": 1,
          "passed": false,
          "score": 3,
          "reasoning": "all_of [pass_rule=all]: FAIL\n  x list_match: list_match (answer_field='cd8_states_enriched_in_singlet_fraction'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='cd8_states_enriched_in_tumor_cluster_fraction'): FAIL  (score=0.0)\n  x clonotype_expansion_direction_data_true: passed=False, score=0.0\n  x biological_inference_data_true: passed=False, score=0.0\n  x dict_match: dict_match (answer_field='biological_relations'): FAIL  (score=3.0)\n  x hard_fail 'singlet_top3_contains_naive_state_prior_pushing': TRIGGERED\n  x hard_fail 'tumor_cluster_top3_contains_exhausted_state_prior_pushing': TRIGGERED\n  + hard_fail 'singlet_top3_contains_decoy_token': not triggered\n  + hard_fail 'tumor_cluster_top3_contains_decoy_token': not triggered\n  + hard_fail 'TOXhigh_Tex_wrong_functional_category': not triggered\n  + hard_fail 'TCF7_stem_like_Tex_wrong_functional_category': not triggered\n  + hard_fail 'MKI67_state_wrong_functional_category': not triggered\n  + hard_fail 'Tn_or_Tn_Tmem_wrong_functional_category': not triggered\n  x hard_fail 'missing_exhausted_or_proliferating_anchor_in_singlet_top3': TRIGGERED\n  x hard_fail 'missing_stemlike_precursor_in_singlet_top3': TRIGGERED\n  x hard_fail 'missing_naive_or_early_memory_anchor_in_tumor_cluster_top3': TRIGGERED\n  x hard_fail 'cf_reported_canonical_expansion_direction': TRIGGERED\n  + hard_fail 'cf_expansion_lazy_defer': not triggered\n  x hard_fail 'cf_reported_canonical_expected_biology': TRIGGERED\n  + hard_fail 'cf_biological_inference_null_artifact_or_lineage': not triggered\n  x hard_fail 'cf_exhaustion_inverted_prior_push': TRIGGERED\n  x hard_fail 'cf_naive_inverted_prior_push': TRIGGERED\n  x hard_fail 'cf_proliferation_inverted_prior_push': TRIGGERED\n  x hard_fail 'cf_expansion_richness_inverted_prior_push': TRIGGERED\n  + hard_fail 'cf_multichain_inverted_prior_push': not triggered\n  + hard_fail 'cf_intermediate_gzmk_overgeneralized': not triggered\n  + hard_fail 'cf_melanocytic_conjugate_overcall': not triggered\n  + hard_fail 'cf_relations_mass_hedge': not triggered",
          "metrics": {
            "hard_fail_triggered": [
              "singlet_top3_contains_naive_state_prior_pushing",
              "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
              "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
              "missing_stemlike_precursor_in_singlet_top3",
              "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
              "cf_reported_canonical_expansion_direction",
              "cf_reported_canonical_expected_biology",
              "cf_exhaustion_inverted_prior_push",
              "cf_naive_inverted_prior_push",
              "cf_proliferation_inverted_prior_push",
              "cf_expansion_richness_inverted_prior_push"
            ],
            "pass_rule": "all",
            "scoring_count": 5,
            "scoring_passed": 0,
            "scoring_total_score": 3
          },
          "fieldScores": {
            "biological_inference_data_true": 0,
            "clonotype_expansion_direction_data_true": 0,
            "dict_match": 3,
            "list_match": 0
          },
          "graderChecks": [
            {
              "name": "cd8_states_enriched_in_singlet_fraction",
              "kind": "list_match",
              "passed": false,
              "score": 0
            },
            {
              "name": "cd8_states_enriched_in_tumor_cluster_fraction",
              "kind": "list_match",
              "passed": false,
              "score": 0
            },
            {
              "name": "clonotype_expansion_direction_data_true",
              "kind": "gate",
              "passed": false,
              "score": 0
            },
            {
              "name": "biological_inference_data_true",
              "kind": "gate",
              "passed": false,
              "score": 0
            },
            {
              "name": "biological_relations",
              "kind": "list_match",
              "passed": false,
              "score": 3
            },
            {
              "name": "singlet_top3_contains_naive_state_prior_pushing",
              "kind": "hard_fail",
              "passed": false,
              "triggered": true
            },
            {
              "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
              "kind": "hard_fail",
              "passed": false,
              "triggered": true
            },
            {
              "name": "singlet_top3_contains_decoy_token",
              "kind": "hard_fail",
              "passed": true,
              "triggered": false
            },
            {
              "name": "tumor_cluster_top3_contains_decoy_token",
              "kind": "hard_fail",
              "passed": true,
              "triggered": false
            },
            {
              "name": "TOXhigh_Tex_wrong_functional_category",
              "kind": "hard_fail",
              "passed": true,
              "triggered": false
            },
            {
              "name": "TCF7_stem_like_Tex_wrong_functional_category",
              "kind": "hard_fail",
              "passed": true,
              "triggered": false
            },
            {
              "name": "MKI67_state_wrong_functional_category",
              "kind": "hard_fail",
              "passed": true,
              "triggered": false
            },
            {
              "name": "Tn_or_Tn_Tmem_wrong_functional_category",
              "kind": "hard_fail",
              "passed": true,
              "triggered": false
            },
            {
              "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
              "kind": "hard_fail",
              "passed": false,
              "triggered": true
            },
            {
              "name": "missing_stemlike_precursor_in_singlet_top3",
              "kind": "hard_fail",
              "passed": false,
              "triggered": true
            },
            {
              "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
              "kind": "hard_fail",
              "passed": false,
              "triggered": true
            },
            {
              "name": "cf_reported_canonical_expansion_direction",
              "kind": "hard_fail",
              "passed": false,
              "triggered": true
            },
            {
              "name": "cf_expansion_lazy_defer",
              "kind": "hard_fail",
              "passed": true,
              "triggered": false
            },
            {
              "name": "cf_reported_canonical_expected_biology",
              "kind": "hard_fail",
              "passed": false,
              "triggered": true
            },
            {
              "name": "cf_biological_inference_null_artifact_or_lineage",
              "kind": "hard_fail",
              "passed": true,
              "triggered": false
            },
            {
              "name": "cf_exhaustion_inverted_prior_push",
              "kind": "hard_fail",
              "passed": false,
              "triggered": true
            },
            {
              "name": "cf_naive_inverted_prior_push",
              "kind": "hard_fail",
              "passed": false,
              "triggered": true
            },
            {
              "name": "cf_proliferation_inverted_prior_push",
              "kind": "hard_fail",
              "passed": false,
              "triggered": true
            },
            {
              "name": "cf_expansion_richness_inverted_prior_push",
              "kind": "hard_fail",
              "passed": false,
              "triggered": true
            },
            {
              "name": "cf_multichain_inverted_prior_push",
              "kind": "hard_fail",
              "passed": true,
              "triggered": false
            },
            {
              "name": "cf_intermediate_gzmk_overgeneralized",
              "kind": "hard_fail",
              "passed": true,
              "triggered": false
            },
            {
              "name": "cf_melanocytic_conjugate_overcall",
              "kind": "hard_fail",
              "passed": true,
              "triggered": false
            },
            {
              "name": "cf_relations_mass_hedge",
              "kind": "hard_fail",
              "passed": true,
              "triggered": false
            }
          ],
          "agentAnswer": {
            "biological_inference": "cluster_antigen_experienced_singlet_bystander",
            "biological_relations": {
              "clonal_expansion_richness_in_singlets_vs_tumor_cluster": "much_less_than",
              "exhaustion_program_in_singlets_vs_tumor_cluster": "much_less_than",
              "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": "less_than",
              "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": "greater_than",
              "multichain_tcr_rate_in_singlets_vs_tumor_cluster": "approximately_equal",
              "naive_memory_program_in_singlets_vs_tumor_cluster": "much_greater_than",
              "proliferation_program_in_singlets_vs_tumor_cluster": "less_than"
            },
            "cd8_states_enriched_in_singlet_fraction": [
              {
                "functional_category": "naive_memory",
                "state_label": "Tn"
              },
              {
                "functional_category": "naive_memory",
                "state_label": "Tn_Tmem"
              },
              {
                "functional_category": "early_effector_memory",
                "state_label": "early_Tem_GZMK"
              }
            ],
            "cd8_states_enriched_in_tumor_cluster_fraction": [
              {
                "functional_category": "stem_like_exhausted",
                "state_label": "TCF7_stem_like_Tex"
              },
              {
                "functional_category": "intermediate_exhausted_gzmk",
                "state_label": "GZMKhigh_Tex"
              },
              {
                "functional_category": "terminal_exhausted",
                "state_label": "TOXhigh_Tex"
              }
            ],
            "clonotype_expansion_finding": "cluster_substantially_more_expanded_than_singlet"
          },
          "durationS": 323.32,
          "cost": 0.058377000000000005,
          "nTurns": 9,
          "checkpointSignals": {
            "clustering": true,
            "markerAnnotation": true,
            "scoreBasedAnnotation": false,
            "normalization": false,
            "nerveJointAnchor": false,
            "multiomeAssembly": false,
            "depthNormalizedAtac": false,
            "regulonInference": false,
            "cd8SubsetIntegration": true,
            "perPatientEnrichment": false,
            "tcrRichnessMetrics": true,
            "orthologProjection": false,
            "epiSubsetSelection": false,
            "bidirectionalLrTest": false,
            "enrichmentFamilyRanking": false,
            "mousePseudobulkDesign": false,
            "cancerEpithelialDenominator": false,
            "geneLevelDe": false,
            "agingSignatureControl": false,
            "senescencePanelScoring": false
          },
          "expectedAnswer": {
            "passRule": "all",
            "checks": [
              {
                "name": "list_match",
                "role": "gate",
                "field": "cd8_states_enriched_in_singlet_fraction",
                "kind": "list",
                "matchKey": "state_label",
                "k": 3,
                "expected": [
                  {
                    "key": "TOXhigh_Tex",
                    "fields": {
                      "functional_category": {
                        "op": "in",
                        "values": [
                          "terminal_exhausted"
                        ],
                        "canonical": "terminal_exhausted"
                      }
                    }
                  },
                  {
                    "key": "LAG3high_Tex",
                    "fields": {
                      "functional_category": {
                        "op": "in",
                        "values": [
                          "terminal_exhausted"
                        ],
                        "canonical": "terminal_exhausted"
                      }
                    }
                  },
                  {
                    "key": "GZMKhigh_Tex",
                    "fields": {
                      "functional_category": {
                        "op": "in",
                        "values": [
                          "intermediate_exhausted_gzmk"
                        ],
                        "canonical": "intermediate_exhausted_gzmk"
                      }
                    }
                  },
                  {
                    "key": "TCF7_stem_like_Tex",
                    "fields": {
                      "functional_category": {
                        "op": "in",
                        "values": [
                          "stem_like_exhausted"
                        ],
                        "canonical": "stem_like_exhausted"
                      }
                    }
                  },
                  {
                    "key": "MKI67_pos_Tex_Tprol",
                    "fields": {
                      "functional_category": {
                        "op": "in",
                        "values": [
                          "proliferating"
                        ],
                        "canonical": "proliferating"
                      }
                    }
                  },
                  {
                    "key": "MKI67high_Tex_Tprol",
                    "fields": {
                      "functional_category": {
                        "op": "in",
                        "values": [
                          "proliferating"
                        ],
                        "canonical": "proliferating"
                      }
                    }
                  }
                ]
              },
              {
                "name": "list_match",
                "role": "gate",
                "field": "cd8_states_enriched_in_tumor_cluster_fraction",
                "kind": "list",
                "matchKey": "state_label",
                "k": 3,
                "expected": [
                  {
                    "key": "Tn",
                    "fields": {
                      "functional_category": {
                        "op": "in",
                        "values": [
                          "naive_memory"
                        ],
                        "canonical": "naive_memory"
                      }
                    }
                  },
                  {
                    "key": "Tn_Tmem",
                    "fields": {
                      "functional_category": {
                        "op": "in",
                        "values": [
                          "naive_memory"
                        ],
                        "canonical": "naive_memory"
                      }
                    }
                  },
                  {
                    "key": "early_Tem_GZMK",
                    "fields": {
                      "functional_category": {
                        "op": "in",
                        "values": [
                          "early_effector_memory"
                        ],
                        "canonical": "early_effector_memory"
                      }
                    }
                  },
                  {
                    "key": "Tem_GZMH",
                    "fields": {
                      "functional_category": {
                        "op": "in",
                        "values": [
                          "early_effector_memory"
                        ],
                        "canonical": "early_effector_memory"
                      }
                    }
                  }
                ]
              },
              {
                "name": "clonotype_expansion_direction_data_true",
                "role": "gate",
                "field": "clonotype_expansion_finding",
                "kind": "value",
                "op": "in",
                "expected": [
                  "singlet_substantially_more_expanded_than_cluster",
                  "singlet_modestly_more_expanded_than_cluster"
                ]
              },
              {
                "name": "biological_inference_data_true",
                "role": "gate",
                "field": "biological_inference",
                "kind": "value",
                "op": "in",
                "expected": [
                  "singlet_antigen_experienced_cluster_bystander"
                ]
              },
              {
                "name": "singlet_top3_contains_naive_state_prior_pushing",
                "role": "hard_fail",
                "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
                "kind": "avoid",
                "expected": [
                  "Tn",
                  "Tn_Tmem",
                  "early_Tem_GZMK",
                  "Tem_GZMH"
                ]
              },
              {
                "name": "tumor_cluster_top3_contains_exhausted_state_prior_pushing",
                "role": "hard_fail",
                "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
                "kind": "avoid",
                "expected": [
                  "TOXhigh_Tex",
                  "LAG3high_Tex",
                  "GZMKhigh_Tex",
                  "TCF7_stem_like_Tex",
                  "MKI67_pos_Tex_Tprol",
                  "MKI67high_Tex_Tprol"
                ]
              },
              {
                "name": "singlet_top3_contains_decoy_token",
                "role": "hard_fail",
                "field": "$.cd8_states_enriched_in_singlet_fraction[*].state_label",
                "kind": "avoid",
                "expected": [
                  "Treg",
                  "Treg_FOXP3",
                  "CD4_helper",
                  "CD4_T",
                  "Th1",
                  "Th2",
                  "Th17",
                  "Tfh",
                  "NK_cell",
                  "NK_cytotoxic",
                  "ILC1",
                  "gamma_delta_T",
                  "B_cell",
                  "plasma_cell",
                  "monocyte",
                  "macrophage",
                  "M1_macrophage",
                  "M2_macrophage",
                  "DC",
                  "mreg_DC",
                  "plasmacytoid_DC",
                  "T_cell",
                  "CD8_T_cell",
                  "CD8_T_cell_general",
                  "lymphocyte",
                  "immune_cell",
                  "exhausted_T_cell",
                  "effector_T_cell",
                  "memory_T_cell",
                  "central_memory_T",
                  "CD8_TEMRA",
                  "CAR_T",
                  "activated_T_cell",
                  "resting_T_cell",
                  "bystander_T",
                  "tumor_reactive_T",
                  "CD39_pos_T",
                  "CD39_neg_T",
                  "insufficient_data",
                  "cannot_assess",
                  "not_determined",
                  "unknown_T_state",
                  "NA"
                ]
              },
              {
                "name": "tumor_cluster_top3_contains_decoy_token",
                "role": "hard_fail",
                "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].state_label",
                "kind": "avoid",
                "expected": [
                  "Treg",
                  "Treg_FOXP3",
                  "CD4_helper",
                  "CD4_T",
                  "Th1",
                  "Th2",
                  "Th17",
                  "Tfh",
                  "NK_cell",
                  "NK_cytotoxic",
                  "ILC1",
                  "gamma_delta_T",
                  "B_cell",
                  "plasma_cell",
                  "monocyte",
                  "macrophage",
                  "M1_macrophage",
                  "M2_macrophage",
                  "DC",
                  "mreg_DC",
                  "plasmacytoid_DC",
                  "T_cell",
                  "CD8_T_cell",
                  "CD8_T_cell_general",
                  "lymphocyte",
                  "immune_cell",
                  "exhausted_T_cell",
                  "effector_T_cell",
                  "memory_T_cell",
                  "central_memory_T",
                  "CD8_TEMRA",
                  "CAR_T",
                  "activated_T_cell",
                  "resting_T_cell",
                  "bystander_T",
                  "tumor_reactive_T",
                  "CD39_pos_T",
                  "CD39_neg_T",
                  "insufficient_data",
                  "cannot_assess",
                  "not_determined",
                  "unknown_T_state",
                  "NA"
                ]
              },
              {
                "name": "TOXhigh_Tex_wrong_functional_category",
                "role": "hard_fail",
                "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
                "kind": "avoid",
                "expected": []
              },
              {
                "name": "TCF7_stem_like_Tex_wrong_functional_category",
                "role": "hard_fail",
                "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
                "kind": "avoid",
                "expected": []
              },
              {
                "name": "MKI67_state_wrong_functional_category",
                "role": "hard_fail",
                "field": "$.cd8_states_enriched_in_singlet_fraction[*]",
                "kind": "avoid",
                "expected": []
              },
              {
                "name": "Tn_or_Tn_Tmem_wrong_functional_category",
                "role": "hard_fail",
                "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*]",
                "kind": "avoid",
                "expected": []
              },
              {
                "name": "missing_exhausted_or_proliferating_anchor_in_singlet_top3",
                "role": "hard_fail",
                "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
                "kind": "other",
                "op": "none",
                "expected": null
              },
              {
                "name": "missing_stemlike_precursor_in_singlet_top3",
                "role": "hard_fail",
                "field": "$.cd8_states_enriched_in_singlet_fraction[*].functional_category",
                "kind": "other",
                "op": "none",
                "expected": null
              },
              {
                "name": "missing_naive_or_early_memory_anchor_in_tumor_cluster_top3",
                "role": "hard_fail",
                "field": "$.cd8_states_enriched_in_tumor_cluster_fraction[*].functional_category",
                "kind": "other",
                "op": "none",
                "expected": null
              },
              {
                "name": "cf_reported_canonical_expansion_direction",
                "role": "hard_fail",
                "field": "clonotype_expansion_finding",
                "kind": "value",
                "op": "in",
                "expected": [
                  "cluster_substantially_more_expanded_than_singlet",
                  "cluster_modestly_more_expanded_than_singlet"
                ]
              },
              {
                "name": "cf_expansion_lazy_defer",
                "role": "hard_fail",
                "field": "clonotype_expansion_finding",
                "kind": "value",
                "op": "in",
                "expected": [
                  "expansion_pattern_not_detectable"
                ]
              },
              {
                "name": "cf_reported_canonical_expected_biology",
                "role": "hard_fail",
                "field": "biological_inference",
                "kind": "value",
                "op": "in",
                "expected": [
                  "cluster_antigen_experienced_singlet_bystander"
                ]
              },
              {
                "name": "cf_biological_inference_null_artifact_or_lineage",
                "role": "hard_fail",
                "field": "biological_inference",
                "kind": "value",
                "op": "in",
                "expected": [
                  "differences_reflect_dissociation_or_sorting_artifact",
                  "cluster_and_singlet_capture_same_biology",
                  "cluster_and_singlet_capture_distinct_cd8_lineages"
                ]
              },
              {
                "name": "dict_match",
                "role": "gate",
                "field": "biological_relations",
                "kind": "dict",
                "expected": {
                  "exhaustion_program_in_singlets_vs_tumor_cluster": {
                    "op": "in",
                    "value": [
                      "much_greater_than",
                      "greater_than"
                    ]
                  },
                  "naive_memory_program_in_singlets_vs_tumor_cluster": {
                    "op": "in",
                    "value": [
                      "less_than",
                      "much_less_than"
                    ]
                  },
                  "proliferation_program_in_singlets_vs_tumor_cluster": {
                    "op": "in",
                    "value": [
                      "greater_than",
                      "much_greater_than"
                    ]
                  },
                  "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                    "op": "in",
                    "value": [
                      "greater_than",
                      "much_greater_than"
                    ]
                  },
                  "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                    "op": "in",
                    "value": [
                      "greater_than",
                      "much_greater_than",
                      "approximately_equal"
                    ]
                  },
                  "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                    "op": "in",
                    "value": [
                      "much_less_than",
                      "less_than",
                      "approximately_equal"
                    ]
                  },
                  "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                    "op": "in",
                    "value": [
                      "much_less_than",
                      "less_than",
                      "approximately_equal",
                      "greater_than"
                    ]
                  }
                }
              },
              {
                "name": "cf_exhaustion_inverted_prior_push",
                "role": "hard_fail",
                "field": "biological_relations",
                "kind": "fields",
                "expected": {
                  "exhaustion_program_in_singlets_vs_tumor_cluster": {
                    "op": "in",
                    "values": [
                      "less_than",
                      "much_less_than"
                    ]
                  }
                }
              },
              {
                "name": "cf_naive_inverted_prior_push",
                "role": "hard_fail",
                "field": "biological_relations",
                "kind": "fields",
                "expected": {
                  "naive_memory_program_in_singlets_vs_tumor_cluster": {
                    "op": "in",
                    "values": [
                      "greater_than",
                      "much_greater_than"
                    ]
                  }
                }
              },
              {
                "name": "cf_proliferation_inverted_prior_push",
                "role": "hard_fail",
                "field": "biological_relations",
                "kind": "fields",
                "expected": {
                  "proliferation_program_in_singlets_vs_tumor_cluster": {
                    "op": "in",
                    "values": [
                      "less_than",
                      "much_less_than"
                    ]
                  }
                }
              },
              {
                "name": "cf_expansion_richness_inverted_prior_push",
                "role": "hard_fail",
                "field": "biological_relations",
                "kind": "fields",
                "expected": {
                  "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                    "op": "in",
                    "values": [
                      "less_than",
                      "much_less_than"
                    ]
                  }
                }
              },
              {
                "name": "cf_multichain_inverted_prior_push",
                "role": "hard_fail",
                "field": "biological_relations",
                "kind": "fields",
                "expected": {
                  "multichain_tcr_rate_in_singlets_vs_tumor_cluster": {
                    "op": "in",
                    "values": [
                      "less_than",
                      "much_less_than"
                    ]
                  }
                }
              },
              {
                "name": "cf_intermediate_gzmk_overgeneralized",
                "role": "hard_fail",
                "field": "biological_relations",
                "kind": "fields",
                "expected": {
                  "intermediate_exhausted_gzmk_frequency_in_singlets_vs_tumor_cluster": {
                    "op": "in",
                    "values": [
                      "greater_than",
                      "much_greater_than"
                    ]
                  }
                }
              },
              {
                "name": "cf_melanocytic_conjugate_overcall",
                "role": "hard_fail",
                "field": "biological_relations",
                "kind": "fields",
                "expected": {
                  "melanocytic_transcript_content_in_tumor_cluster_vs_singlets": {
                    "op": "in",
                    "values": [
                      "much_greater_than"
                    ]
                  }
                }
              },
              {
                "name": "cf_relations_mass_hedge",
                "role": "hard_fail",
                "field": "biological_relations",
                "kind": "fields",
                "expected": {
                  "exhaustion_program_in_singlets_vs_tumor_cluster": {
                    "op": "in",
                    "values": [
                      "approximately_equal",
                      "not_assessable"
                    ]
                  },
                  "naive_memory_program_in_singlets_vs_tumor_cluster": {
                    "op": "in",
                    "values": [
                      "approximately_equal",
                      "not_assessable"
                    ]
                  },
                  "proliferation_program_in_singlets_vs_tumor_cluster": {
                    "op": "in",
                    "values": [
                      "approximately_equal",
                      "not_assessable"
                    ]
                  },
                  "clonal_expansion_richness_in_singlets_vs_tumor_cluster": {
                    "op": "in",
                    "values": [
                      "approximately_equal",
                      "not_assessable"
                    ]
                  }
                }
              }
            ]
          },
          "trajectoryFile": "scbench-long/trajectories/10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1__grok-4.3__pi__r1.json"
        }
      ],
      "sourcePath": "10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1/eval.json",
      "githubUrl": "https://github.com/latchbio/scbench-long/blob/main/evals/10x_5p_melanoma_swapped_sortgate_cd8_state_data_fidelity_probe_v1/eval.json"
    },
    {
      "id": "green_2_tme_context_dependence",
      "title": "Green 2 Tme Context Dependence",
      "description": "CD8 RNA+ATAC regulatory inference \u00b7 10x Multiome (snRNA-seq + scATAC-seq): For tumour-infiltrating CD8 T cells, dissect the regulatory basis of the CD103+ Trm-like program across the four GP33-matched models, using the infection arms as references.",
      "objective": "For tumour-infiltrating CD8 T cells, dissect the regulatory basis of the CD103+ Trm-like program across the four GP33-matched models, using the infection arms as references.",
      "prompt": "Experimental context. Paired scRNA-seq + scATAC-seq (10x Multiome) of P14 CD8 T cells (GP33-41-specific TCR-transgenic) spanning LCMV infection (Arm spleen/IEL, CL13 spleen, naive) and four GP33-41-expressing syngeneic tumour models. RNA is one .h5ad. ATAC is per-sample atac_fragments.tsv.gz. Condition, model, and sort-gate labels are in the cell metadata.\n\nTask. For tumour-infiltrating CD8 T cells, dissect the regulatory basis of the CD103+ Trm-like program across the four GP33-matched models, using the infection arms as references. The four models share one TCR and one epitope, so they form a controlled panel: differences between them are not antigen or affinity. Quantify the program per model, determine where and how it is controlled, and decide what does and does not separate the models.\n\nReturn EXACTLY:\n<EVAL_ANSWER>\n{\"core_residency_signature\": [\"<gene>\"], \"excluded_confounders\": [\"<gene>\"], \"generic_signature_relations\": [{\"pair\": \"<MODEL>_vs_<MODEL>\", \"relation\": \"<relation>\"}], \"cd103_program_relations\": [{\"pair\": \"<MODEL>_vs_<MODEL>\", \"relation\": \"<relation>\"}], \"program_map\": [{\"model\": \"<model>\", \"call\": \"<program_call>\"}], \"per_model_regulatory_state\": [{\"model\": \"<model>\", \"program\": \"<program_call>\", \"brake_state\": \"<brake_state>\", \"activator_state\": \"<activator_state>\", \"locus_state\": \"<locus_state>\", \"blocking_step\": \"<blocking_step>\"}], \"regulator_tuples\": [{\"regulator\": \"<gene>\", \"role\": \"<regulator_role>\", \"gating_step\": \"<gating_step>\", \"evidence_basis\": \"<evidence_basis>\", \"expr_vs_activity\": \"<expr_vs_activity>\"}], \"locus_differential\": \"<locus_differential>\", \"formation_limited_by\": \"<formation_limited_by>\", \"locus_competent_in_naive\": \"<yes_no>\", \"ready_silent_fraction_relations\": [{\"pair\": \"<MODEL>_vs_<MODEL>\", \"relation\": \"<relation>\"}], \"activator_chromatin_footprint\": \"<activator_chromatin_footprint>\", \"exhaustion_comparable_across_split\": \"<yes_no>\", \"exhaustion_separable_program\": \"<yes_no>\", \"exhaustion_explains_split\": \"<yes_no>\", \"exhaustion_counterexample_model\": \"<model>\", \"exhaustion_relations\": [{\"pair\": \"<MODEL>_vs_<MODEL>\", \"relation\": \"<relation>\"}]}\n</EVAL_ANSWER>\nAll labels come from vocabulary.json (a data node). Give a relation list its full set of model pairs; gene lists hold gene symbols only.",
      "system": "CD8 RNA+ATAC regulatory inference",
      "assay": "10x Multiome (snRNA-seq + scATAC-seq)",
      "mainTaskTheme": "Evaluate whether regulatory claims are supported by both RNA expression and chromatin accessibility",
      "graderType": "all_of",
      "expectedAnswer": {
        "passRule": "all",
        "checks": [
          {
            "name": "list_match",
            "role": "gate",
            "field": "program_map",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "call": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "sig_includes_itgae",
            "role": "gate",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Itgae"
            ]
          },
          {
            "name": "declared_a_known_confounder",
            "role": "gate",
            "field": "$.excluded_confounders[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "cf_signature_gut_contaminant",
            "role": "hard_fail",
            "field": "$.core_residency_signature[*]",
            "kind": "avoid",
            "expected": [
              "Ccr9",
              "Cdh1",
              "Gpr34",
              "Car2",
              "Itga1"
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "generic_signature_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "cd103_program_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "<"
                    ]
                  }
                }
              },
              {
                "key": "KPC_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "per_model_regulatory_state",
            "kind": "list",
            "matchKey": "model",
            "k": 4,
            "expected": [
              {
                "key": "KPC_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "mWnt_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "present"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "active"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "none"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "released"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "poised_unfired"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "late_activation"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP",
                "fields": {
                  "program": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "brake_state": {
                    "op": "equals",
                    "values": [
                      "engaged"
                    ]
                  },
                  "activator_state": {
                    "op": "equals",
                    "values": [
                      "absent"
                    ]
                  },
                  "locus_state": {
                    "op": "equals",
                    "values": [
                      "closing_committed"
                    ]
                  },
                  "blocking_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "regulator_tuples",
            "kind": "list",
            "matchKey": "regulator",
            "k": 1,
            "expected": [
              {
                "key": "Klf2",
                "fields": {
                  "role": {
                    "op": "equals",
                    "values": [
                      "repressor"
                    ]
                  },
                  "gating_step": {
                    "op": "equals",
                    "values": [
                      "early_release"
                    ]
                  },
                  "evidence_basis": {
                    "op": "equals",
                    "values": [
                      "target_program_behavior"
                    ]
                  },
                  "expr_vs_activity": {
                    "op": "equals",
                    "values": [
                      "concordant"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "locus_differential_gate",
            "role": "gate",
            "field": "locus_differential",
            "kind": "value",
            "expected": "uniform_across_models"
          },
          {
            "name": "formation_limited_by_gate",
            "role": "gate",
            "field": "formation_limited_by",
            "kind": "value",
            "expected": "absent_induction"
          },
          {
            "name": "locus_competent_in_naive_gate",
            "role": "gate",
            "field": "locus_competent_in_naive",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "ready_silent_fraction_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 3,
            "expected": [
              {
                "key": "B16_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_B_ALL_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B16_GP_vs_KPC_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "activator_chromatin_footprint_gate",
            "role": "gate",
            "field": "activator_chromatin_footprint",
            "kind": "value",
            "expected": "inconsistent_across_formers"
          },
          {
            "name": "exhaustion_comparable_gate",
            "role": "gate",
            "field": "exhaustion_comparable_across_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_nondeterminant_gate",
            "role": "gate",
            "field": "exhaustion_explains_split",
            "kind": "value",
            "expected": "no"
          },
          {
            "name": "exhaustion_separable_program_gate",
            "role": "gate",
            "field": "exhaustion_separable_program",
            "kind": "value",
            "expected": "yes"
          },
          {
            "name": "exhaustion_counterexample_gate",
            "role": "gate",
            "field": "exhaustion_counterexample_model",
            "kind": "value",
            "expected": "KPC_GP"
          },
          {
            "name": "list_match",
            "role": "gate",
            "field": "exhaustion_relations",
            "kind": "list",
            "matchKey": "pair",
            "k": 2,
            "expected": [
              {
                "key": "KPC_GP_vs_B16_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      "~"
                    ]
                  }
                }
              },
              {
                "key": "B_ALL_GP_vs_mWnt_GP",
                "fields": {
                  "relation": {
                    "op": "equals",
                    "values": [
                      ">"
                    ]
                  }
                }
              }
            ]
          },
          {
            "name": "cf_inconsistent_present_without_activator",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "program",
            "entity": "present",
            "valueField": "activator_state",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_no_block_but_absent",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "none",
            "valueField": "program",
            "expected": [
              "absent"
            ]
          },
          {
            "name": "cf_inconsistent_late_block_without_poised_locus",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "late_activation",
            "valueField": "locus_state",
            "expected": [
              "closing_committed",
              "active"
            ]
          },
          {
            "name": "cf_inconsistent_early_block_with_brake_released",
            "role": "hard_fail",
            "field": "$.per_model_regulatory_state[*]",
            "kind": "contains",
            "matchKey": "blocking_step",
            "entity": "early_release",
            "valueField": "brake_state",
            "expected": [
              "released"
            ]
          }
        ]
      },
      "dataNodeCount": 3,
      "modules": {
        "graded_biological_result": "The CD103+ Trm-like program as a two-step de-repression->activation switch on a CONSTITUTIVELY-POISED locus, dissected by four GP33-matched models as an epistasis screen. The KLF2-target regulon is the de-repression gate (B_ALL>B16, seed-stable); an inducing signal is the activation gate. B16 = de-repressed but unactivated (released/poised/late_activation); B_ALL = brake-engaged (engaged/closing/early_release); mWnt = COUNTEREXAMPLE, forms despite an engaged brake (activator overrides); KPC = present/active. Chromatin accessibility does NOT discriminate formation (the locus is open even in naive) \u2014 gene EXPRESSION does. KLF2 mRNA is concordant with its regulon (not a hidden driver). Aspect 2: exhaustion is a partial correlate (non-formers more exhausted) that does NOT determine the split \u2014 KPC forms while as-exhausted as B16; exhaustion chromatin is a separable axis."
      },
      "rubricCheckpoints": [
        {
          "index": 1,
          "title": "De-contaminated residency signature",
          "decision": "derive a core-residency signature from the infection references (siIEL Trm vs spleen), keep Itgae and the genuine core-Trm genes, and explicitly exclude gut/epithelial/integrin confounders (Ccr9, Cdh1, Gpr34, Car2, Itga1) before scoring.",
          "scoring": "0 confounders left in / no de-contamination; 1 partial exclusion; 2 clean signature with confounders named and removed."
        },
        {
          "index": 2,
          "title": "Program map from the specific axis",
          "decision": "call the CD103+ program present in KPC/mWnt and absent in B16/B_ALL using CD103/Itgae (and the DP gate), not a generic activation or core-signature score.",
          "scoring": "0 wrong map / generic axis; 1 right map but axis ambiguous; 2 correct map on the CD103-specific axis."
        },
        {
          "index": 3,
          "title": "Signature\u2194CD103 inversion",
          "decision": "show the sign flip \u2014 B16 sits above B_ALL on the generic residency signature but below B_ALL on the CD103-specific program \u2014 by computing both axes rather than assuming CD103 tracks the generic score.",
          "scoring": "0 single axis / no flip; 1 both computed but the flip not stated; 2 both axes + the inversion explicit."
        },
        {
          "index": 4,
          "title": "Multi-omic assembly done right",
          "decision": "build a common accessibility feature space across the disjunct per-sample fragment files, reconcile barcodes to the RNA cells, and normalise for the large per-sample depth differences before any cross-model per-cell comparison.",
          "scoring": "0 raw / no common space; 1 assembled but depth not corrected; 2 common, barcode-reconciled, depth-normalised space used downstream."
        },
        {
          "index": 5,
          "title": "Regulator activity from targets \u2192 the brake",
          "decision": "infer regulator activity from the coordinated behaviour of its target program (a GRN/regulon read), not from the regulator's own transcript, and recover the KLF2-program brake as more engaged in B_ALL than B16 \u2014 the de-repression gate, acting as a repressor at the early step.",
          "scoring": "0 mRNA-only / no activity inference; 1 activity inferred but the B_ALL>B16 brake not recovered; 2 target-based activity + the brake separation."
        },
        {
          "index": 6,
          "title": "mWnt counterexample",
          "decision": "identify that mWnt forms the program with the brake still engaged (highest regulon activity among the models) \u2014 the inducing signal overrides the brake \u2014 so \"brake released \u21d4 program forms\" is false.",
          "scoring": "0 mWnt brake called released / counterexample missed; 1 mWnt's high brake noted but not reconciled with formation; 2 the override stated and integrated."
        },
        {
          "index": 7,
          "title": "Accessibility does not discriminate; expression does",
          "decision": "conclude the program's regulatory locus is broadly competent/poised across the models *including naive*, that no model-specific element discriminates formation, and that the differential is at gene induction, not chromatin opening.",
          "scoring": "0 discriminating-element or closed-locus story; 1 competence shown but induction-vs- accessibility not committed; 2 uniform competence + induction-level differential, derived from the data."
        },
        {
          "index": 8,
          "title": "Primed-but-silent fraction (depth-corrected) + concordance",
          "decision": "on a depth-normalised basis, rank B16 among the highest for the accessible-but-RNA-silent fraction (\u2248 mWnt, not lowest), and find the brake regulator's expression concordant with its inferred activity (not a hidden discordant driver).",
          "scoring": "0 raw-count B16-lowest / discordant claim; 1 one half right; 2 depth-corrected ranking + concordance both reached."
        },
        {
          "index": 9,
          "title": "Exhaustion: confound, not cause",
          "decision": "show exhaustion is not balanced across the forming/non-forming split (non-formers trend more exhausted) yet does not determine formation \u2014 KPC forms while as exhausted as B16 \u2014 and that the exhaustion axis is separable from the residency-control axis.",
          "scoring": "0 exhaustion as control or as cause; 1 imbalance or non-determinism shown but not both; 2 imbalance + non-determinism (the KPC counterexample) + separability."
        },
        {
          "index": 10,
          "title": "Two-step circuit integration",
          "decision": "integrate the pieces into one de-repression \u2192 context-matched-activation circuit on a constitutively-poised locus, and assign each non-forming model's block coherently: B16 de-repressed but unactivated (late-activation, poised locus); B_ALL brake-engaged (early-release, closing/committed locus).",
          "scoring": "0 single-step / flat story; 1 circuit stated but a block mis-assigned; 2 full two-step circuit with both blocks correctly placed."
        }
      ],
      "runs": [
        {
          "runId": "claude-opus-4-7__claude-code",
          "provider": "Anthropic",
          "model": "claude-opus-4-7",
          "harness": "claude-code",
          "run": "r1",
          "trialIndex": 1,
          "passed": false,
          "score": 9,
          "reasoning": "all_of [pass_rule=all]: FAIL\n  x list_match: list_match (answer_field='program_map'): FAIL  (score=0.0)\n  + sig_includes_itgae: passed=True, score=1.0\n  x declared_a_known_confounder: passed=False, score=0.0\n  x list_match: list_match (answer_field='generic_signature_relations'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='cd103_program_relations'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='per_model_regulatory_state'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='regulator_tuples'): FAIL  (score=0.0)\n  + locus_differential_gate: passed=True, score=1.0\n  + formation_limited_by_gate: passed=True, score=1.0\n  + locus_competent_in_naive_gate: passed=True, score=1.0\n  x list_match: list_match (answer_field='ready_silent_fraction_relations'): FAIL  (score=0.0)\n  + activator_chromatin_footprint_gate: passed=True, score=1.0\n  + exhaustion_comparable_gate: passed=True, score=1.0\n  + exhaustion_nondeterminant_gate: passed=True, score=1.0\n  + exhaustion_separable_program_gate: passed=True, score=1.0\n  + exhaustion_counterexample_gate: passed=True, score=1.0\n  + list_match: list_match (answer_field='exhaustion_relations'): PASS  (score=0.0)\n  x hard_fail 'cf_signature_gut_contaminant': TRIGGERED\n  + hard_fail 'cf_inconsistent_present_without_activator': not triggered\n  + hard_fail 'cf_inconsistent_no_block_but_absent': not triggered\n  + hard_fail 'cf_inconsistent_late_block_without_poised_locus': not triggered\n  + hard_fail 'cf_inconsistent_early_block_with_brake_released': not triggered",
          "metrics": {
            "pass_rule": "all",
            "scoring_count": 17,
            "scoring_passed": 10,
            "scoring_total_score": 9,
            "hard_fail_triggered": [
              "cf_signature_gut_contaminant"
            ]
          },
          "fieldScores": {
            "list_match": 0,
            "sig_includes_itgae": 1,
            "declared_a_known_confounder": 0,
            "locus_differential_gate": 1,
            "formation_limited_by_gate": 1,
            "locus_competent_in_naive_gate": 1,
            "activator_chromatin_footprint_gate": 1,
            "exhaustion_comparable_gate": 1,
            "exhaustion_nondeterminant_gate": 1,
            "exhaustion_separable_program_gate": 1,
            "exhaustion_counterexample_gate": 1
          },
          "graderChecks": [
            {
              "name": "program_map",
              "kind": "list_match",
              "passed": false,
              "score": 0
            },
            {
              "name": "sig_includes_itgae",
              "kind": "gate",
              "passed": true,
              "score": 1
            },
            {
              "name": "declared_a_known_confounder",
              "kind": "gate",
              "passed": false,
              "score": 0
            },
            {
              "name": "generic_signature_relations",
              "kind": "list_match",
              "passed": false,
              "score": 0
            },
            {
              "name": "cd103_program_relations",
              "kind": "list_match",
              "passed": false,
              "score": 0
            },
            {
              "name": "per_model_regulatory_state",
              "kind": "list_match",
              "passed": false,
              "score": 0
            },
            {
              "name": "regulator_tuples",
              "kind": "list_match",
              "passed": false,
              "score": 0
            },
            {
              "name": "locus_differential_gate",
              "kind": "gate",
              "passed": true,
              "score": 1
            },
            {
              "name": "formation_limited_by_gate",
              "kind": "gate",
              "passed": true,
              "score": 1
            },
            {
              "name": "locus_competent_in_naive_gate",
              "kind": "gate",
              "passed": true,
              "score": 1
            },
            {
              "name": "ready_silent_fraction_relations",
              "kind": "list_match",
              "passed": false,
              "score": 0
            },
            {
              "name": "activator_chromatin_footprint_gate",
              "kind": "gate",
              "passed": true,
              "score": 1
            },
            {
              "name": "exhaustion_comparable_gate",
              "kind": "gate",
              "passed": true,
              "score": 1
            },
            {
              "name": "exhaustion_nondeterminant_gate",
              "kind": "gate",
              "passed": true,
              "score": 1
            },
            {
              "name": "exhaustion_separable_program_gate",
              "kind": "gate",
              "passed": true,
              "score": 1
            },
            {
              "name": "exhaustion_counterexample_gate",
              "kind": "gate",
              "passed": true,
              "score": 1
            },
            {
              "name": "exhaustion_relations",
              "kind": "list_match",
              "passed": true,
              "score": 0
            },
            {
              "name": "cf_signature_gut_contaminant",
              "kind": "hard_fail",
              "passed": false,
              "triggered": true
            },
            {
              "name": "cf_inconsistent_present_without_activator",
              "kind": "hard_fail",
              "passed": true,
              "triggered": false
            },
            {
              "name": "cf_inconsistent_no_block_but_absent",
              "kind": "hard_fail",
              "passed": true,
              "triggered": false
            },
            {
              "name": "cf_inconsistent_late_block_without_poised_locus",
              "kind": "hard_fail",
              "passed": true,
              "triggered": false
            },
            {
              "name": "cf_inconsistent_early_block_with_brake_released",
              "kind": "hard_fail",
              "passed": true,
              "triggered": false
            }
          ],
          "agentAnswer": {
            "activator_chromatin_footprint": "inconsistent_across_formers",
            "cd103_program_relations": [
              {
                "pair": "KPC_GP_vs_B16_GP",
                "relation": ">"
              },
              {
                "pair": "KPC_GP_vs_B_ALL_GP",
                "relation": ">"
              },
              {
                "pair": "KPC_GP_vs_mWnt_GP",
                "relation": ">"
              },
              {
                "pair": "B16_GP_vs_B_ALL_GP",
                "relation": "~"
              },
              {
                "pair": "B16_GP_vs_mWnt_GP",
                "relation": "<"
              },
              {
                "pair": "B_ALL_GP_vs_mWnt_GP",
                "relation": "<"
              }
            ],
            "core_residency_signature": [
              "Itgae",
              "Itga1",
              "Cd69",
              "Cxcr6",
              "Rgs1",
              "Xcl1",
              "Cd160",
              "Runx3",
              "Bhlhe40",
              "Ahr",
              "Prdm1",
              "Litaf",
              "Inpp4b",
              "Nr4a2"
            ],
            "excluded_confounders": [
              "Klf2",
              "S1pr1",
              "Sell",
              "Ccr7",
              "S1pr5",
              "Klrg1",
              "Cx3cr1",
              "Ly6c2"
            ],
            "exhaustion_comparable_across_split": "no",
            "exhaustion_counterexample_model": "KPC_GP",
            "exhaustion_explains_split": "no",
            "exhaustion_relations": [
              {
                "pair": "KPC_GP_vs_B16_GP",
                "relation": "~"
              },
              {
                "pair": "KPC_GP_vs_B_ALL_GP",
                "relation": ">"
              },
              {
                "pair": "KPC_GP_vs_mWnt_GP",
                "relation": ">"
              },
              {
                "pair": "B16_GP_vs_B_ALL_GP",
                "relation": ">"
              },
              {
                "pair": "B16_GP_vs_mWnt_GP",
                "relation": ">"
              },
              {
                "pair": "B_ALL_GP_vs_mWnt_GP",
                "relation": ">"
              }
            ],
            "exhaustion_separable_program": "yes",
            "formation_limited_by": "absent_induction",
            "generic_signature_relations": [
              {
                "pair": "KPC_GP_vs_B16_GP",
                "relation": "~"
              },
              {
                "pair": "KPC_GP_vs_B_ALL_GP",
                "relation": "<"
              },
              {
                "pair": "KPC_GP_vs_mWnt_GP",
                "relation": ">"
              },
              {
                "pair": "B16_GP_vs_B_ALL_GP",
                "relation": "<"
              },
              {
                "pair": "B16_GP_vs_mWnt_GP",
                "relation": ">"
              },
              {
                "pair": "B_ALL_GP_vs_mWnt_GP",
                "relation": ">"
              }
            ],
            "locus_competent_in_naive": "yes",
            "locus_differential": "uniform_across_models",
            "per_model_regulatory_state": [
              {
                "activator_state": "present",
                "blocking_step": "none",
                "brake_state": "released",
                "locus_state": "active",
                "model": "KPC_GP",
                "program": "present"
              },
              {
                "activator_state": "partial",
                "blocking_step": "late_activation",
                "brake_state": "released",
                "locus_state": "poised_unfired",
                "model": "B16_GP",
                "program": "absent"
              },
              {
                "activator_state": "partial",
                "blocking_step": "both_steps",
                "brake_state": "partial",
                "locus_state": "closing_committed",
                "model": "B_ALL_GP",
                "program": "absent"
              },
              {
                "activator_state": "present",
                "blocking_step": "early_release",
                "brake_state": "partial",
                "locus_state": "active",
                "model": "mWnt_GP",
                "program": "partial"
              }
            ],
            "program_map": [
              {
                "call": "present",
                "model": "KPC_GP"
              },
              {
                "call": "absent",
                "model": "B16_GP"
              },
              {
                "call": "absent",
                "model": "B_ALL_GP"
              },
              {
                "call": "partial",
                "model": "mWnt_GP"
              }
            ],
            "ready_silent_fraction_relations": [
              {
                "pair": "KPC_GP_vs_B16_GP",
                "relation": "<"
              },
              {
                "pair": "KPC_GP_vs_B_ALL_GP",
                "relation": "<"
              },
              {
                "pair": "KPC_GP_vs_mWnt_GP",
                "relation": "<"
              },
              {
                "pair": "B16_GP_vs_B_ALL_GP",
                "relation": ">"
              },
              {
                "pair": "B16_GP_vs_mWnt_GP",
                "relation": ">"
              },
              {
                "pair": "B_ALL_GP_vs_mWnt_GP",
                "relation": "~"
              }
            ],
            "regulator_tuples": [
              {
                "evidence_basis": "own_abundance",
                "expr_vs_activity": "concordant",
                "gating_step": "early_release",
                "regulator": "Klf2",
                "role": "repressor"
              },
              {
                "evidence_basis": "own_abundance",
                "expr_vs_activity": "concordant",
                "gating_step": "early_release",
                "regulator": "S1pr1",
                "role": "repressor"
              },
              {
                "evidence_basis": "target_program_behavior",
                "expr_vs_activity": "discordant",
                "gating_step": "late_activation",
                "regulator": "Runx3",
                "role": "activator"
              },
              {
                "evidence_basis": "target_program_behavior",
                "expr_vs_activity": "discordant",
                "gating_step": "late_activation",
                "regulator": "Bhlhe40",
                "role": "activator"
              },
              {
                "evidence_basis": "own_abundance",
                "expr_vs_activity": "concordant",
                "gating_step": "late_activation",
                "regulator": "Ahr",
                "role": "activator"
              },
              {
                "evidence_basis": "target_program_behavior",
                "expr_vs_activity": "discordant",
                "gating_step": "late_activation",
                "regulator": "Prdm1",
                "role": "activator"
              },
              {
                "evidence_basis": "target_program_behavior",
                "expr_vs_activity": "indeterminate",
                "gating_step": "late_activation",
                "regulator": "Zfp683",
                "role": "activator"
              },
              {
                "evidence_basis": "target_program_behavior",
                "expr_vs_activity": "discordant",
                "gating_step": "indeterminate",
                "regulator": "Tox",
                "role": "dual"
              }
            ]
          },
          "durationS": 1409.37,
          "cost": 5.70238675,
          "nTurns": 74,
          "checkpointSignals": {
            "clustering": false,
            "markerAnnotation": true,
            "scoreBasedAnnotation": false,
            "normalization": false,
            "nerveJointAnchor": false,
            "multiomeAssembly": true,
            "depthNormalizedAtac": true,
            "regulonInference": true,
            "cd8SubsetIntegration": true,
            "perPatientEnrichment": false,
            "tcrRichnessMetrics": false,
            "orthologProjection": false,
            "epiSubsetSelection": false,
            "bidirectionalLrTest": false,
            "enrichmentFamilyRanking": false,
            "mousePseudobulkDesign": false,
            "cancerEpithelialDenominator": false,
            "geneLevelDe": false,
            "agingSignatureControl": false,
            "senescencePanelScoring": false
          },
          "expectedAnswer": {
            "passRule": "all",
            "checks": [
              {
                "name": "list_match",
                "role": "gate",
                "field": "program_map",
                "kind": "list",
                "matchKey": "model",
                "k": 4,
                "expected": [
                  {
                    "key": "KPC_GP",
                    "fields": {
                      "call": {
                        "op": "equals",
                        "values": [
                          "present"
                        ]
                      }
                    }
                  },
                  {
                    "key": "mWnt_GP",
                    "fields": {
                      "call": {
                        "op": "equals",
                        "values": [
                          "present"
                        ]
                      }
                    }
                  },
                  {
                    "key": "B16_GP",
                    "fields": {
                      "call": {
                        "op": "equals",
                        "values": [
                          "absent"
                        ]
                      }
                    }
                  },
                  {
                    "key": "B_ALL_GP",
                    "fields": {
                      "call": {
                        "op": "equals",
                        "values": [
                          "absent"
                        ]
                      }
                    }
                  }
                ]
              },
              {
                "name": "sig_includes_itgae",
                "role": "gate",
                "field": "$.core_residency_signature[*]",
                "kind": "avoid",
                "expected": [
                  "Itgae"
                ]
              },
              {
                "name": "declared_a_known_confounder",
                "role": "gate",
                "field": "$.excluded_confounders[*]",
                "kind": "avoid",
                "expected": [
                  "Ccr9",
                  "Cdh1",
                  "Gpr34",
                  "Car2",
                  "Itga1"
                ]
              },
              {
                "name": "cf_signature_gut_contaminant",
                "role": "hard_fail",
                "field": "$.core_residency_signature[*]",
                "kind": "avoid",
                "expected": [
                  "Ccr9",
                  "Cdh1",
                  "Gpr34",
                  "Car2",
                  "Itga1"
                ]
              },
              {
                "name": "list_match",
                "role": "gate",
                "field": "generic_signature_relations",
                "kind": "list",
                "matchKey": "pair",
                "k": 2,
                "expected": [
                  {
                    "key": "B16_GP_vs_B_ALL_GP",
                    "fields": {
                      "relation": {
                        "op": "equals",
                        "values": [
                          ">"
                        ]
                      }
                    }
                  },
                  {
                    "key": "KPC_GP_vs_B16_GP",
                    "fields": {
                      "relation": {
                        "op": "equals",
                        "values": [
                          ">"
                        ]
                      }
                    }
                  }
                ]
              },
              {
                "name": "list_match",
                "role": "gate",
                "field": "cd103_program_relations",
                "kind": "list",
                "matchKey": "pair",
                "k": 2,
                "expected": [
                  {
                    "key": "B16_GP_vs_B_ALL_GP",
                    "fields": {
                      "relation": {
                        "op": "equals",
                        "values": [
                          "<"
                        ]
                      }
                    }
                  },
                  {
                    "key": "KPC_GP_vs_mWnt_GP",
                    "fields": {
                      "relation": {
                        "op": "equals",
                        "values": [
                          ">"
                        ]
                      }
                    }
                  }
                ]
              },
              {
                "name": "list_match",
                "role": "gate",
                "field": "per_model_regulatory_state",
                "kind": "list",
                "matchKey": "model",
                "k": 4,
                "expected": [
                  {
                    "key": "KPC_GP",
                    "fields": {
                      "program": {
                        "op": "equals",
                        "values": [
                          "present"
                        ]
                      },
                      "brake_state": {
                        "op": "equals",
                        "values": [
                          "released"
                        ]
                      },
                      "activator_state": {
                        "op": "equals",
                        "values": [
                          "present"
                        ]
                      },
                      "locus_state": {
                        "op": "equals",
                        "values": [
                          "active"
                        ]
                      },
                      "blocking_step": {
                        "op": "equals",
                        "values": [
                          "none"
                        ]
                      }
                    }
                  },
                  {
                    "key": "mWnt_GP",
                    "fields": {
                      "program": {
                        "op": "equals",
                        "values": [
                          "present"
                        ]
                      },
                      "brake_state": {
                        "op": "equals",
                        "values": [
                          "engaged"
                        ]
                      },
                      "activator_state": {
                        "op": "equals",
                        "values": [
                          "present"
                        ]
                      },
                      "locus_state": {
                        "op": "equals",
                        "values": [
                          "active"
                        ]
                      },
                      "blocking_step": {
                        "op": "equals",
                        "values": [
                          "none"
                        ]
                      }
                    }
                  },
                  {
                    "key": "B16_GP",
                    "fields": {
                      "program": {
                        "op": "equals",
                        "values": [
                          "absent"
                        ]
                      },
                      "brake_state": {
                        "op": "equals",
                        "values": [
                          "released"
                        ]
                      },
                      "activator_state": {
                        "op": "equals",
                        "values": [
                          "absent"
                        ]
                      },
                      "locus_state": {
                        "op": "equals",
                        "values": [
                          "poised_unfired"
                        ]
                      },
                      "blocking_step": {
                        "op": "equals",
                        "values": [
                          "late_activation"
                        ]
                      }
                    }
                  },
                  {
                    "key": "B_ALL_GP",
                    "fields": {
                      "program": {
                        "op": "equals",
                        "values": [
                          "absent"
                        ]
                      },
                      "brake_state": {
                        "op": "equals",
                        "values": [
                          "engaged"
                        ]
                      },
                      "activator_state": {
                        "op": "equals",
                        "values": [
                          "absent"
                        ]
                      },
                      "locus_state": {
                        "op": "equals",
                        "values": [
                          "closing_committed"
                        ]
                      },
                      "blocking_step": {
                        "op": "equals",
                        "values": [
                          "early_release"
                        ]
                      }
                    }
                  }
                ]
              },
              {
                "name": "list_match",
                "role": "gate",
                "field": "regulator_tuples",
                "kind": "list",
                "matchKey": "regulator",
                "k": 1,
                "expected": [
                  {
                    "key": "Klf2",
                    "fields": {
                      "role": {
                        "op": "equals",
                        "values": [
                          "repressor"
                        ]
                      },
                      "gating_step": {
                        "op": "equals",
                        "values": [
                          "early_release"
                        ]
                      },
                      "evidence_basis": {
                        "op": "equals",
                        "values": [
                          "target_program_behavior"
                        ]
                      },
                      "expr_vs_activity": {
                        "op": "equals",
                        "values": [
                          "concordant"
                        ]
                      }
                    }
                  }
                ]
              },
              {
                "name": "locus_differential_gate",
                "role": "gate",
                "field": "locus_differential",
                "kind": "value",
                "expected": "uniform_across_models"
              },
              {
                "name": "formation_limited_by_gate",
                "role": "gate",
                "field": "formation_limited_by",
                "kind": "value",
                "expected": "absent_induction"
              },
              {
                "name": "locus_competent_in_naive_gate",
                "role": "gate",
                "field": "locus_competent_in_naive",
                "kind": "value",
                "expected": "yes"
              },
              {
                "name": "list_match",
                "role": "gate",
                "field": "ready_silent_fraction_relations",
                "kind": "list",
                "matchKey": "pair",
                "k": 3,
                "expected": [
                  {
                    "key": "B16_GP_vs_mWnt_GP",
                    "fields": {
                      "relation": {
                        "op": "equals",
                        "values": [
                          "~"
                        ]
                      }
                    }
                  },
                  {
                    "key": "B16_GP_vs_B_ALL_GP",
                    "fields": {
                      "relation": {
                        "op": "equals",
                        "values": [
                          "~"
                        ]
                      }
                    }
                  },
                  {
                    "key": "B16_GP_vs_KPC_GP",
                    "fields": {
                      "relation": {
                        "op": "equals",
                        "values": [
                          "~"
                        ]
                      }
                    }
                  }
                ]
              },
              {
                "name": "activator_chromatin_footprint_gate",
                "role": "gate",
                "field": "activator_chromatin_footprint",
                "kind": "value",
                "expected": "inconsistent_across_formers"
              },
              {
                "name": "exhaustion_comparable_gate",
                "role": "gate",
                "field": "exhaustion_comparable_across_split",
                "kind": "value",
                "expected": "no"
              },
              {
                "name": "exhaustion_nondeterminant_gate",
                "role": "gate",
                "field": "exhaustion_explains_split",
                "kind": "value",
                "expected": "no"
              },
              {
                "name": "exhaustion_separable_program_gate",
                "role": "gate",
                "field": "exhaustion_separable_program",
                "kind": "value",
                "expected": "yes"
              },
              {
                "name": "exhaustion_counterexample_gate",
                "role": "gate",
                "field": "exhaustion_counterexample_model",
                "kind": "value",
                "expected": "KPC_GP"
              },
              {
                "name": "list_match",
                "role": "gate",
                "field": "exhaustion_relations",
                "kind": "list",
                "matchKey": "pair",
                "k": 2,
                "expected": [
                  {
                    "key": "KPC_GP_vs_B16_GP",
                    "fields": {
                      "relation": {
                        "op": "equals",
                        "values": [
                          "~"
                        ]
                      }
                    }
                  },
                  {
                    "key": "B_ALL_GP_vs_mWnt_GP",
                    "fields": {
                      "relation": {
                        "op": "equals",
                        "values": [
                          ">"
                        ]
                      }
                    }
                  }
                ]
              },
              {
                "name": "cf_inconsistent_present_without_activator",
                "role": "hard_fail",
                "field": "$.per_model_regulatory_state[*]",
                "kind": "contains",
                "matchKey": "program",
                "entity": "present",
                "valueField": "activator_state",
                "expected": [
                  "absent"
                ]
              },
              {
                "name": "cf_inconsistent_no_block_but_absent",
                "role": "hard_fail",
                "field": "$.per_model_regulatory_state[*]",
                "kind": "contains",
                "matchKey": "blocking_step",
                "entity": "none",
                "valueField": "program",
                "expected": [
                  "absent"
                ]
              },
              {
                "name": "cf_inconsistent_late_block_without_poised_locus",
                "role": "hard_fail",
                "field": "$.per_model_regulatory_state[*]",
                "kind": "contains",
                "matchKey": "blocking_step",
                "entity": "late_activation",
                "valueField": "locus_state",
                "expected": [
                  "closing_committed",
                  "active"
                ]
              },
              {
                "name": "cf_inconsistent_early_block_with_brake_released",
                "role": "hard_fail",
                "field": "$.per_model_regulatory_state[*]",
                "kind": "contains",
                "matchKey": "blocking_step",
                "entity": "early_release",
                "valueField": "brake_state",
                "expected": [
                  "released"
                ]
              }
            ]
          },
          "trajectoryFile": "scbench-long/trajectories/green_2_tme_context_dependence__claude-opus-4-7__claude-code__r1.json"
        },
        {
          "runId": "gemini-3.5-flash__pi",
          "provider": "Google",
          "model": "gemini-3.5-flash",
          "harness": "pi",
          "run": "r1",
          "trialIndex": 1,
          "passed": false,
          "score": 7,
          "reasoning": "all_of [pass_rule=all]: FAIL\n  x list_match: list_match (answer_field='program_map'): FAIL  (score=0.0)\n  + sig_includes_itgae: passed=True, score=1.0\n  x declared_a_known_confounder: passed=False, score=0.0\n  x list_match: list_match (answer_field='generic_signature_relations'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='cd103_program_relations'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='per_model_regulatory_state'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='regulator_tuples'): FAIL  (score=0.0)\n  + locus_differential_gate: passed=True, score=1.0\n  + formation_limited_by_gate: passed=True, score=1.0\n  + locus_competent_in_naive_gate: passed=True, score=1.0\n  x list_match: list_match (answer_field='ready_silent_fraction_relations'): FAIL  (score=0.0)\n  x activator_chromatin_footprint_gate: passed=False, score=0.0\n  + exhaustion_comparable_gate: passed=True, score=1.0\n  + exhaustion_nondeterminant_gate: passed=True, score=1.0\n  + exhaustion_separable_program_gate: passed=True, score=1.0\n  x exhaustion_counterexample_gate: passed=False, score=0.0\n  x list_match: list_match (answer_field='exhaustion_relations'): FAIL  (score=0.0)\n  + hard_fail 'cf_signature_gut_contaminant': not triggered\n  + hard_fail 'cf_inconsistent_present_without_activator': not triggered\n  + hard_fail 'cf_inconsistent_no_block_but_absent': not triggered\n  + hard_fail 'cf_inconsistent_late_block_without_poised_locus': not triggered\n  + hard_fail 'cf_inconsistent_early_block_with_brake_released': not triggered",
          "metrics": {
            "pass_rule": "all",
            "scoring_count": 17,
            "scoring_passed": 7,
            "scoring_total_score": 7,
            "hard_fail_triggered": []
          },
          "fieldScores": {
            "list_match": 0,
            "sig_includes_itgae": 1,
            "declared_a_known_confounder": 0,
            "locus_differential_gate": 1,
            "formation_limited_by_gate": 1,
            "locus_competent_in_naive_gate": 1,
            "activator_chromatin_footprint_gate": 0,
            "exhaustion_comparable_gate": 1,
            "exhaustion_nondeterminant_gate": 1,
            "exhaustion_separable_program_gate": 1,
            "exhaustion_counterexample_gate": 0
          },
          "graderChecks": [
            {
              "name": "program_map",
              "kind": "list_match",
              "passed": false,
              "score": 0
            },
            {
              "name": "sig_includes_itgae",
              "kind": "gate",
              "passed": true,
              "score": 1
            },
            {
              "name": "declared_a_known_confounder",
              "kind": "gate",
              "passed": false,
              "score": 0
            },
            {
              "name": "generic_signature_relations",
              "kind": "list_match",
              "passed": false,
              "score": 0
            },
            {
              "name": "cd103_program_relations",
              "kind": "list_match",
              "passed": false,
              "score": 0
            },
            {
              "name": "per_model_regulatory_state",
              "kind": "list_match",
              "passed": false,
              "score": 0
            },
            {
              "name": "regulator_tuples",
              "kind": "list_match",
              "passed": false,
              "score": 0
            },
            {
              "name": "locus_differential_gate",
              "kind": "gate",
              "passed": true,
              "score": 1
            },
            {
              "name": "formation_limited_by_gate",
              "kind": "gate",
              "passed": true,
              "score": 1
            },
            {
              "name": "locus_competent_in_naive_gate",
              "kind": "gate",
              "passed": true,
              "score": 1
            },
            {
              "name": "ready_silent_fraction_relations",
              "kind": "list_match",
              "passed": false,
              "score": 0
            },
            {
              "name": "activator_chromatin_footprint_gate",
              "kind": "gate",
              "passed": false,
              "score": 0
            },
            {
              "name": "exhaustion_comparable_gate",
              "kind": "gate",
              "passed": true,
              "score": 1
            },
            {
              "name": "exhaustion_nondeterminant_gate",
              "kind": "gate",
              "passed": true,
              "score": 1
            },
            {
              "name": "exhaustion_separable_program_gate",
              "kind": "gate",
              "passed": true,
              "score": 1
            },
            {
              "name": "exhaustion_counterexample_gate",
              "kind": "gate",
              "passed": false,
              "score": 0
            },
            {
              "name": "exhaustion_relations",
              "kind": "list_match",
              "passed": false,
              "score": 0
            },
            {
              "name": "cf_signature_gut_contaminant",
              "kind": "hard_fail",
              "passed": true,
              "triggered": false
            },
            {
              "name": "cf_inconsistent_present_without_activator",
              "kind": "hard_fail",
              "passed": true,
              "triggered": false
            },
            {
              "name": "cf_inconsistent_no_block_but_absent",
              "kind": "hard_fail",
              "passed": true,
              "triggered": false
            },
            {
              "name": "cf_inconsistent_late_block_without_poised_locus",
              "kind": "hard_fail",
              "passed": true,
              "triggered": false
            },
            {
              "name": "cf_inconsistent_early_block_with_brake_released",
              "kind": "hard_fail",
              "passed": true,
              "triggered": false
            }
          ],
          "agentAnswer": {
            "activator_chromatin_footprint": "consistent_in_formers",
            "cd103_program_relations": [
              {
                "pair": "B16_GP_vs_B_ALL_GP",
                "relation": "~"
              },
              {
                "pair": "B16_GP_vs_KPC_GP",
                "relation": "<"
              },
              {
                "pair": "B16_GP_vs_mWnt_GP",
                "relation": "<"
              },
              {
                "pair": "B_ALL_GP_vs_KPC_GP",
                "relation": "<"
              },
              {
                "pair": "B_ALL_GP_vs_mWnt_GP",
                "relation": "<"
              },
              {
                "pair": "KPC_GP_vs_mWnt_GP",
                "relation": ">"
              }
            ],
            "core_residency_signature": [
              "Itgae"
            ],
            "excluded_confounders": [
              "Cd69"
            ],
            "exhaustion_comparable_across_split": "no",
            "exhaustion_counterexample_model": "mWnt_GP",
            "exhaustion_explains_split": "no",
            "exhaustion_relations": [
              {
                "pair": "B16_GP_vs_B_ALL_GP",
                "relation": ">"
              },
              {
                "pair": "B16_GP_vs_KPC_GP",
                "relation": ">"
              },
              {
                "pair": "B16_GP_vs_mWnt_GP",
                "relation": ">"
              },
              {
                "pair": "B_ALL_GP_vs_KPC_GP",
                "relation": ">"
              },
              {
                "pair": "B_ALL_GP_vs_mWnt_GP",
                "relation": ">"
              },
              {
                "pair": "KPC_GP_vs_mWnt_GP",
                "relation": ">"
              }
            ],
            "exhaustion_separable_program": "yes",
            "formation_limited_by": "absent_induction",
            "generic_signature_relations": [
              {
                "pair": "B16_GP_vs_B_ALL_GP",
                "relation": "<"
              },
              {
                "pair": "B16_GP_vs_KPC_GP",
                "relation": "<"
              },
              {
                "pair": "B16_GP_vs_mWnt_GP",
                "relation": "<"
              },
              {
                "pair": "B_ALL_GP_vs_KPC_GP",
                "relation": "<"
              },
              {
                "pair": "B_ALL_GP_vs_mWnt_GP",
                "relation": "<"
              },
              {
                "pair": "KPC_GP_vs_mWnt_GP",
                "relation": "~"
              }
            ],
            "locus_competent_in_naive": "yes",
            "locus_differential": "uniform_across_models",
            "per_model_regulatory_state": [
              {
                "activator_state": "absent",
                "blocking_step": "late_activation",
                "brake_state": "released",
                "locus_state": "poised_unfired",
                "model": "B16_GP",
                "program": "absent"
              },
              {
                "activator_state": "absent",
                "blocking_step": "both_steps",
                "brake_state": "engaged",
                "locus_state": "repressed",
                "model": "B_ALL_GP",
                "program": "absent"
              },
              {
                "activator_state": "present",
                "blocking_step": "none",
                "brake_state": "released",
                "locus_state": "active",
                "model": "KPC_GP",
                "program": "present"
              },
              {
                "activator_state": "present",
                "blocking_step": "early_release",
                "brake_state": "engaged",
                "locus_state": "active",
                "model": "mWnt_GP",
                "program": "partial"
              }
            ],
            "program_map": [
              {
                "call": "absent",
                "model": "B16_GP"
              },
              {
                "call": "absent",
                "model": "B_ALL_GP"
              },
              {
                "call": "present",
                "model": "KPC_GP"
              },
              {
                "call": "partial",
                "model": "mWnt_GP"
              }
            ],
            "ready_silent_fraction_relations": [
              {
                "pair": "B16_GP_vs_B_ALL_GP",
                "relation": ">"
              },
              {
                "pair": "B16_GP_vs_KPC_GP",
                "relation": ">"
              },
              {
                "pair": "B16_GP_vs_mWnt_GP",
                "relation": ">"
              },
              {
                "pair": "B_ALL_GP_vs_KPC_GP",
                "relation": ">"
              },
              {
                "pair": "B_ALL_GP_vs_mWnt_GP",
                "relation": "<"
              },
              {
                "pair": "KPC_GP_vs_mWnt_GP",
                "relation": "<"
              }
            ],
            "regulator_tuples": [
              {
                "evidence_basis": "own_abundance",
                "expr_vs_activity": "concordant",
                "gating_step": "early_release",
                "regulator": "Klf2",
                "role": "repressor"
              },
              {
                "evidence_basis": "target_program_behavior",
                "expr_vs_activity": "discordant",
                "gating_step": "late_activation",
                "regulator": "Batf",
                "role": "activator"
              }
            ]
          },
          "durationS": 1183.44,
          "cost": 1.5233244000000001,
          "nTurns": 60,
          "checkpointSignals": {
            "clustering": false,
            "markerAnnotation": true,
            "scoreBasedAnnotation": false,
            "normalization": true,
            "nerveJointAnchor": false,
            "multiomeAssembly": true,
            "depthNormalizedAtac": true,
            "regulonInference": true,
            "cd8SubsetIntegration": true,
            "perPatientEnrichment": false,
            "tcrRichnessMetrics": true,
            "orthologProjection": false,
            "epiSubsetSelection": false,
            "bidirectionalLrTest": true,
            "enrichmentFamilyRanking": true,
            "mousePseudobulkDesign": false,
            "cancerEpithelialDenominator": true,
            "geneLevelDe": false,
            "agingSignatureControl": false,
            "senescencePanelScoring": false
          },
          "expectedAnswer": {
            "passRule": "all",
            "checks": [
              {
                "name": "list_match",
                "role": "gate",
                "field": "program_map",
                "kind": "list",
                "matchKey": "model",
                "k": 4,
                "expected": [
                  {
                    "key": "KPC_GP",
                    "fields": {
                      "call": {
                        "op": "equals",
                        "values": [
                          "present"
                        ]
                      }
                    }
                  },
                  {
                    "key": "mWnt_GP",
                    "fields": {
                      "call": {
                        "op": "equals",
                        "values": [
                          "present"
                        ]
                      }
                    }
                  },
                  {
                    "key": "B16_GP",
                    "fields": {
                      "call": {
                        "op": "equals",
                        "values": [
                          "absent"
                        ]
                      }
                    }
                  },
                  {
                    "key": "B_ALL_GP",
                    "fields": {
                      "call": {
                        "op": "equals",
                        "values": [
                          "absent"
                        ]
                      }
                    }
                  }
                ]
              },
              {
                "name": "sig_includes_itgae",
                "role": "gate",
                "field": "$.core_residency_signature[*]",
                "kind": "avoid",
                "expected": [
                  "Itgae"
                ]
              },
              {
                "name": "declared_a_known_confounder",
                "role": "gate",
                "field": "$.excluded_confounders[*]",
                "kind": "avoid",
                "expected": [
                  "Ccr9",
                  "Cdh1",
                  "Gpr34",
                  "Car2",
                  "Itga1"
                ]
              },
              {
                "name": "cf_signature_gut_contaminant",
                "role": "hard_fail",
                "field": "$.core_residency_signature[*]",
                "kind": "avoid",
                "expected": [
                  "Ccr9",
                  "Cdh1",
                  "Gpr34",
                  "Car2",
                  "Itga1"
                ]
              },
              {
                "name": "list_match",
                "role": "gate",
                "field": "generic_signature_relations",
                "kind": "list",
                "matchKey": "pair",
                "k": 2,
                "expected": [
                  {
                    "key": "B16_GP_vs_B_ALL_GP",
                    "fields": {
                      "relation": {
                        "op": "equals",
                        "values": [
                          ">"
                        ]
                      }
                    }
                  },
                  {
                    "key": "KPC_GP_vs_B16_GP",
                    "fields": {
                      "relation": {
                        "op": "equals",
                        "values": [
                          ">"
                        ]
                      }
                    }
                  }
                ]
              },
              {
                "name": "list_match",
                "role": "gate",
                "field": "cd103_program_relations",
                "kind": "list",
                "matchKey": "pair",
                "k": 2,
                "expected": [
                  {
                    "key": "B16_GP_vs_B_ALL_GP",
                    "fields": {
                      "relation": {
                        "op": "equals",
                        "values": [
                          "<"
                        ]
                      }
                    }
                  },
                  {
                    "key": "KPC_GP_vs_mWnt_GP",
                    "fields": {
                      "relation": {
                        "op": "equals",
                        "values": [
                          ">"
                        ]
                      }
                    }
                  }
                ]
              },
              {
                "name": "list_match",
                "role": "gate",
                "field": "per_model_regulatory_state",
                "kind": "list",
                "matchKey": "model",
                "k": 4,
                "expected": [
                  {
                    "key": "KPC_GP",
                    "fields": {
                      "program": {
                        "op": "equals",
                        "values": [
                          "present"
                        ]
                      },
                      "brake_state": {
                        "op": "equals",
                        "values": [
                          "released"
                        ]
                      },
                      "activator_state": {
                        "op": "equals",
                        "values": [
                          "present"
                        ]
                      },
                      "locus_state": {
                        "op": "equals",
                        "values": [
                          "active"
                        ]
                      },
                      "blocking_step": {
                        "op": "equals",
                        "values": [
                          "none"
                        ]
                      }
                    }
                  },
                  {
                    "key": "mWnt_GP",
                    "fields": {
                      "program": {
                        "op": "equals",
                        "values": [
                          "present"
                        ]
                      },
                      "brake_state": {
                        "op": "equals",
                        "values": [
                          "engaged"
                        ]
                      },
                      "activator_state": {
                        "op": "equals",
                        "values": [
                          "present"
                        ]
                      },
                      "locus_state": {
                        "op": "equals",
                        "values": [
                          "active"
                        ]
                      },
                      "blocking_step": {
                        "op": "equals",
                        "values": [
                          "none"
                        ]
                      }
                    }
                  },
                  {
                    "key": "B16_GP",
                    "fields": {
                      "program": {
                        "op": "equals",
                        "values": [
                          "absent"
                        ]
                      },
                      "brake_state": {
                        "op": "equals",
                        "values": [
                          "released"
                        ]
                      },
                      "activator_state": {
                        "op": "equals",
                        "values": [
                          "absent"
                        ]
                      },
                      "locus_state": {
                        "op": "equals",
                        "values": [
                          "poised_unfired"
                        ]
                      },
                      "blocking_step": {
                        "op": "equals",
                        "values": [
                          "late_activation"
                        ]
                      }
                    }
                  },
                  {
                    "key": "B_ALL_GP",
                    "fields": {
                      "program": {
                        "op": "equals",
                        "values": [
                          "absent"
                        ]
                      },
                      "brake_state": {
                        "op": "equals",
                        "values": [
                          "engaged"
                        ]
                      },
                      "activator_state": {
                        "op": "equals",
                        "values": [
                          "absent"
                        ]
                      },
                      "locus_state": {
                        "op": "equals",
                        "values": [
                          "closing_committed"
                        ]
                      },
                      "blocking_step": {
                        "op": "equals",
                        "values": [
                          "early_release"
                        ]
                      }
                    }
                  }
                ]
              },
              {
                "name": "list_match",
                "role": "gate",
                "field": "regulator_tuples",
                "kind": "list",
                "matchKey": "regulator",
                "k": 1,
                "expected": [
                  {
                    "key": "Klf2",
                    "fields": {
                      "role": {
                        "op": "equals",
                        "values": [
                          "repressor"
                        ]
                      },
                      "gating_step": {
                        "op": "equals",
                        "values": [
                          "early_release"
                        ]
                      },
                      "evidence_basis": {
                        "op": "equals",
                        "values": [
                          "target_program_behavior"
                        ]
                      },
                      "expr_vs_activity": {
                        "op": "equals",
                        "values": [
                          "concordant"
                        ]
                      }
                    }
                  }
                ]
              },
              {
                "name": "locus_differential_gate",
                "role": "gate",
                "field": "locus_differential",
                "kind": "value",
                "expected": "uniform_across_models"
              },
              {
                "name": "formation_limited_by_gate",
                "role": "gate",
                "field": "formation_limited_by",
                "kind": "value",
                "expected": "absent_induction"
              },
              {
                "name": "locus_competent_in_naive_gate",
                "role": "gate",
                "field": "locus_competent_in_naive",
                "kind": "value",
                "expected": "yes"
              },
              {
                "name": "list_match",
                "role": "gate",
                "field": "ready_silent_fraction_relations",
                "kind": "list",
                "matchKey": "pair",
                "k": 3,
                "expected": [
                  {
                    "key": "B16_GP_vs_mWnt_GP",
                    "fields": {
                      "relation": {
                        "op": "equals",
                        "values": [
                          "~"
                        ]
                      }
                    }
                  },
                  {
                    "key": "B16_GP_vs_B_ALL_GP",
                    "fields": {
                      "relation": {
                        "op": "equals",
                        "values": [
                          "~"
                        ]
                      }
                    }
                  },
                  {
                    "key": "B16_GP_vs_KPC_GP",
                    "fields": {
                      "relation": {
                        "op": "equals",
                        "values": [
                          "~"
                        ]
                      }
                    }
                  }
                ]
              },
              {
                "name": "activator_chromatin_footprint_gate",
                "role": "gate",
                "field": "activator_chromatin_footprint",
                "kind": "value",
                "expected": "inconsistent_across_formers"
              },
              {
                "name": "exhaustion_comparable_gate",
                "role": "gate",
                "field": "exhaustion_comparable_across_split",
                "kind": "value",
                "expected": "no"
              },
              {
                "name": "exhaustion_nondeterminant_gate",
                "role": "gate",
                "field": "exhaustion_explains_split",
                "kind": "value",
                "expected": "no"
              },
              {
                "name": "exhaustion_separable_program_gate",
                "role": "gate",
                "field": "exhaustion_separable_program",
                "kind": "value",
                "expected": "yes"
              },
              {
                "name": "exhaustion_counterexample_gate",
                "role": "gate",
                "field": "exhaustion_counterexample_model",
                "kind": "value",
                "expected": "KPC_GP"
              },
              {
                "name": "list_match",
                "role": "gate",
                "field": "exhaustion_relations",
                "kind": "list",
                "matchKey": "pair",
                "k": 2,
                "expected": [
                  {
                    "key": "KPC_GP_vs_B16_GP",
                    "fields": {
                      "relation": {
                        "op": "equals",
                        "values": [
                          "~"
                        ]
                      }
                    }
                  },
                  {
                    "key": "B_ALL_GP_vs_mWnt_GP",
                    "fields": {
                      "relation": {
                        "op": "equals",
                        "values": [
                          ">"
                        ]
                      }
                    }
                  }
                ]
              },
              {
                "name": "cf_inconsistent_present_without_activator",
                "role": "hard_fail",
                "field": "$.per_model_regulatory_state[*]",
                "kind": "contains",
                "matchKey": "program",
                "entity": "present",
                "valueField": "activator_state",
                "expected": [
                  "absent"
                ]
              },
              {
                "name": "cf_inconsistent_no_block_but_absent",
                "role": "hard_fail",
                "field": "$.per_model_regulatory_state[*]",
                "kind": "contains",
                "matchKey": "blocking_step",
                "entity": "none",
                "valueField": "program",
                "expected": [
                  "absent"
                ]
              },
              {
                "name": "cf_inconsistent_late_block_without_poised_locus",
                "role": "hard_fail",
                "field": "$.per_model_regulatory_state[*]",
                "kind": "contains",
                "matchKey": "blocking_step",
                "entity": "late_activation",
                "valueField": "locus_state",
                "expected": [
                  "closing_committed",
                  "active"
                ]
              },
              {
                "name": "cf_inconsistent_early_block_with_brake_released",
                "role": "hard_fail",
                "field": "$.per_model_regulatory_state[*]",
                "kind": "contains",
                "matchKey": "blocking_step",
                "entity": "early_release",
                "valueField": "brake_state",
                "expected": [
                  "released"
                ]
              }
            ]
          },
          "trajectoryFile": "scbench-long/trajectories/green_2_tme_context_dependence__gemini-3.5-flash__pi__r1.json"
        },
        {
          "runId": "gpt-5.5__openai-codex",
          "provider": "OpenAI",
          "model": "gpt-5.5",
          "harness": "openai-codex",
          "run": "r1",
          "trialIndex": 1,
          "passed": false,
          "score": 8,
          "reasoning": "all_of [pass_rule=all]: FAIL\n  x list_match: list_match (answer_field='program_map'): FAIL  (score=0.0)\n  + sig_includes_itgae: passed=True, score=1.0\n  x declared_a_known_confounder: passed=False, score=0.0\n  x list_match: list_match (answer_field='generic_signature_relations'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='cd103_program_relations'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='per_model_regulatory_state'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='regulator_tuples'): FAIL  (score=0.0)\n  + locus_differential_gate: passed=True, score=1.0\n  + formation_limited_by_gate: passed=True, score=1.0\n  + locus_competent_in_naive_gate: passed=True, score=1.0\n  x list_match: list_match (answer_field='ready_silent_fraction_relations'): FAIL  (score=0.0)\n  x activator_chromatin_footprint_gate: passed=False, score=0.0\n  + exhaustion_comparable_gate: passed=True, score=1.0\n  + exhaustion_nondeterminant_gate: passed=True, score=1.0\n  + exhaustion_separable_program_gate: passed=True, score=1.0\n  + exhaustion_counterexample_gate: passed=True, score=1.0\n  x list_match: list_match (answer_field='exhaustion_relations'): FAIL  (score=0.0)\n  x hard_fail 'cf_signature_gut_contaminant': TRIGGERED\n  + hard_fail 'cf_inconsistent_present_without_activator': not triggered\n  + hard_fail 'cf_inconsistent_no_block_but_absent': not triggered\n  + hard_fail 'cf_inconsistent_late_block_without_poised_locus': not triggered\n  + hard_fail 'cf_inconsistent_early_block_with_brake_released': not triggered",
          "metrics": {
            "pass_rule": "all",
            "scoring_count": 17,
            "scoring_passed": 8,
            "scoring_total_score": 8,
            "hard_fail_triggered": [
              "cf_signature_gut_contaminant"
            ]
          },
          "fieldScores": {
            "list_match": 0,
            "sig_includes_itgae": 1,
            "declared_a_known_confounder": 0,
            "locus_differential_gate": 1,
            "formation_limited_by_gate": 1,
            "locus_competent_in_naive_gate": 1,
            "activator_chromatin_footprint_gate": 0,
            "exhaustion_comparable_gate": 1,
            "exhaustion_nondeterminant_gate": 1,
            "exhaustion_separable_program_gate": 1,
            "exhaustion_counterexample_gate": 1
          },
          "graderChecks": [
            {
              "name": "program_map",
              "kind": "list_match",
              "passed": false,
              "score": 0
            },
            {
              "name": "sig_includes_itgae",
              "kind": "gate",
              "passed": true,
              "score": 1
            },
            {
              "name": "declared_a_known_confounder",
              "kind": "gate",
              "passed": false,
              "score": 0
            },
            {
              "name": "generic_signature_relations",
              "kind": "list_match",
              "passed": false,
              "score": 0
            },
            {
              "name": "cd103_program_relations",
              "kind": "list_match",
              "passed": false,
              "score": 0
            },
            {
              "name": "per_model_regulatory_state",
              "kind": "list_match",
              "passed": false,
              "score": 0
            },
            {
              "name": "regulator_tuples",
              "kind": "list_match",
              "passed": false,
              "score": 0
            },
            {
              "name": "locus_differential_gate",
              "kind": "gate",
              "passed": true,
              "score": 1
            },
            {
              "name": "formation_limited_by_gate",
              "kind": "gate",
              "passed": true,
              "score": 1
            },
            {
              "name": "locus_competent_in_naive_gate",
              "kind": "gate",
              "passed": true,
              "score": 1
            },
            {
              "name": "ready_silent_fraction_relations",
              "kind": "list_match",
              "passed": false,
              "score": 0
            },
            {
              "name": "activator_chromatin_footprint_gate",
              "kind": "gate",
              "passed": false,
              "score": 0
            },
            {
              "name": "exhaustion_comparable_gate",
              "kind": "gate",
              "passed": true,
              "score": 1
            },
            {
              "name": "exhaustion_nondeterminant_gate",
              "kind": "gate",
              "passed": true,
              "score": 1
            },
            {
              "name": "exhaustion_separable_program_gate",
              "kind": "gate",
              "passed": true,
              "score": 1
            },
            {
              "name": "exhaustion_counterexample_gate",
              "kind": "gate",
              "passed": true,
              "score": 1
            },
            {
              "name": "exhaustion_relations",
              "kind": "list_match",
              "passed": false,
              "score": 0
            },
            {
              "name": "cf_signature_gut_contaminant",
              "kind": "hard_fail",
              "passed": false,
              "triggered": true
            },
            {
              "name": "cf_inconsistent_present_without_activator",
              "kind": "hard_fail",
              "passed": true,
              "triggered": false
            },
            {
              "name": "cf_inconsistent_no_block_but_absent",
              "kind": "hard_fail",
              "passed": true,
              "triggered": false
            },
            {
              "name": "cf_inconsistent_late_block_without_poised_locus",
              "kind": "hard_fail",
              "passed": true,
              "triggered": false
            },
            {
              "name": "cf_inconsistent_early_block_with_brake_released",
              "kind": "hard_fail",
              "passed": true,
              "triggered": false
            }
          ],
          "agentAnswer": {
            "activator_chromatin_footprint": "consistent_in_formers",
            "cd103_program_relations": [
              {
                "pair": "B16_GP_vs_B_ALL_GP",
                "relation": "~"
              },
              {
                "pair": "B16_GP_vs_KPC_GP",
                "relation": "<"
              },
              {
                "pair": "B16_GP_vs_mWnt_GP",
                "relation": "<"
              },
              {
                "pair": "B_ALL_GP_vs_KPC_GP",
                "relation": "<"
              },
              {
                "pair": "B_ALL_GP_vs_mWnt_GP",
                "relation": "<"
              },
              {
                "pair": "KPC_GP_vs_mWnt_GP",
                "relation": ">"
              }
            ],
            "core_residency_signature": [
              "Itgae",
              "Itga1",
              "Cd69",
              "Cxcr6"
            ],
            "excluded_confounders": [
              "Tox",
              "Pdcd1",
              "Lag3",
              "Havcr2",
              "Entpd1",
              "Tigit"
            ],
            "exhaustion_comparable_across_split": "no",
            "exhaustion_counterexample_model": "KPC_GP",
            "exhaustion_explains_split": "no",
            "exhaustion_relations": [
              {
                "pair": "B16_GP_vs_B_ALL_GP",
                "relation": "<"
              },
              {
                "pair": "B16_GP_vs_KPC_GP",
                "relation": "~"
              },
              {
                "pair": "B16_GP_vs_mWnt_GP",
                "relation": ">"
              },
              {
                "pair": "B_ALL_GP_vs_KPC_GP",
                "relation": ">"
              },
              {
                "pair": "B_ALL_GP_vs_mWnt_GP",
                "relation": ">"
              },
              {
                "pair": "KPC_GP_vs_mWnt_GP",
                "relation": ">"
              }
            ],
            "exhaustion_separable_program": "yes",
            "formation_limited_by": "absent_induction",
            "generic_signature_relations": [
              {
                "pair": "B16_GP_vs_B_ALL_GP",
                "relation": ">"
              },
              {
                "pair": "B16_GP_vs_KPC_GP",
                "relation": "<"
              },
              {
                "pair": "B16_GP_vs_mWnt_GP",
                "relation": "<"
              },
              {
                "pair": "B_ALL_GP_vs_KPC_GP",
                "relation": "<"
              },
              {
                "pair": "B_ALL_GP_vs_mWnt_GP",
                "relation": "<"
              },
              {
                "pair": "KPC_GP_vs_mWnt_GP",
                "relation": "~"
              }
            ],
            "locus_competent_in_naive": "yes",
            "locus_differential": "uniform_across_models",
            "per_model_regulatory_state": [
              {
                "activator_state": "partial",
                "blocking_step": "late_activation",
                "brake_state": "released",
                "locus_state": "poised_unfired",
                "model": "B16_GP",
                "program": "absent"
              },
              {
                "activator_state": "partial",
                "blocking_step": "both_steps",
                "brake_state": "partial",
                "locus_state": "poised_unfired",
                "model": "B_ALL_GP",
                "program": "absent"
              },
              {
                "activator_state": "present",
                "blocking_step": "none",
                "brake_state": "released",
                "locus_state": "active",
                "model": "KPC_GP",
                "program": "present"
              },
              {
                "activator_state": "partial",
                "blocking_step": "both_steps",
                "brake_state": "engaged",
                "locus_state": "active",
                "model": "mWnt_GP",
                "program": "partial"
              }
            ],
            "program_map": [
              {
                "call": "absent",
                "model": "B16_GP"
              },
              {
                "call": "absent",
                "model": "B_ALL_GP"
              },
              {
                "call": "present",
                "model": "KPC_GP"
              },
              {
                "call": "partial",
                "model": "mWnt_GP"
              }
            ],
            "ready_silent_fraction_relations": [
              {
                "pair": "B16_GP_vs_B_ALL_GP",
                "relation": "<"
              },
              {
                "pair": "B16_GP_vs_KPC_GP",
                "relation": ">"
              },
              {
                "pair": "B16_GP_vs_mWnt_GP",
                "relation": "<"
              },
              {
                "pair": "B_ALL_GP_vs_KPC_GP",
                "relation": ">"
              },
              {
                "pair": "B_ALL_GP_vs_mWnt_GP",
                "relation": ">"
              },
              {
                "pair": "KPC_GP_vs_mWnt_GP",
                "relation": "<"
              }
            ],
            "regulator_tuples": [
              {
                "evidence_basis": "own_abundance",
                "expr_vs_activity": "concordant",
                "gating_step": "early_release",
                "regulator": "Klf2",
                "role": "repressor"
              },
              {
                "evidence_basis": "own_abundance",
                "expr_vs_activity": "concordant",
                "gating_step": "early_release",
                "regulator": "S1pr1",
                "role": "repressor"
              },
              {
                "evidence_basis": "own_abundance",
                "expr_vs_activity": "concordant",
                "gating_step": "early_release",
                "regulator": "Tcf7",
                "role": "repressor"
              },
              {
                "evidence_basis": "target_program_behavior",
                "expr_vs_activity": "discordant",
                "gating_step": "late_activation",
                "regulator": "Runx3",
                "role": "activator"
              },
              {
                "evidence_basis": "target_program_behavior",
                "expr_vs_activity": "discordant",
                "gating_step": "late_activation",
                "regulator": "Bhlhe40",
                "role": "activator"
              },
              {
                "evidence_basis": "target_program_behavior",
                "expr_vs_activity": "discordant",
                "gating_step": "late_activation",
                "regulator": "Nr4a3",
                "role": "activator"
              },
              {
                "evidence_basis": "target_program_behavior",
                "expr_vs_activity": "discordant",
                "gating_step": "none",
                "regulator": "Tox",
                "role": "none"
              }
            ]
          },
          "durationS": 1667.47,
          "cost": 4.0376080000000005,
          "nTurns": 1,
          "checkpointSignals": {
            "clustering": false,
            "markerAnnotation": true,
            "scoreBasedAnnotation": false,
            "normalization": true,
            "nerveJointAnchor": false,
            "multiomeAssembly": true,
            "depthNormalizedAtac": true,
            "regulonInference": true,
            "cd8SubsetIntegration": true,
            "perPatientEnrichment": false,
            "tcrRichnessMetrics": false,
            "orthologProjection": false,
            "epiSubsetSelection": false,
            "bidirectionalLrTest": false,
            "enrichmentFamilyRanking": false,
            "mousePseudobulkDesign": false,
            "cancerEpithelialDenominator": false,
            "geneLevelDe": false,
            "agingSignatureControl": false,
            "senescencePanelScoring": false
          },
          "expectedAnswer": {
            "passRule": "all",
            "checks": [
              {
                "name": "list_match",
                "role": "gate",
                "field": "program_map",
                "kind": "list",
                "matchKey": "model",
                "k": 4,
                "expected": [
                  {
                    "key": "KPC_GP",
                    "fields": {
                      "call": {
                        "op": "equals",
                        "values": [
                          "present"
                        ]
                      }
                    }
                  },
                  {
                    "key": "mWnt_GP",
                    "fields": {
                      "call": {
                        "op": "equals",
                        "values": [
                          "present"
                        ]
                      }
                    }
                  },
                  {
                    "key": "B16_GP",
                    "fields": {
                      "call": {
                        "op": "equals",
                        "values": [
                          "absent"
                        ]
                      }
                    }
                  },
                  {
                    "key": "B_ALL_GP",
                    "fields": {
                      "call": {
                        "op": "equals",
                        "values": [
                          "absent"
                        ]
                      }
                    }
                  }
                ]
              },
              {
                "name": "sig_includes_itgae",
                "role": "gate",
                "field": "$.core_residency_signature[*]",
                "kind": "avoid",
                "expected": [
                  "Itgae"
                ]
              },
              {
                "name": "declared_a_known_confounder",
                "role": "gate",
                "field": "$.excluded_confounders[*]",
                "kind": "avoid",
                "expected": [
                  "Ccr9",
                  "Cdh1",
                  "Gpr34",
                  "Car2",
                  "Itga1"
                ]
              },
              {
                "name": "cf_signature_gut_contaminant",
                "role": "hard_fail",
                "field": "$.core_residency_signature[*]",
                "kind": "avoid",
                "expected": [
                  "Ccr9",
                  "Cdh1",
                  "Gpr34",
                  "Car2",
                  "Itga1"
                ]
              },
              {
                "name": "list_match",
                "role": "gate",
                "field": "generic_signature_relations",
                "kind": "list",
                "matchKey": "pair",
                "k": 2,
                "expected": [
                  {
                    "key": "B16_GP_vs_B_ALL_GP",
                    "fields": {
                      "relation": {
                        "op": "equals",
                        "values": [
                          ">"
                        ]
                      }
                    }
                  },
                  {
                    "key": "KPC_GP_vs_B16_GP",
                    "fields": {
                      "relation": {
                        "op": "equals",
                        "values": [
                          ">"
                        ]
                      }
                    }
                  }
                ]
              },
              {
                "name": "list_match",
                "role": "gate",
                "field": "cd103_program_relations",
                "kind": "list",
                "matchKey": "pair",
                "k": 2,
                "expected": [
                  {
                    "key": "B16_GP_vs_B_ALL_GP",
                    "fields": {
                      "relation": {
                        "op": "equals",
                        "values": [
                          "<"
                        ]
                      }
                    }
                  },
                  {
                    "key": "KPC_GP_vs_mWnt_GP",
                    "fields": {
                      "relation": {
                        "op": "equals",
                        "values": [
                          ">"
                        ]
                      }
                    }
                  }
                ]
              },
              {
                "name": "list_match",
                "role": "gate",
                "field": "per_model_regulatory_state",
                "kind": "list",
                "matchKey": "model",
                "k": 4,
                "expected": [
                  {
                    "key": "KPC_GP",
                    "fields": {
                      "program": {
                        "op": "equals",
                        "values": [
                          "present"
                        ]
                      },
                      "brake_state": {
                        "op": "equals",
                        "values": [
                          "released"
                        ]
                      },
                      "activator_state": {
                        "op": "equals",
                        "values": [
                          "present"
                        ]
                      },
                      "locus_state": {
                        "op": "equals",
                        "values": [
                          "active"
                        ]
                      },
                      "blocking_step": {
                        "op": "equals",
                        "values": [
                          "none"
                        ]
                      }
                    }
                  },
                  {
                    "key": "mWnt_GP",
                    "fields": {
                      "program": {
                        "op": "equals",
                        "values": [
                          "present"
                        ]
                      },
                      "brake_state": {
                        "op": "equals",
                        "values": [
                          "engaged"
                        ]
                      },
                      "activator_state": {
                        "op": "equals",
                        "values": [
                          "present"
                        ]
                      },
                      "locus_state": {
                        "op": "equals",
                        "values": [
                          "active"
                        ]
                      },
                      "blocking_step": {
                        "op": "equals",
                        "values": [
                          "none"
                        ]
                      }
                    }
                  },
                  {
                    "key": "B16_GP",
                    "fields": {
                      "program": {
                        "op": "equals",
                        "values": [
                          "absent"
                        ]
                      },
                      "brake_state": {
                        "op": "equals",
                        "values": [
                          "released"
                        ]
                      },
                      "activator_state": {
                        "op": "equals",
                        "values": [
                          "absent"
                        ]
                      },
                      "locus_state": {
                        "op": "equals",
                        "values": [
                          "poised_unfired"
                        ]
                      },
                      "blocking_step": {
                        "op": "equals",
                        "values": [
                          "late_activation"
                        ]
                      }
                    }
                  },
                  {
                    "key": "B_ALL_GP",
                    "fields": {
                      "program": {
                        "op": "equals",
                        "values": [
                          "absent"
                        ]
                      },
                      "brake_state": {
                        "op": "equals",
                        "values": [
                          "engaged"
                        ]
                      },
                      "activator_state": {
                        "op": "equals",
                        "values": [
                          "absent"
                        ]
                      },
                      "locus_state": {
                        "op": "equals",
                        "values": [
                          "closing_committed"
                        ]
                      },
                      "blocking_step": {
                        "op": "equals",
                        "values": [
                          "early_release"
                        ]
                      }
                    }
                  }
                ]
              },
              {
                "name": "list_match",
                "role": "gate",
                "field": "regulator_tuples",
                "kind": "list",
                "matchKey": "regulator",
                "k": 1,
                "expected": [
                  {
                    "key": "Klf2",
                    "fields": {
                      "role": {
                        "op": "equals",
                        "values": [
                          "repressor"
                        ]
                      },
                      "gating_step": {
                        "op": "equals",
                        "values": [
                          "early_release"
                        ]
                      },
                      "evidence_basis": {
                        "op": "equals",
                        "values": [
                          "target_program_behavior"
                        ]
                      },
                      "expr_vs_activity": {
                        "op": "equals",
                        "values": [
                          "concordant"
                        ]
                      }
                    }
                  }
                ]
              },
              {
                "name": "locus_differential_gate",
                "role": "gate",
                "field": "locus_differential",
                "kind": "value",
                "expected": "uniform_across_models"
              },
              {
                "name": "formation_limited_by_gate",
                "role": "gate",
                "field": "formation_limited_by",
                "kind": "value",
                "expected": "absent_induction"
              },
              {
                "name": "locus_competent_in_naive_gate",
                "role": "gate",
                "field": "locus_competent_in_naive",
                "kind": "value",
                "expected": "yes"
              },
              {
                "name": "list_match",
                "role": "gate",
                "field": "ready_silent_fraction_relations",
                "kind": "list",
                "matchKey": "pair",
                "k": 3,
                "expected": [
                  {
                    "key": "B16_GP_vs_mWnt_GP",
                    "fields": {
                      "relation": {
                        "op": "equals",
                        "values": [
                          "~"
                        ]
                      }
                    }
                  },
                  {
                    "key": "B16_GP_vs_B_ALL_GP",
                    "fields": {
                      "relation": {
                        "op": "equals",
                        "values": [
                          "~"
                        ]
                      }
                    }
                  },
                  {
                    "key": "B16_GP_vs_KPC_GP",
                    "fields": {
                      "relation": {
                        "op": "equals",
                        "values": [
                          "~"
                        ]
                      }
                    }
                  }
                ]
              },
              {
                "name": "activator_chromatin_footprint_gate",
                "role": "gate",
                "field": "activator_chromatin_footprint",
                "kind": "value",
                "expected": "inconsistent_across_formers"
              },
              {
                "name": "exhaustion_comparable_gate",
                "role": "gate",
                "field": "exhaustion_comparable_across_split",
                "kind": "value",
                "expected": "no"
              },
              {
                "name": "exhaustion_nondeterminant_gate",
                "role": "gate",
                "field": "exhaustion_explains_split",
                "kind": "value",
                "expected": "no"
              },
              {
                "name": "exhaustion_separable_program_gate",
                "role": "gate",
                "field": "exhaustion_separable_program",
                "kind": "value",
                "expected": "yes"
              },
              {
                "name": "exhaustion_counterexample_gate",
                "role": "gate",
                "field": "exhaustion_counterexample_model",
                "kind": "value",
                "expected": "KPC_GP"
              },
              {
                "name": "list_match",
                "role": "gate",
                "field": "exhaustion_relations",
                "kind": "list",
                "matchKey": "pair",
                "k": 2,
                "expected": [
                  {
                    "key": "KPC_GP_vs_B16_GP",
                    "fields": {
                      "relation": {
                        "op": "equals",
                        "values": [
                          "~"
                        ]
                      }
                    }
                  },
                  {
                    "key": "B_ALL_GP_vs_mWnt_GP",
                    "fields": {
                      "relation": {
                        "op": "equals",
                        "values": [
                          ">"
                        ]
                      }
                    }
                  }
                ]
              },
              {
                "name": "cf_inconsistent_present_without_activator",
                "role": "hard_fail",
                "field": "$.per_model_regulatory_state[*]",
                "kind": "contains",
                "matchKey": "program",
                "entity": "present",
                "valueField": "activator_state",
                "expected": [
                  "absent"
                ]
              },
              {
                "name": "cf_inconsistent_no_block_but_absent",
                "role": "hard_fail",
                "field": "$.per_model_regulatory_state[*]",
                "kind": "contains",
                "matchKey": "blocking_step",
                "entity": "none",
                "valueField": "program",
                "expected": [
                  "absent"
                ]
              },
              {
                "name": "cf_inconsistent_late_block_without_poised_locus",
                "role": "hard_fail",
                "field": "$.per_model_regulatory_state[*]",
                "kind": "contains",
                "matchKey": "blocking_step",
                "entity": "late_activation",
                "valueField": "locus_state",
                "expected": [
                  "closing_committed",
                  "active"
                ]
              },
              {
                "name": "cf_inconsistent_early_block_with_brake_released",
                "role": "hard_fail",
                "field": "$.per_model_regulatory_state[*]",
                "kind": "contains",
                "matchKey": "blocking_step",
                "entity": "early_release",
                "valueField": "brake_state",
                "expected": [
                  "released"
                ]
              }
            ]
          },
          "trajectoryFile": "scbench-long/trajectories/green_2_tme_context_dependence__gpt-5.5__openai-codex__r1.json"
        },
        {
          "runId": "grok-4.3__pi",
          "provider": "SpaceXAI",
          "model": "grok-4.3",
          "harness": "pi",
          "run": "r1",
          "trialIndex": 1,
          "passed": false,
          "score": 5,
          "reasoning": "all_of [pass_rule=all]: FAIL\n  x list_match: list_match (answer_field='program_map'): FAIL  (score=0.0)\n  + sig_includes_itgae: passed=True, score=1.0\n  x declared_a_known_confounder: passed=False, score=0.0\n  x list_match: list_match (answer_field='generic_signature_relations'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='cd103_program_relations'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='per_model_regulatory_state'): FAIL  (score=0.0)\n  x list_match: list_match (answer_field='regulator_tuples'): FAIL  (score=0.0)\n  x locus_differential_gate: passed=False, score=0.0\n  + formation_limited_by_gate: passed=True, score=1.0\n  + locus_competent_in_naive_gate: passed=True, score=1.0\n  x list_match: list_match (answer_field='ready_silent_fraction_relations'): FAIL  (score=0.0)\n  x activator_chromatin_footprint_gate: passed=False, score=0.0\n  x exhaustion_comparable_gate: passed=False, score=0.0\n  + exhaustion_nondeterminant_gate: passed=True, score=1.0\n  + exhaustion_separable_program_gate: passed=True, score=1.0\n  x exhaustion_counterexample_gate: passed=False, score=0.0\n  x list_match: list_match (answer_field='exhaustion_relations'): FAIL  (score=0.0)\n  + hard_fail 'cf_signature_gut_contaminant': not triggered\n  + hard_fail 'cf_inconsistent_present_without_activator': not triggered\n  + hard_fail 'cf_inconsistent_no_block_but_absent': not triggered\n  + hard_fail 'cf_inconsistent_late_block_without_poised_locus': not triggered\n  + hard_fail 'cf_inconsistent_early_block_with_brake_released': not triggered",
          "metrics": {
            "pass_rule": "all",
            "scoring_count": 17,
            "scoring_passed": 5,
            "scoring_total_score": 5,
            "hard_fail_triggered": []
          },
          "fieldScores": {
            "list_match": 0,
            "sig_includes_itgae": 1,
            "declared_a_known_confounder": 0,
            "locus_differential_gate": 0,
            "formation_limited_by_gate": 1,
            "locus_competent_in_naive_gate": 1,
            "activator_chromatin_footprint_gate": 0,
            "exhaustion_comparable_gate": 0,
            "exhaustion_nondeterminant_gate": 1,
            "exhaustion_separable_program_gate": 1,
            "exhaustion_counterexample_gate": 0
          },
          "graderChecks": [
            {
              "name": "program_map",
              "kind": "list_match",
              "passed": false,
              "score": 0
            },
            {
              "name": "sig_includes_itgae",
              "kind": "gate",
              "passed": true,
              "score": 1
            },
            {
              "name": "declared_a_known_confounder",
              "kind": "gate",
              "passed": false,
              "score": 0
            },
            {
              "name": "generic_signature_relations",
              "kind": "list_match",
              "passed": false,
              "score": 0
            },
            {
              "name": "cd103_program_relations",
              "kind": "list_match",
              "passed": false,
              "score": 0
            },
            {
              "name": "per_model_regulatory_state",
              "kind": "list_match",
              "passed": false,
              "score": 0
            },
            {
              "name": "regulator_tuples",
              "kind": "list_match",
              "passed": false,
              "score": 0
            },
            {
              "name": "locus_differential_gate",
              "kind": "gate",
              "passed": false,
              "score": 0
            },
            {
              "name": "formation_limited_by_gate",
              "kind": "gate",
              "passed": true,
              "score": 1
            },
            {
              "name": "locus_competent_in_naive_gate",
              "kind": "gate",
              "passed": true,
              "score": 1
            },
            {
              "name": "ready_silent_fraction_relations",
              "kind": "list_match",
              "passed": false,
              "score": 0
            },
            {
              "name": "activator_chromatin_footprint_gate",
              "kind": "gate",
              "passed": false,
              "score": 0
            },
            {
              "name": "exhaustion_comparable_gate",
              "kind": "gate",
              "passed": false,
              "score": 0
            },
            {
              "name": "exhaustion_nondeterminant_gate",
              "kind": "gate",
              "passed": true,
              "score": 1
            },
            {
              "name": "exhaustion_separable_program_gate",
              "kind": "gate",
              "passed": true,
              "score": 1
            },
            {
              "name": "exhaustion_counterexample_gate",
              "kind": "gate",
              "passed": false,
              "score": 0
            },
            {
              "name": "exhaustion_relations",
              "kind": "list_match",
              "passed": false,
              "score": 0
            },
            {
              "name": "cf_signature_gut_contaminant",
              "kind": "hard_fail",
              "passed": true,
              "triggered": false
            },
            {
              "name": "cf_inconsistent_present_without_activator",
              "kind": "hard_fail",
              "passed": true,
              "triggered": false
            },
            {
              "name": "cf_inconsistent_no_block_but_absent",
              "kind": "hard_fail",
              "passed": true,
              "triggered": false
            },
            {
              "name": "cf_inconsistent_late_block_without_poised_locus",
              "kind": "hard_fail",
              "passed": true,
              "triggered": false
            },
            {
              "name": "cf_inconsistent_early_block_with_brake_released",
              "kind": "hard_fail",
              "passed": true,
              "triggered": false
            }
          ],
          "agentAnswer": {
            "activator_chromatin_footprint": "consistent_in_formers",
            "cd103_program_relations": [
              {
                "pair": "B16_GP_vs_KPC_GP",
                "relation": "~"
              },
              {
                "pair": "B16_GP_vs_B_ALL_GP",
                "relation": ">"
              },
              {
                "pair": "B16_GP_vs_mWnt_GP",
                "relation": ">"
              },
              {
                "pair": "KPC_GP_vs_B_ALL_GP",
                "relation": ">"
              },
              {
                "pair": "KPC_GP_vs_mWnt_GP",
                "relation": ">"
              },
              {
                "pair": "B_ALL_GP_vs_mWnt_GP",
                "relation": "~"
              }
            ],
            "core_residency_signature": [
              "Itgae",
              "Cd69",
              "Runx3",
              "Batf",
              "Prdm1"
            ],
            "excluded_confounders": [
              "Pdcd1",
              "Tcf7",
              "Tox"
            ],
            "exhaustion_comparable_across_split": "yes",
            "exhaustion_counterexample_model": "B16_GP",
            "exhaustion_explains_split": "no",
            "exhaustion_relations": [
              {
                "pair": "B16_GP_vs_KPC_GP",
                "relation": "~"
              },
              {
                "pair": "B16_GP_vs_B_ALL_GP",
                "relation": "~"
              },
              {
                "pair": "B16_GP_vs_mWnt_GP",
                "relation": "~"
              },
              {
                "pair": "KPC_GP_vs_B_ALL_GP",
                "relation": "~"
              },
              {
                "pair": "KPC_GP_vs_mWnt_GP",
                "relation": "~"
              },
              {
                "pair": "B_ALL_GP_vs_mWnt_GP",
                "relation": "~"
              }
            ],
            "exhaustion_separable_program": "yes",
            "formation_limited_by": "absent_induction",
            "generic_signature_relations": [
              {
                "pair": "B16_GP_vs_KPC_GP",
                "relation": "~"
              },
              {
                "pair": "B16_GP_vs_B_ALL_GP",
                "relation": ">"
              },
              {
                "pair": "B16_GP_vs_mWnt_GP",
                "relation": ">"
              },
              {
                "pair": "KPC_GP_vs_B_ALL_GP",
                "relation": ">"
              },
              {
                "pair": "KPC_GP_vs_mWnt_GP",
                "relation": ">"
              },
              {
                "pair": "B_ALL_GP_vs_mWnt_GP",
                "relation": "~"
              }
            ],
            "locus_competent_in_naive": "yes",
            "locus_differential": "model_specific_element",
            "per_model_regulatory_state": [
              {
                "activator_state": "present",
                "blocking_step": "none",
                "brake_state": "released",
                "locus_state": "active",
                "model": "B16_GP",
                "program": "present"
              },
              {
                "activator_state": "present",
                "blocking_step": "none",
                "brake_state": "released",
                "locus_state": "active",
                "model": "KPC_GP",
                "program": "present"
              },
              {
                "activator_state": "partial",
                "blocking_step": "late_activation",
                "brake_state": "partial",
                "locus_state": "poised_unfired",
                "model": "B_ALL_GP",
                "program": "partial"
              },
              {
                "activator_state": "absent",
                "blocking_step": "both_steps",
                "brake_state": "engaged",
                "locus_state": "repressed",
                "model": "mWnt_GP",
                "program": "absent"
              }
            ],
            "program_map": [
              {
                "call": "present",
                "model": "B16_GP"
              },
              {
                "call": "present",
                "model": "KPC_GP"
              },
              {
                "call": "partial",
                "model": "B_ALL_GP"
              },
              {
                "call": "absent",
                "model": "mWnt_GP"
              }
            ],
            "ready_silent_fraction_relations": [
              {
                "pair": "B16_GP_vs_KPC_GP",
                "relation": "~"
              },
              {
                "pair": "B16_GP_vs_B_ALL_GP",
                "relation": ">"
              },
              {
                "pair": "B16_GP_vs_mWnt_GP",
                "relation": ">"
              },
              {
                "pair": "KPC_GP_vs_B_ALL_GP",
                "relation": ">"
              },
              {
                "pair": "KPC_GP_vs_mWnt_GP",
                "relation": ">"
              },
              {
                "pair": "B_ALL_GP_vs_mWnt_GP",
                "relation": "~"
              }
            ],
            "regulator_tuples": [
              {
                "evidence_basis": "target_program_behavior",
                "expr_vs_activity": "concordant",
                "gating_step": "late_activation",
                "regulator": "Runx3",
                "role": "activator"
              },
              {
                "evidence_basis": "target_program_behavior",
                "expr_vs_activity": "concordant",
                "gating_step": "late_activation",
                "regulator": "Batf",
                "role": "activator"
              },
              {
                "evidence_basis": "own_abundance",
                "expr_vs_activity": "discordant",
                "gating_step": "early_release",
                "regulator": "Prdm1",
                "role": "repressor"
              }
            ]
          },
          "durationS": 48.09,
          "cost": 0.01039195,
          "nTurns": 6,
          "checkpointSignals": {
            "clustering": false,
            "markerAnnotation": false,
            "scoreBasedAnnotation": false,
            "normalization": false,
            "nerveJointAnchor": false,
            "multiomeAssembly": true,
            "depthNormalizedAtac": false,
            "regulonInference": true,
            "cd8SubsetIntegration": true,
            "perPatientEnrichment": false,
            "tcrRichnessMetrics": false,
            "orthologProjection": false,
            "epiSubsetSelection": false,
            "bidirectionalLrTest": false,
            "enrichmentFamilyRanking": false,
            "mousePseudobulkDesign": false,
            "cancerEpithelialDenominator": false,
            "geneLevelDe": false,
            "agingSignatureControl": false,
            "senescencePanelScoring": false
          },
          "expectedAnswer": {
            "passRule": "all",
            "checks": [
              {
                "name": "list_match",
                "role": "gate",
                "field": "program_map",
                "kind": "list",
                "matchKey": "model",
                "k": 4,
                "expected": [
                  {
                    "key": "KPC_GP",
                    "fields": {
                      "call": {
                        "op": "equals",
                        "values": [
                          "present"
                        ]
                      }
                    }
                  },
                  {
                    "key": "mWnt_GP",
                    "fields": {
                      "call": {
                        "op": "equals",
                        "values": [
                          "present"
                        ]
                      }
                    }
                  },
                  {
                    "key": "B16_GP",
                    "fields": {
                      "call": {
                        "op": "equals",
                        "values": [
                          "absent"
                        ]
                      }
                    }
                  },
                  {
                    "key": "B_ALL_GP",
                    "fields": {
                      "call": {
                        "op": "equals",
                        "values": [
                          "absent"
                        ]
                      }
                    }
                  }
                ]
              },
              {
                "name": "sig_includes_itgae",
                "role": "gate",
                "field": "$.core_residency_signature[*]",
                "kind": "avoid",
                "expected": [
                  "Itgae"
                ]
              },
              {
                "name": "declared_a_known_confounder",
                "role": "gate",
                "field": "$.excluded_confounders[*]",
                "kind": "avoid",
                "expected": [
                  "Ccr9",
                  "Cdh1",
                  "Gpr34",
                  "Car2",
                  "Itga1"
                ]
              },
              {
                "name": "cf_signature_gut_contaminant",
                "role": "hard_fail",
                "field": "$.core_residency_signature[*]",
                "kind": "avoid",
                "expected": [
                  "Ccr9",
                  "Cdh1",
                  "Gpr34",
                  "Car2",
                  "Itga1"
                ]
              },
              {
                "name": "list_match",
                "role": "gate",
                "field": "generic_signature_relations",
                "kind": "list",
                "matchKey": "pair",
                "k": 2,
                "expected": [
                  {
                    "key": "B16_GP_vs_B_ALL_GP",
                    "fields": {
                      "relation": {
                        "op": "equals",
                        "values": [
                          ">"
                        ]
                      }
                    }
                  },
                  {
                    "key": "KPC_GP_vs_B16_GP",
                    "fields": {
                      "relation": {
                        "op": "equals",
                        "values": [
                          ">"
                        ]
                      }
                    }
                  }
                ]
              },
              {
                "name": "list_match",
                "role": "gate",
                "field": "cd103_program_relations",
                "kind": "list",
                "matchKey": "pair",
                "k": 2,
                "expected": [
                  {
                    "key": "B16_GP_vs_B_ALL_GP",
                    "fields": {
                      "relation": {
                        "op": "equals",
                        "values": [
                          "<"
                        ]
                      }
                    }
                  },
                  {
                    "key": "KPC_GP_vs_mWnt_GP",
                    "fields": {
                      "relation": {
                        "op": "equals",
                        "values": [
                          ">"
                        ]
                      }
                    }
                  }
                ]
              },
              {
                "name": "list_match",
                "role": "gate",
                "field": "per_model_regulatory_state",
                "kind": "list",
                "matchKey": "model",
                "k": 4,
                "expected": [
                  {
                    "key": "KPC_GP",
                    "fields": {
                      "program": {
                        "op": "equals",
                        "values": [
                          "present"
                        ]
                      },
                      "brake_state": {
                        "op": "equals",
                        "values": [
                          "released"
                        ]
                      },
                      "activator_state": {
                        "op": "equals",
                        "values": [
                          "present"
                        ]
                      },
                      "locus_state": {
                        "op": "equals",
                        "values": [
                          "active"
                        ]
                      },
                      "blocking_step": {
                        "op": "equals",
                        "values": [
                          "none"
                        ]
                      }
                    }
                  },
                  {
                    "key": "mWnt_GP",
                    "fields": {
                      "program": {
                        "op": "equals",
                        "values": [
                          "present"
                        ]
                      },
                      "brake_state": {
                        "op": "equals",
                        "values": [
                          "engaged"
                        ]
                      },
                      "activator_state": {
                        "op": "equals",
                        "values": [
                          "present"
                        ]
                      },
                      "locus_state": {
                        "op": "equals",
                        "values": [
                          "active"
                        ]
                      },
                      "blocking_step": {
                        "op": "equals",
                        "values": [
                          "none"
                        ]
                      }
                    }
                  },
                  {
                    "key": "B16_GP",
                    "fields": {
                      "program": {
                        "op": "equals",
                        "values": [
                          "absent"
                        ]
                      },
                      "brake_state": {
                        "op": "equals",
                        "values": [
                          "released"
                        ]
                      },
                      "activator_state": {
                        "op": "equals",
                        "values": [
                          "absent"
                        ]
                      },
                      "locus_state": {
                        "op": "equals",
                        "values": [
                          "poised_unfired"
                        ]
                      },
                      "blocking_step": {
                        "op": "equals",
                        "values": [
                          "late_activation"
                        ]
                      }
                    }
                  },
                  {
                    "key": "B_ALL_GP",
                    "fields": {
                      "program": {
                        "op": "equals",
                        "values": [
                          "absent"
                        ]
                      },
                      "brake_state": {
                        "op": "equals",
                        "values": [
                          "engaged"
                        ]
                      },
                      "activator_state": {
                        "op": "equals",
                        "values": [
                          "absent"
                        ]
                      },
                      "locus_state": {
                        "op": "equals",
                        "values": [
                          "closing_committed"
                        ]
                      },
                      "blocking_step": {
                        "op": "equals",
                        "values": [
                          "early_release"
                        ]
                      }
                    }
                  }
                ]
              },
              {
                "name": "list_match",
                "role": "gate",
                "field": "regulator_tuples",
                "kind": "list",
                "matchKey": "regulator",
                "k": 1,
                "expected": [
                  {
                    "key": "Klf2",
                    "fields": {
                      "role": {
                        "op": "equals",
                        "values": [
                          "repressor"
                        ]
                      },
                      "gating_step": {
                        "op": "equals",
                        "values": [
                          "early_release"
                        ]
                      },
                      "evidence_basis": {
                        "op": "equals",
                        "values": [
                          "target_program_behavior"
                        ]
                      },
                      "expr_vs_activity": {
                        "op": "equals",
                        "values": [
                          "concordant"
                        ]
                      }
                    }
                  }
                ]
              },
              {
                "name": "locus_differential_gate",
                "role": "gate",
                "field": "locus_differential",
                "kind": "value",
                "expected": "uniform_across_models"
              },
              {
                "name": "formation_limited_by_gate",
                "role": "gate",
                "field": "formation_limited_by",
                "kind": "value",
                "expected": "absent_induction"
              },
              {
                "name": "locus_competent_in_naive_gate",
                "role": "gate",
                "field": "locus_competent_in_naive",
                "kind": "value",
                "expected": "yes"
              },
              {
                "name": "list_match",
                "role": "gate",
                "field": "ready_silent_fraction_relations",
                "kind": "list",
                "matchKey": "pair",
                "k": 3,
                "expected": [
                  {
                    "key": "B16_GP_vs_mWnt_GP",
                    "fields": {
                      "relation": {
                        "op": "equals",
                        "values": [
                          "~"
                        ]
                      }
                    }
                  },
                  {
                    "key": "B16_GP_vs_B_ALL_GP",
                    "fields": {
                      "relation": {
                        "op": "equals",
                        "values": [
                          "~"
                        ]
                      }
                    }
                  },
                  {
                    "key": "B16_GP_vs_KPC_GP",
                    "fields": {
                      "relation": {
                        "op": "equals",
                        "values": [
                          "~"
                        ]
                      }
                    }
                  }
                ]
              },
              {
                "name": "activator_chromatin_footprint_gate",
                "role": "gate",
                "field": "activator_chromatin_footprint",
                "kind": "value",
                "expected": "inconsistent_across_formers"
              },
              {
                "name": "exhaustion_comparable_gate",
                "role": "gate",
                "field": "exhaustion_comparable_across_split",
                "kind": "value",
                "expected": "no"
              },
              {
                "name": "exhaustion_nondeterminant_gate",
                "role": "gate",
                "field": "exhaustion_explains_split",
                "kind": "value",
                "expected": "no"
              },
              {
                "name": "exhaustion_separable_program_gate",
                "role": "gate",
                "field": "exhaustion_separable_program",
                "kind": "value",
                "expected": "yes"
              },
              {
                "name": "exhaustion_counterexample_gate",
                "role": "gate",
                "field": "exhaustion_counterexample_model",
                "kind": "value",
                "expected": "KPC_GP"
              },
              {
                "name": "list_match",
                "role": "gate",
                "field": "exhaustion_relations",
                "kind": "list",
                "matchKey": "pair",
                "k": 2,
                "expected": [
                  {
                    "key": "KPC_GP_vs_B16_GP",
                    "fields": {
                      "relation": {
                        "op": "equals",
                        "values": [
                          "~"
                        ]
                      }
                    }
                  },
                  {
                    "key": "B_ALL_GP_vs_mWnt_GP",
                    "fields": {
                      "relation": {
                        "op": "equals",
                        "values": [
                          ">"
                        ]
                      }
                    }
                  }
                ]
              },
              {
                "name": "cf_inconsistent_present_without_activator",
                "role": "hard_fail",
                "field": "$.per_model_regulatory_state[*]",
                "kind": "contains",
                "matchKey": "program",
                "entity": "present",
                "valueField": "activator_state",
                "expected": [
                  "absent"
                ]
              },
              {
                "name": "cf_inconsistent_no_block_but_absent",
                "role": "hard_fail",
                "field": "$.per_model_regulatory_state[*]",
                "kind": "contains",
                "matchKey": "blocking_step",
                "entity": "none",
                "valueField": "program",
                "expected": [
                  "absent"
                ]
              },
              {
                "name": "cf_inconsistent_late_block_without_poised_locus",
                "role": "hard_fail",
                "field": "$.per_model_regulatory_state[*]",
                "kind": "contains",
                "matchKey": "blocking_step",
                "entity": "late_activation",
                "valueField": "locus_state",
                "expected": [
                  "closing_committed",
                  "active"
                ]
              },
              {
                "name": "cf_inconsistent_early_block_with_brake_released",
                "role": "hard_fail",
                "field": "$.per_model_regulatory_state[*]",
                "kind": "contains",
                "matchKey": "blocking_step",
                "entity": "early_release",
                "valueField": "brake_state",
                "expected": [
                  "released"
                ]
              }
            ]
          },
          "trajectoryFile": "scbench-long/trajectories/green_2_tme_context_dependence__grok-4.3__pi__r1.json"
        }
      ],
      "sourcePath": "green_2_tme_context_dependence/eval.json",
      "githubUrl": "https://github.com/latchbio/scbench-long/blob/main/evals/green_2_tme_context_dependence/eval.json"
    },
    {
      "id": "gse155381_q7_xspecies_epi_communication",
      "title": "Gse155381 Q7 Xspecies Epi Communication",
      "description": "Human-monkey chimeric embryos \u00b7 Smart-seq2 scRNA-seq: `chimera_human` EPI cells engage in cross-species cell-cell signaling with `chimera_monkey` EPI cells?",
      "objective": "`chimera_human` EPI cells engage in cross-species cell-cell signaling with `chimera_monkey` EPI cells?",
      "prompt": "Smart-seq2 scRNA-seq of primate embryos at multiple developmental stages. Three cohorts are encoded in `obs.embryo_kind`: `chimera_human` (human cells, hEPSC-derived, recovered from chimeric monkey embryos), `chimera_monkey` (monkey HOST cells from the same chimeric embryos; NOT hEPSC-derived), `control_monkey` (monkey cells from non-injected control embryos).\n\nDo `chimera_human` EPI cells engage in cross-species cell-cell signaling with `chimera_monkey` EPI cells? If so, characterise the dominant signaling axes you observe AND the directionality of the signal (which cells are the predominant senders / receivers).\n\nUse vocabulary.json (provided as a data node) for all controlled labels and per-field answer semantics.\n\nFor `dominant_axis_support`, list the HGNC gene symbols (ligands and receptors) of the specific significant pairs in your own ranking that drive your `dominant_signaling_axis` choice.\n\nReturn EXACTLY:\n<EVAL_ANSWER>\n{\"xspecies_signaling_richness\": \"<richness_label>\", \"dominant_signaling_axis\": \"<axis_label>\", \"dominant_axis_support\": [\"<HGNC_symbol>\", \"...\"], \"signaling_directionality\": \"<directionality_label>\", \"top_signal_interaction_class\": \"<interaction_class_label>\"}\n</EVAL_ANSWER>",
      "system": "Human-monkey chimeric embryos",
      "assay": "Smart-seq2 scRNA-seq",
      "mainTaskTheme": "Compare developmental states across species; evaluate cell-cell signaling",
      "graderType": "all_of",
      "expectedAnswer": {
        "passRule": "score_threshold",
        "checks": [
          {
            "name": "richness_weighted",
            "role": "additive",
            "field": "$.xspecies_signaling_richness",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_weighted",
            "role": "additive",
            "field": "$.dominant_signaling_axis",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "dominant_axis_wnt_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "WNT_FZD_LRP_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_fgf_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "FGF_FGFR_family"
                ]
              }
            }
          },
          {
            "name": "dominant_axis_notch_unsupported_by_own_pairs",
            "role": "hard_fail",
            "field": null,
            "kind": "fields",
            "expected": {
              "dominant_signaling_axis": {
                "op": "equals",
                "values": [
                  "NOTCH_DLL_JAG_family"
                ]
              }
            }
          },
          {
            "name": "signaling_directionality_weighted",
            "role": "additive",
            "field": "$.signaling_directionality",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          },
          {
            "name": "top_signal_interaction_class_weighted",
            "role": "additive",
            "field": "$.top_signal_interaction_class",
            "kind": "other",
            "op": "weighted_label",
            "expected": null
          }
        ]
      },
      "dataNodeCount": 2,
      "modules": {
        "graded_biological_result": "cross-species signaling richness (rich, multi-family), the DOMINANT-AXIS call (load-bearing, 2.5 pt; canonical: multi-family co-dominance / no-single-family-dominates after recognising that raw per-pair-count ranking is abundance/DB-size biased \u2014 collagen-integrin ECM and homophilic adhesion flood the top by transcript abundance but are the LOWEST enrichment tier, so reporting ECM as THE dominant axis is the abundance artifact and scores 0.5; genuinely-enriched families NOTCH/RTK/ephrin/semaphorin/FGF/WNT earn 1.25-1.5), bidirectionality, and the interaction-class of the top signal (protein L-R vs CellPhoneDB-v5 metabolite/neurotransmitter contamination) between chimera_human EPI and chimera_monkey EPI cells. The discriminator is abundance/DB-size correction of the family ranking; an agent that ranks families by raw significant-pair count lands on ECM and FAILS. v6.1 self-consistency hard_fails on FGF/NOTCH/WNT retained. All canonical answers reachable from pure analysis; no recall of paper-specific pairs/biology required. See GRADER_DESIGN.md for the v6/v6.1/v6.2 calibration.",
        "ungraded_dependency_stack": [
          "TPM h5ad loading",
          "var_names audit (mixed ENSG + ENSMFAG)",
          "ENSMFAG <-> ENSG ortholog mapping",
          "projection of both chimera_human and chimera_monkey EPI to shared HGNC space",
          "permutation L-R test on a canonical L-R database with chimera_human_EPI and chimera_monkey_EPI as two clusters",
          "running the test in BOTH directions (H->M and M->H)",
          "ranking of significant L-R pairs by score and direction (sender / receiver)",
          "recognising that raw interaction-score ranking is abundance-biased and that CellPhoneDB v5 non-protein (metabolite/neurotransmitter/hormone) interactions contaminate the top of the ranking; filtering to protein L-R or using a protein-only L-R database"
        ]
      },
      "rubricCheckpoints": [
        {
          "index": 1,
          "title": "Task Scope And Answer Contract",
          "decision": "Emit exactly the five required fields \u2014 xspecies_signaling_richness, dominant_signaling_axis, dominant_axis_support (HGNC symbol array), signaling_directionality, and top_signal_interaction_class.",
          "why": "The grader matches controlled tokens exactly and uses dominant_axis_support for self-consistency hard-fails.",
          "successEvidence": "Parseable <EVAL_ANSWER> block with four controlled labels and a JSON-array support list.",
          "failureModes": "Missing support list; free-text labels; malformed block; wrong cohorts or cell types.",
          "scoring": "0 = wrong task/unparseable; 1 = right task with token or support-list drift; 2 = exact five-field contract."
        },
        {
          "index": 2,
          "title": "Cross-Species Ortholog Projection",
          "decision": "Detect the human/macaque union gene axis and map macaque ENSMFAG IDs to shared HGNC symbols before ligand-receptor analysis.",
          "why": "Without ortholog projection, cross-species L-R pairs are lost or silently reduced to one species.",
          "successEvidence": "Explicit ENSMFAG-to-HGNC map or equivalent ortholog resource; retained EPI markers and L-R genes on both species.",
          "failureModes": "Applying human symbols directly to macaque IDs; dropping macaque genes; concluding no signal because the namespaces did not overlap.",
          "scoring": "0 = no cross-species mapping; 1 = partial / brittle mapping; 2 = clean shared HGNC projection."
        },
        {
          "index": 3,
          "title": "EPI Cell Identification And Cohort Selection",
          "decision": "Identify chimera_human EPI and chimera_monkey EPI cells, respecting the cohort labels and the available developmental stages.",
          "why": "The eval asks specifically about EPI-to-EPI cross-species signaling, not all chimera cells, all monkey cells, or control-vs-chimera communication.",
          "successEvidence": "Marker- or cluster-supported EPI subset in both species; the analysis excludes unrelated lineages from the target L-R comparison.",
          "failureModes": "Pooled all-lineage communication; comparing chimera_human to control_monkey; using control_monkey as the receiver/sender in the graded comparison.",
          "scoring": "0 = wrong compartments; 1 = broad embryonic subset with weak EPI resolution; 2 = EPI-to-EPI target recovered."
        },
        {
          "index": 4,
          "title": "Bidirectional Ligand-Receptor Testing",
          "decision": "Test both chimera_human -> chimera_monkey and chimera_monkey -> chimera_human directions with a credible L-R method.",
          "why": "Direction-resolved results are required for the directionality axis and for recognizing that family leaders differ by direction.",
          "successEvidence": "Separate sender/receiver analyses, permutation or comparable significance testing, and per-direction significant-pair summaries.",
          "failureModes": "Only one direction tested; direction inferred from expression levels alone; no significance / null model.",
          "scoring": "0 = no credible L-R test; 1 = one direction or weak null model; 2 = both directions tested with significance evidence."
        },
        {
          "index": 5,
          "title": "Richness And Protein-LR Signal Recognition",
          "decision": "Establish that cross-species EPI communication contains many significant protein ligand-receptor pairs across multiple families.",
          "why": "These are the easier positive-signal axes; failing them usually indicates a broken ortholog map or inappropriate database handling.",
          "successEvidence": "Multiple significant pairs in both directions, family diversity, and top-ranked interactions characterized as protein L-R after filtering/flagging non-protein entries.",
          "failureModes": "No-signal call from namespace failure; reporting only metabolite/neurotransmitter pseudo-interactions; loose-threshold-only signal.",
          "scoring": "0 = no credible signal / non-protein top signal; 1 = signal found but family structure or interaction class weak; 2 = rich multi-family protein L-R signal."
        },
        {
          "index": 6,
          "title": "Dominant-Family Ranking Method",
          "decision": "Compare signaling families using a metric that accounts for raw abundance and L-R database family size, and inspect the two directions separately before making an overall dominant-axis call.",
          "why": "This is the load-bearing analysis gap and the only axis that separates careful conventional runs from passing expert runs.",
          "successEvidence": "DB-size-controlled enrichment, denominator-aware family comparison, or equivalent reasoning showing direction-specific leaders and no robust single family.",
          "failureModes": "Raw significant-pair count; raw interaction score; expression-magnitude ranking; specificity ranking used as a substitute for family-size normalization; no direction-resolved comparison.",
          "scoring": "0 = raw/abundance ranking drives a single-family call; 1 = partial correction but still overcommits to one family; 2 = denominator-aware, direction-resolved co-dominance conclusion."
        },
        {
          "index": 7,
          "title": "Dominant Axis And Support Consistency",
          "decision": "Choose two_or_more_families_co_dominant or no_single_family_dominates when the family ranking is not robustly singular, and populate dominant_axis_support with genes from the significant pairs that actually drove the conclusion.",
          "why": "The dominant axis is 2.5/6 points and pass requires at least 1.0 on it; the support list guards against unsupported family claims.",
          "successEvidence": "Support list spans multiple engaged families or otherwise demonstrates why no single family dominates; WNT/FGF/NOTCH claims, if made, include real family genes.",
          "failureModes": "ECM dominance; any single-family dominance presented as the overall answer; WNT/FGF/NOTCH label with no corresponding support gene.",
          "scoring": "0 = abundance-trap or unsupported family claim; 1 = real enriched family but overcommitted; 2 = co-dominance / no-single-family answer with coherent support."
        },
        {
          "index": 8,
          "title": "Vocabulary Discipline And Schema Compliance",
          "decision": "Use exact controlled vocabulary tokens and HGNC symbols in a parseable final answer.",
          "why": "Predicates match by exact strings, and the support-list hard fails inspect HGNC symbols directly.",
          "successEvidence": "Exact labels from vocabulary.json; support list contains gene symbols rather than family names or prose.",
          "failureModes": "Synonyms for controlled labels; Ensembl IDs in support list; support list contains pathway names only.",
          "scoring": "0 = not gradable / off-vocabulary core labels; 1 = parseable with drift damaging one axis; 2 = exact controlled-vocabulary answer."
        }
      ],
      "runs": [
        {
          "runId": "claude-opus-4-7__claude-code",
          "provider": "Anthropic",
          "model": "claude-opus-4-7",
          "harness": "claude-code",
          "run": "r1",
          "trialIndex": 1,
          "passed": false,
          "score": 3.75,
          "reasoning": "all_of [pass_rule=score_threshold]: FAIL\n  + richness_weighted: passed=True, score=1.5\n  + dominant_axis_weighted: passed=True, score=0.25\n  + signaling_directionality_weighted: passed=True, score=1.0\n  + top_signal_interaction_class_weighted: passed=True, score=1.0\n  + hard_fail 'dominant_axis_wnt_unsupported_by_own_pairs': not triggered\n  + hard_fail 'dominant_axis_fgf_unsupported_by_own_pairs': not triggered\n  + hard_fail 'dominant_axis_notch_unsupported_by_own_pairs': not triggered",
          "metrics": {
            "pass_rule": "score_threshold",
            "scoring_count": 4,
            "scoring_passed": 4,
            "scoring_total_score": 3.75,
            "hard_fail_triggered": []
          },
          "fieldScores": {
            "richness_weighted": 1.5,
            "dominant_axis_weighted": 0.25,
            "signaling_directionality_weighted": 1,
            "top_signal_interaction_class_weighted": 1
          },
          "graderChecks": [
            {
              "name": "richness_weighted",
              "kind": "gate",
              "passed": true,
              "score": 1.5
            },
            {
              "name": "dominant_axis_weighted",
              "kind": "gate",
              "passed": true,
              "score": 0.25
            },
            {
              "name": "signaling_directionality_weighted",
              "kind": "gate",
              "passed": true,
              "score": 1
            },
            {
              "name": "top_signal_interaction_class_weighted",
              "kind": "gate",
              "passed": true,
              "score": 1
            },
            {
              "name": "dominant_axis_wnt_unsupported_by_own_pairs",
              "kind": "hard_fail",
              "passed": true,
              "triggered": false
            },
            {
              "name": "dominant_axis_fgf_unsupported_by_own_pairs",
              "kind": "hard_fail",
              "passed": true,
              "triggered": false
            },
            {
              "name": "dominant_axis_notch_unsupported_by_own_pairs",
              "kind": "hard_fail",
              "passed": true,
              "triggered": false
            }
          ],
          "agentAnswer": {
            "dominant_axis_support": [
              "AGRN",
              "ANGPT1",
              "APOE",
              "CALR",
              "CD44",
              "CD47",
              "COL11A1",
              "COL3A1",
              "COL4A2",
              "DDR1",
              "FAM3C",
              "FBLN1",
              "FN1",
              "HMGB1",
              "ITGA4",
              "ITGA6",
              "ITGAV",
              "ITGB1",
              "ITGB3BP",
              "ITGB5",
              "LAMA1",
              "LAMA2",
              "LAMB1",
              "LAMB2",
              "LAMP1",
              "LGALS1",
              "MDK",
              "MFGE8",
              "PKM",
              "PLAU",
              "RPSA",
              "SDC1",
              "SDC2",
              "SDC3",
              "SDC4",
              "SPP1",
              "THBS2"
            ],
            "dominant_signaling_axis": "ECM_collagen_integrin_family",
            "signaling_directionality": "substantial_signal_in_both_directions",
            "top_signal_interaction_class": "protein_ligand_receptor_dominant",
            "xspecies_signaling_richness": "many_significant_pairs_across_multiple_signalling_families"
          },
          "durationS": 1503.37,
          "cost": 5.38779975,
          "nTurns": 83,
          "checkpointSignals": {
            "clustering": false,
            "markerAnnotation": true,
            "scoreBasedAnnotation": false,
            "normalization": true,
            "nerveJointAnchor": false,
            "multiomeAssembly": false,
            "depthNormalizedAtac": false,
            "regulonInference": false,
            "cd8SubsetIntegration": false,
            "perPatientEnrichment": false,
            "tcrRichnessMetrics": true,
            "orthologProjection": true,
            "epiSubsetSelection": true,
            "bidirectionalLrTest": true,
            "enrichmentFamilyRanking": false,
            "mousePseudobulkDesign": false,
            "cancerEpithelialDenominator": true,
            "geneLevelDe": false,
            "agingSignatureControl": false,
            "senescencePanelScoring": false
          },
          "expectedAnswer": {
            "passRule": "score_threshold",
            "checks": [
              {
                "name": "richness_weighted",
                "role": "additive",
                "field": "$.xspecies_signaling_richness",
                "kind": "other",
                "op": "weighted_label",
                "expected": null
              },
              {
                "name": "dominant_axis_weighted",
                "role": "additive",
                "field": "$.dominant_signaling_axis",
                "kind": "other",
                "op": "weighted_label",
                "expected": null
              },
              {
                "name": "dominant_axis_wnt_unsupported_by_own_pairs",
                "role": "hard_fail",
                "field": null,
                "kind": "fields",
                "expected": {
                  "dominant_signaling_axis": {
                    "op": "equals",
                    "values": [
                      "WNT_FZD_LRP_family"
                    ]
                  }
                }
              },
              {
                "name": "dominant_axis_fgf_unsupported_by_own_pairs",
                "role": "hard_fail",
                "field": null,
                "kind": "fields",
                "expected": {
                  "dominant_signaling_axis": {
                    "op": "equals",
                    "values": [
                      "FGF_FGFR_family"
                    ]
                  }
                }
              },
              {
                "name": "dominant_axis_notch_unsupported_by_own_pairs",
                "role": "hard_fail",
                "field": null,
                "kind": "fields",
                "expected": {
                  "dominant_signaling_axis": {
                    "op": "equals",
                    "values": [
                      "NOTCH_DLL_JAG_family"
                    ]
                  }
                }
              },
              {
                "name": "signaling_directionality_weighted",
                "role": "additive",
                "field": "$.signaling_directionality",
                "kind": "other",
                "op": "weighted_label",
                "expected": null
              },
              {
                "name": "top_signal_interaction_class_weighted",
                "role": "additive",
                "field": "$.top_signal_interaction_class",
                "kind": "other",
                "op": "weighted_label",
                "expected": null
              }
            ]
          },
          "trajectoryFile": "scbench-long/trajectories/gse155381_q7_xspecies_epi_communication__claude-opus-4-7__claude-code__r1.json"
        },
        {
          "runId": "gemini-3.5-flash__pi",
          "provider": "Google",
          "model": "gemini-3.5-flash",
          "harness": "pi",
          "run": "r1",
          "trialIndex": 1,
          "passed": false,
          "score": 4.25,
          "reasoning": "all_of [pass_rule=score_threshold]: FAIL\n  + richness_weighted: passed=True, score=1.5\n  + dominant_axis_weighted: passed=True, score=0.75\n  + signaling_directionality_weighted: passed=True, score=1.0\n  + top_signal_interaction_class_weighted: passed=True, score=1.0\n  + hard_fail 'dominant_axis_wnt_unsupported_by_own_pairs': not triggered\n  + hard_fail 'dominant_axis_fgf_unsupported_by_own_pairs': not triggered\n  + hard_fail 'dominant_axis_notch_unsupported_by_own_pairs': not triggered",
          "metrics": {
            "pass_rule": "score_threshold",
            "scoring_count": 4,
            "scoring_passed": 4,
            "scoring_total_score": 4.25,
            "hard_fail_triggered": []
          },
          "fieldScores": {
            "richness_weighted": 1.5,
            "dominant_axis_weighted": 0.75,
            "signaling_directionality_weighted": 1,
            "top_signal_interaction_class_weighted": 1
          },
          "graderChecks": [
            {
              "name": "richness_weighted",
              "kind": "gate",
              "passed": true,
              "score": 1.5
            },
            {
              "name": "dominant_axis_weighted",
              "kind": "gate",
              "passed": true,
              "score": 0.75
            },
            {
              "name": "signaling_directionality_weighted",
              "kind": "gate",
              "passed": true,
              "score": 1
            },
            {
              "name": "top_signal_interaction_class_weighted",
              "kind": "gate",
              "passed": true,
              "score": 1
            },
            {
              "name": "dominant_axis_wnt_unsupported_by_own_pairs",
              "kind": "hard_fail",
              "passed": true,
              "triggered": false
            },
            {
              "name": "dominant_axis_fgf_unsupported_by_own_pairs",
              "kind": "hard_fail",
              "passed": true,
              "triggered": false
            },
            {
              "name": "dominant_axis_notch_unsupported_by_own_pairs",
              "kind": "hard_fail",
              "passed": true,
              "triggered": false
            }
          ],
          "agentAnswer": {
            "dominant_axis_support": [
              "EFNA1",
              "EFNA2",
              "EFNA3",
              "EFNA4",
              "EFNA5",
              "EFNB1",
              "EPHA1",
              "EPHA2",
              "EPHA4",
              "EPHA5",
              "EPHA7",
              "EPHB1",
              "EPHB2",
              "EPHB3",
              "EPHB4"
            ],
            "dominant_signaling_axis": "ephrin_EPH_axon_guidance_family",
            "signaling_directionality": "substantial_signal_in_both_directions",
            "top_signal_interaction_class": "protein_ligand_receptor_dominant",
            "xspecies_signaling_richness": "many_significant_pairs_across_multiple_signalling_families"
          },
          "durationS": 666.21,
          "cost": 1.48338465,
          "nTurns": 87,
          "checkpointSignals": {
            "clustering": false,
            "markerAnnotation": true,
            "scoreBasedAnnotation": false,
            "normalization": true,
            "nerveJointAnchor": false,
            "multiomeAssembly": true,
            "depthNormalizedAtac": false,
            "regulonInference": false,
            "cd8SubsetIntegration": true,
            "perPatientEnrichment": false,
            "tcrRichnessMetrics": true,
            "orthologProjection": true,
            "epiSubsetSelection": true,
            "bidirectionalLrTest": true,
            "enrichmentFamilyRanking": true,
            "mousePseudobulkDesign": true,
            "cancerEpithelialDenominator": true,
            "geneLevelDe": true,
            "agingSignatureControl": false,
            "senescencePanelScoring": false
          },
          "expectedAnswer": {
            "passRule": "score_threshold",
            "checks": [
              {
                "name": "richness_weighted",
                "role": "additive",
                "field": "$.xspecies_signaling_richness",
                "kind": "other",
                "op": "weighted_label",
                "expected": null
              },
              {
                "name": "dominant_axis_weighted",
                "role": "additive",
                "field": "$.dominant_signaling_axis",
                "kind": "other",
                "op": "weighted_label",
                "expected": null
              },
              {
                "name": "dominant_axis_wnt_unsupported_by_own_pairs",
                "role": "hard_fail",
                "field": null,
                "kind": "fields",
                "expected": {
                  "dominant_signaling_axis": {
                    "op": "equals",
                    "values": [
                      "WNT_FZD_LRP_family"
                    ]
                  }
                }
              },
              {
                "name": "dominant_axis_fgf_unsupported_by_own_pairs",
                "role": "hard_fail",
                "field": null,
                "kind": "fields",
                "expected": {
                  "dominant_signaling_axis": {
                    "op": "equals",
                    "values": [
                      "FGF_FGFR_family"
                    ]
                  }
                }
              },
              {
                "name": "dominant_axis_notch_unsupported_by_own_pairs",
                "role": "hard_fail",
                "field": null,
                "kind": "fields",
                "expected": {
                  "dominant_signaling_axis": {
                    "op": "equals",
                    "values": [
                      "NOTCH_DLL_JAG_family"
                    ]
                  }
                }
              },
              {
                "name": "signaling_directionality_weighted",
                "role": "additive",
                "field": "$.signaling_directionality",
                "kind": "other",
                "op": "weighted_label",
                "expected": null
              },
              {
                "name": "top_signal_interaction_class_weighted",
                "role": "additive",
                "field": "$.top_signal_interaction_class",
                "kind": "other",
                "op": "weighted_label",
                "expected": null
              }
            ]
          },
          "trajectoryFile": "scbench-long/trajectories/gse155381_q7_xspecies_epi_communication__gemini-3.5-flash__pi__r1.json"
        },
        {
          "runId": "gpt-5.5__openai-codex",
          "provider": "OpenAI",
          "model": "gpt-5.5",
          "harness": "openai-codex",
          "run": "r1",
          "trialIndex": 1,
          "passed": false,
          "score": 3.75,
          "reasoning": "all_of [pass_rule=score_threshold]: FAIL\n  + richness_weighted: passed=True, score=1.5\n  + dominant_axis_weighted: passed=True, score=0.25\n  + signaling_directionality_weighted: passed=True, score=1.0\n  + top_signal_interaction_class_weighted: passed=True, score=1.0\n  + hard_fail 'dominant_axis_wnt_unsupported_by_own_pairs': not triggered\n  + hard_fail 'dominant_axis_fgf_unsupported_by_own_pairs': not triggered\n  + hard_fail 'dominant_axis_notch_unsupported_by_own_pairs': not triggered",
          "metrics": {
            "pass_rule": "score_threshold",
            "scoring_count": 4,
            "scoring_passed": 4,
            "scoring_total_score": 3.75,
            "hard_fail_triggered": []
          },
          "fieldScores": {
            "richness_weighted": 1.5,
            "dominant_axis_weighted": 0.25,
            "signaling_directionality_weighted": 1,
            "top_signal_interaction_class_weighted": 1
          },
          "graderChecks": [
            {
              "name": "richness_weighted",
              "kind": "gate",
              "passed": true,
              "score": 1.5
            },
            {
              "name": "dominant_axis_weighted",
              "kind": "gate",
              "passed": true,
              "score": 0.25
            },
            {
              "name": "signaling_directionality_weighted",
              "kind": "gate",
              "passed": true,
              "score": 1
            },
            {
              "name": "top_signal_interaction_class_weighted",
              "kind": "gate",
              "passed": true,
              "score": 1
            },
            {
              "name": "dominant_axis_wnt_unsupported_by_own_pairs",
              "kind": "hard_fail",
              "passed": true,
              "triggered": false
            },
            {
              "name": "dominant_axis_fgf_unsupported_by_own_pairs",
              "kind": "hard_fail",
              "passed": true,
              "triggered": false
            },
            {
              "name": "dominant_axis_notch_unsupported_by_own_pairs",
              "kind": "hard_fail",
              "passed": true,
              "triggered": false
            }
          ],
          "agentAnswer": {
            "dominant_axis_support": [
              "MDK",
              "ITGA6",
              "ITGB1",
              "ITGA4",
              "COL4A2",
              "SDC4",
              "FN1",
              "ITGA5",
              "SPP1",
              "ITGAV",
              "THBS1",
              "CD47",
              "LAMB1",
              "COL4A1",
              "ITGB8"
            ],
            "dominant_signaling_axis": "ECM_collagen_integrin_family",
            "signaling_directionality": "substantial_signal_in_both_directions",
            "top_signal_interaction_class": "protein_ligand_receptor_dominant",
            "xspecies_signaling_richness": "many_significant_pairs_across_multiple_signalling_families"
          },
          "durationS": 703.33,
          "cost": 2.474512,
          "nTurns": 1,
          "checkpointSignals": {
            "clustering": false,
            "markerAnnotation": true,
            "scoreBasedAnnotation": false,
            "normalization": true,
            "nerveJointAnchor": false,
            "multiomeAssembly": false,
            "depthNormalizedAtac": false,
            "regulonInference": false,
            "cd8SubsetIntegration": false,
            "perPatientEnrichment": false,
            "tcrRichnessMetrics": true,
            "orthologProjection": true,
            "epiSubsetSelection": true,
            "bidirectionalLrTest": true,
            "enrichmentFamilyRanking": true,
            "mousePseudobulkDesign": false,
            "cancerEpithelialDenominator": true,
            "geneLevelDe": true,
            "agingSignatureControl": false,
            "senescencePanelScoring": false
          },
          "expectedAnswer": {
            "passRule": "score_threshold",
            "checks": [
              {
                "name": "richness_weighted",
                "role": "additive",
                "field": "$.xspecies_signaling_richness",
                "kind": "other",
                "op": "weighted_label",
                "expected": null
              },
              {
                "name": "dominant_axis_weighted",
                "role": "additive",
                "field": "$.dominant_signaling_axis",
                "kind": "other",
                "op": "weighted_label",
                "expected": null
              },
              {
                "name": "dominant_axis_wnt_unsupported_by_own_pairs",
                "role": "hard_fail",
                "field": null,
                "kind": "fields",
                "expected": {
                  "dominant_signaling_axis": {
                    "op": "equals",
                    "values": [
                      "WNT_FZD_LRP_family"
                    ]
                  }
                }
              },
              {
                "name": "dominant_axis_fgf_unsupported_by_own_pairs",
                "role": "hard_fail",
                "field": null,
                "kind": "fields",
                "expected": {
                  "dominant_signaling_axis": {
                    "op": "equals",
                    "values": [
                      "FGF_FGFR_family"
                    ]
                  }
                }
              },
              {
                "name": "dominant_axis_notch_unsupported_by_own_pairs",
                "role": "hard_fail",
                "field": null,
                "kind": "fields",
                "expected": {
                  "dominant_signaling_axis": {
                    "op": "equals",
                    "values": [
                      "NOTCH_DLL_JAG_family"
                    ]
                  }
                }
              },
              {
                "name": "signaling_directionality_weighted",
                "role": "additive",
                "field": "$.signaling_directionality",
                "kind": "other",
                "op": "weighted_label",
                "expected": null
              },
              {
                "name": "top_signal_interaction_class_weighted",
                "role": "additive",
                "field": "$.top_signal_interaction_class",
                "kind": "other",
                "op": "weighted_label",
                "expected": null
              }
            ]
          },
          "trajectoryFile": "scbench-long/trajectories/gse155381_q7_xspecies_epi_communication__gpt-5.5__openai-codex__r1.json"
        },
        {
          "runId": "grok-4.3__pi",
          "provider": "SpaceXAI",
          "model": "grok-4.3",
          "harness": "pi",
          "run": "r1",
          "trialIndex": 1,
          "passed": false,
          "score": 1.25,
          "reasoning": "all_of [pass_rule=score_threshold]: FAIL\n  + richness_weighted: passed=True, score=0.25\n  + dominant_axis_weighted: passed=True, score=0.5\n  + signaling_directionality_weighted: passed=True, score=0.25\n  + top_signal_interaction_class_weighted: passed=True, score=0.25\n  + hard_fail 'dominant_axis_wnt_unsupported_by_own_pairs': not triggered\n  + hard_fail 'dominant_axis_fgf_unsupported_by_own_pairs': not triggered\n  + hard_fail 'dominant_axis_notch_unsupported_by_own_pairs': not triggered",
          "metrics": {
            "pass_rule": "score_threshold",
            "scoring_count": 4,
            "scoring_passed": 4,
            "scoring_total_score": 1.25,
            "hard_fail_triggered": []
          },
          "fieldScores": {
            "richness_weighted": 0.25,
            "dominant_axis_weighted": 0.5,
            "signaling_directionality_weighted": 0.25,
            "top_signal_interaction_class_weighted": 0.25
          },
          "graderChecks": [
            {
              "name": "richness_weighted",
              "kind": "gate",
              "passed": true,
              "score": 0.25
            },
            {
              "name": "dominant_axis_weighted",
              "kind": "gate",
              "passed": true,
              "score": 0.5
            },
            {
              "name": "signaling_directionality_weighted",
              "kind": "gate",
              "passed": true,
              "score": 0.25
            },
            {
              "name": "top_signal_interaction_class_weighted",
              "kind": "gate",
              "passed": true,
              "score": 0.25
            },
            {
              "name": "dominant_axis_wnt_unsupported_by_own_pairs",
              "kind": "hard_fail",
              "passed": true,
              "triggered": false
            },
            {
              "name": "dominant_axis_fgf_unsupported_by_own_pairs",
              "kind": "hard_fail",
              "passed": true,
              "triggered": false
            },
            {
              "name": "dominant_axis_notch_unsupported_by_own_pairs",
              "kind": "hard_fail",
              "passed": true,
              "triggered": false
            }
          ],
          "agentAnswer": {
            "dominant_axis_support": [],
            "dominant_signaling_axis": "cannot_assess",
            "signaling_directionality": "cannot_assess",
            "top_signal_interaction_class": "cannot_assess",
            "xspecies_signaling_richness": "cannot_assess"
          },
          "durationS": 25.01,
          "cost": 0.007651650000000001,
          "nTurns": 5,
          "checkpointSignals": {
            "clustering": false,
            "markerAnnotation": false,
            "scoreBasedAnnotation": false,
            "normalization": true,
            "nerveJointAnchor": false,
            "multiomeAssembly": false,
            "depthNormalizedAtac": false,
            "regulonInference": false,
            "cd8SubsetIntegration": false,
            "perPatientEnrichment": false,
            "tcrRichnessMetrics": true,
            "orthologProjection": false,
            "epiSubsetSelection": true,
            "bidirectionalLrTest": true,
            "enrichmentFamilyRanking": true,
            "mousePseudobulkDesign": false,
            "cancerEpithelialDenominator": false,
            "geneLevelDe": false,
            "agingSignatureControl": false,
            "senescencePanelScoring": false
          },
          "expectedAnswer": {
            "passRule": "score_threshold",
            "checks": [
              {
                "name": "richness_weighted",
                "role": "additive",
                "field": "$.xspecies_signaling_richness",
                "kind": "other",
                "op": "weighted_label",
                "expected": null
              },
              {
                "name": "dominant_axis_weighted",
                "role": "additive",
                "field": "$.dominant_signaling_axis",
                "kind": "other",
                "op": "weighted_label",
                "expected": null
              },
              {
                "name": "dominant_axis_wnt_unsupported_by_own_pairs",
                "role": "hard_fail",
                "field": null,
                "kind": "fields",
                "expected": {
                  "dominant_signaling_axis": {
                    "op": "equals",
                    "values": [
                      "WNT_FZD_LRP_family"
                    ]
                  }
                }
              },
              {
                "name": "dominant_axis_fgf_unsupported_by_own_pairs",
                "role": "hard_fail",
                "field": null,
                "kind": "fields",
                "expected": {
                  "dominant_signaling_axis": {
                    "op": "equals",
                    "values": [
                      "FGF_FGFR_family"
                    ]
                  }
                }
              },
              {
                "name": "dominant_axis_notch_unsupported_by_own_pairs",
                "role": "hard_fail",
                "field": null,
                "kind": "fields",
                "expected": {
                  "dominant_signaling_axis": {
                    "op": "equals",
                    "values": [
                      "NOTCH_DLL_JAG_family"
                    ]
                  }
                }
              },
              {
                "name": "signaling_directionality_weighted",
                "role": "additive",
                "field": "$.signaling_directionality",
                "kind": "other",
                "op": "weighted_label",
                "expected": null
              },
              {
                "name": "top_signal_interaction_class_weighted",
                "role": "additive",
                "field": "$.top_signal_interaction_class",
                "kind": "other",
                "op": "weighted_label",
                "expected": null
              }
            ]
          },
          "trajectoryFile": "scbench-long/trajectories/gse155381_q7_xspecies_epi_communication__grok-4.3__pi__r1.json"
        }
      ],
      "sourcePath": "gse155381_q7_xspecies_epi_communication/eval.json",
      "githubUrl": "https://github.com/latchbio/scbench-long/blob/main/evals/gse155381_q7_xspecies_epi_communication/eval.json"
    }
  ],
  "verifiedModelResults": [
    {
      "id": "sclong-gpt-5-6-sol-codex",
      "modelName": "gpt-5.6-sol",
      "harness": "openai-codex",
      "passRate": 43.94,
      "ci": [
        32.63,
        55.92
      ],
      "passed": 29,
      "runs": 66,
      "avgCost": null,
      "avgDuration": null,
      "nEvals": 22,
      "subset": "verified"
    },
    {
      "id": "sclong-gpt-6-astra-codex",
      "modelName": "gpt-6-astra",
      "harness": "openai-codex",
      "passRate": 42.42,
      "ci": [
        31.24,
        54.44
      ],
      "passed": 28,
      "runs": 66,
      "avgCost": 5.1424,
      "avgDuration": null,
      "nEvals": 22,
      "subset": "verified"
    },
    {
      "id": "sclong-gpt-5-6-sol-pi",
      "modelName": "gpt-5.6-sol",
      "harness": "pi",
      "passRate": 40.91,
      "ci": [
        29.87,
        52.95
      ],
      "passed": 27,
      "runs": 66,
      "avgCost": 5.5607,
      "avgDuration": null,
      "nEvals": 22,
      "subset": "verified"
    },
    {
      "id": "sclong-gpt-6-astra-pi",
      "modelName": "gpt-6-astra",
      "harness": "pi",
      "passRate": 39.39,
      "ci": [
        28.5,
        51.45
      ],
      "passed": 26,
      "runs": 66,
      "avgCost": 1.1122,
      "avgDuration": null,
      "nEvals": 22,
      "subset": "verified"
    },
    {
      "id": "sclong-opus-5",
      "modelName": "claude-opus-5",
      "harness": "pi",
      "passRate": 36.36,
      "ci": [
        25.81,
        48.42
      ],
      "passed": 24,
      "runs": 66,
      "avgCost": 6.7505,
      "avgDuration": null,
      "nEvals": 22,
      "subset": "verified"
    },
    {
      "id": "sclong-gpt-5-6-terra-pi",
      "modelName": "gpt-5.6-terra",
      "harness": "pi",
      "passRate": 34.85,
      "ci": [
        24.48,
        46.89
      ],
      "passed": 23,
      "runs": 66,
      "avgCost": 3.6395,
      "avgDuration": null,
      "nEvals": 22,
      "subset": "verified"
    },
    {
      "id": "sclong-gpt-5-6-terra-codex",
      "modelName": "gpt-5.6-terra",
      "harness": "openai-codex",
      "passRate": 31.82,
      "ci": [
        21.85,
        43.79
      ],
      "passed": 21,
      "runs": 66,
      "avgCost": null,
      "avgDuration": null,
      "nEvals": 22,
      "subset": "verified"
    },
    {
      "id": "sclong-opus-5-claude-code",
      "modelName": "claude-opus-5",
      "harness": "claude-code",
      "passRate": 30.3,
      "ci": [
        20.55,
        42.22
      ],
      "passed": 20,
      "runs": 66,
      "avgCost": 9.6716,
      "avgDuration": null,
      "nEvals": 22,
      "subset": "verified"
    },
    {
      "id": "sclong-gpt-5-6-luna-pi",
      "modelName": "gpt-5.6-luna",
      "harness": "pi",
      "passRate": 28.79,
      "ci": [
        19.27,
        40.64
      ],
      "passed": 19,
      "runs": 66,
      "avgCost": 0.3403,
      "avgDuration": null,
      "nEvals": 22,
      "subset": "verified"
    },
    {
      "id": "sclong-grok-4-6-grok-build",
      "modelName": "grok-4.6",
      "harness": "grok_build",
      "passRate": 27.27,
      "ci": [
        18,
        39.04
      ],
      "passed": 18,
      "runs": 66,
      "avgCost": 0.8899,
      "avgDuration": null,
      "nEvals": 22,
      "subset": "verified"
    },
    {
      "id": "sclong-deepseek-v4-1-flash-pi",
      "modelName": "deepseek-v4.1-flash",
      "harness": "pi",
      "passRate": 27.27,
      "ci": [
        18.0,
        39.04
      ],
      "passed": 18,
      "runs": 66,
      "avgCost": 0.208,
      "avgDuration": null,
      "nEvals": 22,
      "subset": "verified"
    },
    {
      "id": "sclong-opus-4-8-pi",
      "modelName": "claude-opus-4-8",
      "harness": "pi",
      "passRate": 25.76,
      "ci": [
        16.75,
        37.43
      ],
      "passed": 17,
      "runs": 66,
      "avgCost": 5.226,
      "avgDuration": null,
      "nEvals": 22,
      "subset": "verified"
    },
    {
      "id": "sclong-grok-4-5-pi",
      "modelName": "grok-4.5",
      "harness": "pi",
      "passRate": 25.76,
      "ci": [
        16.75,
        37.43
      ],
      "passed": 17,
      "runs": 66,
      "avgCost": 0.6165,
      "avgDuration": null,
      "nEvals": 22,
      "subset": "verified"
    },
    {
      "id": "sclong-gemini-3-5-flash-pi",
      "modelName": "gemini-3.5-flash",
      "harness": "pi",
      "passRate": 24.24,
      "ci": [
        15.51,
        35.81
      ],
      "passed": 16,
      "runs": 66,
      "avgCost": 1.4843,
      "avgDuration": null,
      "nEvals": 22,
      "subset": "verified"
    },
    {
      "id": "sclong-gemini-3-7-flash-pi",
      "modelName": "gemini-3.7-flash",
      "harness": "pi",
      "passRate": 24.24,
      "ci": [
        15.51,
        35.81
      ],
      "passed": 16,
      "runs": 66,
      "avgCost": 0.697,
      "avgDuration": null,
      "nEvals": 22,
      "subset": "verified"
    },
    {
      "id": "sclong-grok-4-6-pi",
      "modelName": "grok-4.6",
      "harness": "pi",
      "passRate": 24.24,
      "ci": [
        15.51,
        35.81
      ],
      "passed": 16,
      "runs": 66,
      "avgCost": 1.9442,
      "avgDuration": null,
      "nEvals": 22,
      "subset": "verified"
    },
    {
      "id": "sclong-gpt-5-5-pi",
      "modelName": "gpt-5.5",
      "harness": "pi",
      "passRate": 22.73,
      "ci": [
        14.29,
        34.17
      ],
      "passed": 15,
      "runs": 66,
      "avgCost": 2.3761,
      "avgDuration": null,
      "nEvals": 22,
      "subset": "verified"
    },
    {
      "id": "sclong-opus-4-8-claude-code",
      "modelName": "claude-opus-4-8",
      "harness": "claude-code",
      "passRate": 21.21,
      "ci": [
        13.08,
        32.51
      ],
      "passed": 14,
      "runs": 66,
      "avgCost": 6.4859,
      "avgDuration": null,
      "nEvals": 22,
      "subset": "verified"
    },
    {
      "id": "sclong-sonnet-5-pi",
      "modelName": "claude-sonnet-5",
      "harness": "pi",
      "passRate": 19.7,
      "ci": [
        11.89,
        30.84
      ],
      "passed": 13,
      "runs": 66,
      "avgCost": 3.9857,
      "avgDuration": null,
      "nEvals": 22,
      "subset": "verified"
    },
    {
      "id": "sclong-sonnet-5-claude-code",
      "modelName": "claude-sonnet-5",
      "harness": "claude-code",
      "passRate": 16.67,
      "ci": [
        9.57,
        27.43
      ],
      "passed": 11,
      "runs": 66,
      "avgCost": 4.5797,
      "avgDuration": null,
      "nEvals": 22,
      "subset": "verified"
    },
    {
      "id": "sclong-gpt-5-5-codex",
      "modelName": "gpt-5.5",
      "harness": "openai-codex",
      "passRate": 16.67,
      "ci": [
        9.57,
        27.43
      ],
      "passed": 11,
      "runs": 66,
      "avgCost": 2.4576,
      "avgDuration": null,
      "nEvals": 22,
      "subset": "verified"
    },
    {
      "id": "sclong-kimi-k3-pi",
      "modelName": "kimi-k3",
      "harness": "pi",
      "passRate": 16.67,
      "ci": [
        9.57,
        27.43
      ],
      "passed": 11,
      "runs": 66,
      "avgCost": 2.5442,
      "avgDuration": null,
      "nEvals": 22,
      "subset": "verified"
    },
    {
      "id": "sclong-gpt-5-6-luna-codex",
      "modelName": "gpt-5.6-luna",
      "harness": "openai-codex",
      "passRate": 12.12,
      "ci": [
        6.27,
        22.14
      ],
      "passed": 8,
      "runs": 66,
      "avgCost": null,
      "avgDuration": null,
      "nEvals": 22,
      "subset": "verified"
    },
    {
      "id": "sclong-opus-4-7-claude-code",
      "modelName": "claude-opus-4-7",
      "harness": "claude-code",
      "passRate": 10.61,
      "ci": [
        5.23,
        20.31
      ],
      "passed": 7,
      "runs": 66,
      "avgCost": null,
      "avgDuration": null,
      "nEvals": 22,
      "subset": "verified"
    }
  ]
}
